diff --git a/deploy/all.sh b/deploy/all.sh index cedbb5b8bfc7ef363c1d60a9d8f2b6cef63be384..f93cd92ac5e3189b0dc8fa71d74a586e929aaecc 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -29,9 +29,12 @@ export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"} # By default, only basic components are deployed export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice nbi webui load_generator"} -# Uncomment to activate Monitoring +# Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" +# Uncomment to activate Monitoring Framework (new) +#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api" + # Uncomment to activate BGP-LS Speaker #export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" diff --git a/deploy/kafka.sh b/deploy/kafka.sh new file mode 100755 index 0000000000000000000000000000000000000000..4a91bfc9e657d1b8a6a548b9c0a81a2f8a0b45e0 --- /dev/null +++ b/deploy/kafka.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Read deployment settings +######################################################################################################################## + +# If not already set, set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} + + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +# Constants +TMP_FOLDER="./tmp" +KFK_MANIFESTS_PATH="manifests/kafka" +KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml" +KFK_MANIFEST="02-kafka.yaml" + +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" +mkdir -p ${TMP_MANIFESTS_FOLDER} + +# copy zookeeper and kafka manifest files to temporary manifest location +cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" +cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" + +echo "Apache Kafka Namespace" +echo ">>> Delete Apache Kafka Namespace" +kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found + +echo ">>> Create Apache Kafka Namespace" +kubectl create namespace ${KFK_NAMESPACE} + +echo ">>> Deplying Apache Kafka Zookeeper" +# Kafka zookeeper service should be deployed before the kafka service +kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" + +KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically +KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') + +# Kafka service should be deployed after the zookeeper service +sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" + +echo ">>> Deploying Apache Kafka Broker" +kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" + +echo ">>> Verifing Apache Kafka deployment" +sleep 10 +KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods) +if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then + echo "Deployment Error: \n $KFK_PODS_STATUS" +else + echo "$KFK_PODS_STATUS" +fi \ No newline at end of file diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 04895f98448694c4eca3861079df09c366b0000f..f61cdb991b09b389b20fbb82f619c0f8f3b4cdc2 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -27,7 +27,7 @@ export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"} # If not already set, set the list of components, separated by spaces, you want to build images for, and deploy. # By default, only basic components are deployed -export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device ztp monitoring pathcomp service slice nbi webui load_generator"} +export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice nbi webui load_generator"} # If not already set, set the tag you want to use for your images. export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"} @@ -137,10 +137,23 @@ printf "\n" echo "Create secret with CockroachDB data" CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') +CRDB_DATABASE_CONTEXT=${CRDB_DATABASE} # TODO: change by specific configurable environment variable kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ - --from-literal=CRDB_DATABASE=${CRDB_DATABASE} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_CONTEXT} \ + --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ + --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ + --from-literal=CRDB_SSLMODE=require +printf "\n" + +echo "Create secret with CockroachDB data for KPI Management" +CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') +CRDB_DATABASE_KPI_MGMT="tfs_kpi_mgmt" # TODO: change by specific configurable environment variable +kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ + --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ + --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_KPI_MGMT} \ --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ --from-literal=CRDB_SSLMODE=require diff --git a/install_requirements.sh b/install_requirements.sh index cbd378eca81af17386100fc0ceb3757912d0ebf5..54b660a521dadc08a344d2f79f2db15271131a21 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -22,6 +22,7 @@ ALL_COMPONENTS="context device service nbi monitoring webui interdomain slice" ALL_COMPONENTS="${ALL_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector" ALL_COMPONENTS="${ALL_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector" +ALL_COMPONENTS="${ALL_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api" TFS_COMPONENTS=${TFS_COMPONENTS:-$ALL_COMPONENTS} # Some components require libyang built from source code diff --git a/manifests/kafka/01-zookeeper.yaml b/manifests/kafka/01-zookeeper.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0e87ae0c6f12ed56702220f9e15fbe90b3b9c31 --- /dev/null +++ b/manifests/kafka/01-zookeeper.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + labels: + app: zookeeper-service + name: zookeeper-service + namespace: kafka +spec: + type: NodePort + ports: + - name: zookeeper-port + port: 2181 + nodePort: 30181 + targetPort: 2181 + selector: + app: zookeeper +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: zookeeper + name: zookeeper + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - image: wurstmeister/zookeeper + imagePullPolicy: IfNotPresent + name: zookeeper + ports: + - containerPort: 2181 \ No newline at end of file diff --git a/manifests/kafka/02-kafka.yaml b/manifests/kafka/02-kafka.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e4562e6eabec34bf3b87912310479bd98022aeb --- /dev/null +++ b/manifests/kafka/02-kafka.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + labels: + app: kafka-broker + name: kafka-service + namespace: kafka +spec: + ports: + - port: 9092 + selector: + app: kafka-broker +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: kafka-broker + name: kafka-broker + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-broker + template: + metadata: + labels: + app: kafka-broker + spec: + hostname: kafka-broker + containers: + - env: + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: :2181 + - name: KAFKA_LISTENERS + value: PLAINTEXT://:9092 + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://localhost:9092 + image: wurstmeister/kafka + imagePullPolicy: IfNotPresent + name: kafka-broker + ports: + - containerPort: 9092 \ No newline at end of file diff --git a/manifests/kpi_managerservice.yaml b/manifests/kpi_managerservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..984d783a9de7ed3c0c02e87d82ec673dc19c9508 --- /dev/null +++ b/manifests/kpi_managerservice.yaml @@ -0,0 +1,99 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kpi-managerservice +spec: + selector: + matchLabels: + app: kpi-managerservice + #replicas: 1 + template: + metadata: + annotations: + config.linkerd.io/skip-outbound-ports: "4222" + labels: + app: kpi-managerservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: labs.etsi.org:5050/tfs/controller/kpi_manager:latest + imagePullPolicy: Always + ports: + - containerPort: 30010 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + envFrom: + - secretRef: + name: crdb-kpi-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30010"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30010"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: kpi-managerservice + labels: + app: kpi-managerservice +spec: + type: ClusterIP + selector: + app: kpi-managerservice + ports: + - name: grpc + protocol: TCP + port: 30010 + targetPort: 30010 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: kpi-managerservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: kpi-managerservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/manifests/kpi_value_apiservice.yaml b/manifests/kpi_value_apiservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74eb90f675794f1b451b04af55e191edff58fae5 --- /dev/null +++ b/manifests/kpi_value_apiservice.yaml @@ -0,0 +1,96 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kpi-value-apiservice +spec: + selector: + matchLabels: + app: kpi-value-apiservice + #replicas: 1 + template: + metadata: + annotations: + config.linkerd.io/skip-outbound-ports: "4222" + labels: + app: kpi-value-apiservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: labs.etsi.org:5050/tfs/controller/kpi_value_api:latest + imagePullPolicy: Always + ports: + - containerPort: 30020 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30020"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30020"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: kpi-value-apiservice + labels: + app: kpi-value-apiservice +spec: + type: ClusterIP + selector: + app: kpi-value-apiservice + ports: + - name: grpc + protocol: TCP + port: 30020 + targetPort: 30020 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: kpi-value-apiservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: kpi-value-apiservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/manifests/kpi_value_writerservice.yaml b/manifests/kpi_value_writerservice.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a8e44ec2a571f1290e30a08d1c896a6339cbe46 --- /dev/null +++ b/manifests/kpi_value_writerservice.yaml @@ -0,0 +1,96 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kpi-value-writerservice +spec: + selector: + matchLabels: + app: kpi-value-writerservice + #replicas: 1 + template: + metadata: + annotations: + config.linkerd.io/skip-outbound-ports: "4222" + labels: + app: kpi-value-writerservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: server + image: labs.etsi.org:5050/tfs/controller/kpi_value_writer:latest + imagePullPolicy: Always + ports: + - containerPort: 30030 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30030"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30030"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: kpi-value-writerservice + labels: + app: kpi-value-writerservice +spec: + type: ClusterIP + selector: + app: kpi-value-writerservice + ports: + - name: grpc + protocol: TCP + port: 30030 + targetPort: 30030 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: kpi-value-writerservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: kpi-value-writerservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/my_deploy.sh b/my_deploy.sh index 6007a7ff971231c6c135dfad7b9385187f028421..b89df7481ebd17edf2b966eb818598d1a04a596f 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -22,9 +22,12 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" -# Uncomment to activate Monitoring +# Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" +# Uncomment to activate Monitoring Framework (new) +#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api" + # Uncomment to activate BGP-LS Speaker #export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" @@ -69,7 +72,7 @@ export TFS_K8S_NAMESPACE="tfs" export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" # Uncomment to monitor performance of components -export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" # Uncomment when deploying Optical CyberSecurity #export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" @@ -171,3 +174,10 @@ export PROM_EXT_PORT_HTTP="9090" # Set the external port Grafana HTTP Dashboards will be exposed to. export GRAF_EXT_PORT_HTTP="3000" + + +# ----- Apache Kafka ----------------------------------------------------------- + +# Set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE="kafka" + diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto new file mode 100644 index 0000000000000000000000000000000000000000..096c1ee035ae663359d9f4df1e071d3997a0d351 --- /dev/null +++ b/proto/analytics_frontend.proto @@ -0,0 +1,69 @@ +// Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package analytics_frontend; + +import "context.proto"; +import "kpi_manager.proto"; +//import "kpi_sample_types.proto"; + +service AnalyticsFrontendService { + rpc StartAnalyzer (Analyzer ) returns (AnalyzerId ) {} + rpc StopAnalyzer (AnalyzerId ) returns (context.Empty) {} + rpc SelectAnalyzers(AnalyzerFilter) returns (AnalyzerList ) {} +} + +message AnalyzerId { + context.Uuid analyzer_id = 1; +} + +enum AnalyzerOperationMode { + ANALYZEROPERATIONMODE_BATCH = 0; + ANALYZEROPERATIONMODE_STREAMING = 1; +} + +message Analyzer { + string algorithm_name = 1; // The algorithm to be executed + repeated kpi_manager.KpiId input_kpi_ids = 2; // The KPI Ids to be processed by the analyzer + repeated kpi_manager.KpiId output_kpi_ids = 3; // The KPI Ids produced by the analyzer + AnalyzerOperationMode operation_mode = 4; // Operation mode of the analyzer + + // In batch mode... + float batch_min_duration_s = 5; // ..., min duration to collect before executing batch + float batch_max_duration_s = 6; // ..., max duration collected to execute the batch + uint64 batch_min_size = 7; // ..., min number of samples to collect before executing batch + uint64 batch_max_size = 8; // ..., max number of samples collected to execute the batch +} + +message AnalyzerFilter { + // Analyzer that fulfill the filter are those that match ALL the following fields. + // An empty list means: any value is accepted. + // All fields empty means: list all Analyzers + repeated AnalyzerId analyzer_id = 1; + repeated string algorithm_names = 2; + repeated kpi_manager.KpiId input_kpi_ids = 3; + repeated kpi_manager.KpiId output_kpi_ids = 4; + //repeated kpi_sample_types.KpiSampleType kpi_sample_type = 5; // Not implemented + //repeated context.DeviceId device_id = 6; // Not implemented + //repeated context.EndPointId endpoint_id = 7; // Not implemented + //repeated context.ServiceId service_id = 8; // Not implemented + //repeated context.SliceId slice_id = 9; // Not implemented + //repeated context.ConnectionId connection_id = 10; // Not implemented + //repeated context.LinkId link_id = 11; // Not implemented +} + +message AnalyzerList { + repeated Analyzer analyzer_list = 1; +} diff --git a/proto/device.proto b/proto/device.proto index 3d7ba14bb75e226c51d8d2462fca76a1cab86554..a1882f33f8e177502c456672a0517928f0259ef5 100644 --- a/proto/device.proto +++ b/proto/device.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package device; import "context.proto"; -import "monitoring.proto"; +import "monitoring.proto"; // to be migrated to: "kpi_manager.proto" service DeviceService { rpc AddDevice (context.Device ) returns (context.DeviceId ) {} @@ -27,8 +27,8 @@ service DeviceService { } message MonitoringSettings { - monitoring.KpiId kpi_id = 1; - monitoring.KpiDescriptor kpi_descriptor = 2; - float sampling_duration_s = 3; - float sampling_interval_s = 4; + monitoring.KpiId kpi_id = 1; // to be migrated to: "kpi_manager.KpiId" + monitoring.KpiDescriptor kpi_descriptor = 2; // to be migrated to: "kpi_manager.KpiDescriptor" + float sampling_duration_s = 3; + float sampling_interval_s = 4; } diff --git a/proto/kpi_manager.proto b/proto/kpi_manager.proto new file mode 100644 index 0000000000000000000000000000000000000000..2640b58c60f004e51c8aeacc0ed76963f0436956 --- /dev/null +++ b/proto/kpi_manager.proto @@ -0,0 +1,60 @@ +// Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package kpi_manager; + +import "context.proto"; +import "kpi_sample_types.proto"; + +service KpiManagerService { + rpc SetKpiDescriptor (KpiDescriptor ) returns (KpiId ) {} + rpc DeleteKpiDescriptor (KpiId ) returns (context.Empty ) {} + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} + rpc SelectKpiDescriptor (KpiDescriptorFilter) returns (KpiDescriptorList ) {} +} + +message KpiId { + context.Uuid kpi_id = 1; +} + +message KpiDescriptor { + KpiId kpi_id = 1; + string kpi_description = 2; + kpi_sample_types.KpiSampleType kpi_sample_type = 3; + context.DeviceId device_id = 4; + context.EndPointId endpoint_id = 5; + context.ServiceId service_id = 6; + context.SliceId slice_id = 7; + context.ConnectionId connection_id = 8; + context.LinkId link_id = 9; +} + +message KpiDescriptorFilter { + // KPI Descriptors that fulfill the filter are those that match ALL the following fields. + // An empty list means: any value is accepted. + // All fields empty means: list all KPI Descriptors + repeated KpiId kpi_id = 1; + repeated kpi_sample_types.KpiSampleType kpi_sample_type = 2; + repeated context.DeviceId device_id = 3; + repeated context.EndPointId endpoint_id = 4; + repeated context.ServiceId service_id = 5; + repeated context.SliceId slice_id = 6; + repeated context.ConnectionId connection_id = 7; + repeated context.LinkId link_id = 8; +} + +message KpiDescriptorList { + repeated KpiDescriptor kpi_descriptor_list = 1; +} diff --git a/proto/kpi_value_api.proto b/proto/kpi_value_api.proto new file mode 100644 index 0000000000000000000000000000000000000000..dff96272e3d05756dd19a49ecaede7311b196540 --- /dev/null +++ b/proto/kpi_value_api.proto @@ -0,0 +1,52 @@ +// Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package kpi_value_api; + +import "context.proto"; +import "kpi_manager.proto"; + +service KpiValueAPIService { + rpc StoreKpiValues (KpiValueList) returns (context.Empty) {} + rpc SelectKpiValues (KpiValueFilter) returns (KpiValueList) {} +} + +message KpiValue { + kpi_manager.KpiId kpi_id = 1; + context.Timestamp timestamp = 2; + KpiValueType kpi_value_type = 3; +} + +message KpiValueList { + repeated KpiValue kpi_value_list = 1; +} + +message KpiValueType { + oneof value { + int32 int32Val = 1; + uint32 uint32Val = 2; + int64 int64Val = 3; + uint64 uint64Val = 4; + float floatVal = 5; + string stringVal = 6; + bool boolVal = 7; + } +} + +message KpiValueFilter { + repeated kpi_manager.KpiId kpi_id = 1; + repeated context.Timestamp start_timestamp = 2; + repeated context.Timestamp end_timestamp = 3; +} diff --git a/proto/monitoring.proto b/proto/monitoring.proto old mode 100644 new mode 100755 index 2c1c2f8ad58192586c17e310e33bccebbe775ee8..083bd82854547478d3a8f4a8935fdf75e9070d9d --- a/proto/monitoring.proto +++ b/proto/monitoring.proto @@ -145,12 +145,12 @@ message SubsList { } message AlarmDescriptor { - AlarmID alarm_id = 1; - string alarm_description = 2; - string name = 3; - KpiId kpi_id = 4; - KpiValueRange kpi_value_range = 5; - context.Timestamp timestamp = 6; + AlarmID alarm_id = 1; + string alarm_description = 2; + string name = 3; + KpiId kpi_id = 4; + KpiValueRange kpi_value_range = 5; + context.Timestamp timestamp = 6; } message AlarmID{ @@ -170,5 +170,5 @@ message AlarmResponse { } message AlarmList { - repeated AlarmDescriptor alarm_descriptor = 1; + repeated AlarmDescriptor alarm_descriptor = 1; } diff --git a/proto/optical_attack_detector.proto b/proto/optical_attack_detector.proto index 783e23b35d754db983c75c56dadc203996beadd4..f74eea68b8c5a588f5ecc06a59916058cb8d9695 100644 --- a/proto/optical_attack_detector.proto +++ b/proto/optical_attack_detector.proto @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// protocol buffers documentation: https://developers.google.com/protocol-buffers/docs/proto3 syntax = "proto3"; package optical_attack_detector; import "context.proto"; -import "monitoring.proto"; +import "monitoring.proto"; // to be migrated to: "kpi_manager.proto" service OpticalAttackDetectorService { @@ -28,5 +27,5 @@ service OpticalAttackDetectorService { message DetectionRequest { context.ServiceId service_id = 1; - monitoring.KpiId kpi_id = 2; + monitoring.KpiId kpi_id = 2; // to be migrated to: "kpi_manager.KpiId" } diff --git a/proto/policy_condition.proto b/proto/policy_condition.proto index add3ec1ab127674e171c366ffa49346892b3ff0d..612dcb1af8eb8adb0db65b8ae47301c87ad6b9ef 100644 --- a/proto/policy_condition.proto +++ b/proto/policy_condition.proto @@ -15,13 +15,13 @@ syntax = "proto3"; package policy; -import "monitoring.proto"; +import "monitoring.proto"; // to be migrated to: "kpi_manager.proto" // Condition message PolicyRuleCondition { - monitoring.KpiId kpiId = 1; - NumericalOperator numericalOperator = 2; - monitoring.KpiValue kpiValue = 3; + monitoring.KpiId kpiId = 1; // to be migrated to: "kpi_manager.KpiId" + NumericalOperator numericalOperator = 2; + monitoring.KpiValue kpiValue = 3; } // Operator to be used when comparing Kpis with condition values diff --git a/proto/telemetry_frontend.proto b/proto/telemetry_frontend.proto new file mode 100644 index 0000000000000000000000000000000000000000..dbc1e8bf688f9f2df341484c1929e2338c458bbf --- /dev/null +++ b/proto/telemetry_frontend.proto @@ -0,0 +1,48 @@ +// Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package telemetry_frontend; + +import "context.proto"; +import "kpi_manager.proto"; + +service TelemetryFrontendService { + rpc StartCollector (Collector ) returns (CollectorId ) {} + rpc StopCollector (CollectorId ) returns (context.Empty) {} + rpc SelectCollectors(CollectorFilter) returns (CollectorList) {} +} + +message CollectorId { + context.Uuid collector_id = 1; +} + +message Collector { + CollectorId collector_id = 1; // The Collector ID + kpi_manager.KpiId kpi_id = 2; // The KPI Id to be associated to the collected samples + float duration_s = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely + float interval_s = 4; // Interval between collected samples +} + +message CollectorFilter { + // Collector that fulfill the filter are those that match ALL the following fields. + // An empty list means: any value is accepted. + // All fields empty means: list all Collectors + repeated CollectorId collector_id = 1; + repeated kpi_manager.KpiId kpi_id = 2; +} + +message CollectorList { + repeated Collector collector_list = 1; +} diff --git a/scripts/run_tests_locally-kpi-DB.sh b/scripts/run_tests_locally-kpi-DB.sh new file mode 100755 index 0000000000000000000000000000000000000000..d43be66e1e9843f85e34442d00b10a11d2903c43 --- /dev/null +++ b/scripts/run_tests_locally-kpi-DB.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + kpi_manager/tests/test_kpi_db.py diff --git a/scripts/run_tests_locally-kpi-manager.sh b/scripts/run_tests_locally-kpi-manager.sh new file mode 100755 index 0000000000000000000000000000000000000000..db6e786835bcd4550b53394aef23aaa670d43b08 --- /dev/null +++ b/scripts/run_tests_locally-kpi-manager.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + kpi_manager/tests/test_kpi_manager.py diff --git a/scripts/run_tests_locally-kpi-prom-writer.sh b/scripts/run_tests_locally-kpi-prom-writer.sh new file mode 100755 index 0000000000000000000000000000000000000000..1179cbf866969607b880eb41be5c25ab95007bfc --- /dev/null +++ b/scripts/run_tests_locally-kpi-prom-writer.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + kpi_value_writer/tests/test_metric_writer_to_prom.py diff --git a/scripts/run_tests_locally-kpi-value-API.sh b/scripts/run_tests_locally-kpi-value-API.sh new file mode 100755 index 0000000000000000000000000000000000000000..8dfbfb16237634519dcae2fcc34f850a5188c1e7 --- /dev/null +++ b/scripts/run_tests_locally-kpi-value-API.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src + +RCFILE=$PROJECTDIR/coverage/.coveragerc + +# helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG -o log_cli=true --verbose \ + kpi_value_api/tests/test_kpi_value_api.py diff --git a/scripts/run_tests_locally-kpi-value-writer.sh b/scripts/run_tests_locally-kpi-value-writer.sh new file mode 100755 index 0000000000000000000000000000000000000000..8faaeb6d895a240278d7ceb0c5c0b2855fa25910 --- /dev/null +++ b/scripts/run_tests_locally-kpi-value-writer.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + kpi_value_writer/tests/test_kpi_value_writer.py diff --git a/scripts/run_tests_locally-telemetry-DB.sh b/scripts/run_tests_locally-telemetry-DB.sh new file mode 100755 index 0000000000000000000000000000000000000000..bb1c48b76440c00b398875a8f704c2a82ba4ab50 --- /dev/null +++ b/scripts/run_tests_locally-telemetry-DB.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-cli-level=INFO --verbose \ + telemetry/database/tests/telemetryDBtests.py diff --git a/scripts/run_tests_locally-telemetry-backend.sh b/scripts/run_tests_locally-telemetry-backend.sh new file mode 100755 index 0000000000000000000000000000000000000000..9cf404ffcef6c99b261f81eb0c6b910dd60845e5 --- /dev/null +++ b/scripts/run_tests_locally-telemetry-backend.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \ + telemetry/backend/tests/testTelemetryBackend.py diff --git a/scripts/run_tests_locally-telemetry-frontend.sh b/scripts/run_tests_locally-telemetry-frontend.sh new file mode 100755 index 0000000000000000000000000000000000000000..7652ccb583268285dcd2fcf3090b717dc18e4fc3 --- /dev/null +++ b/scripts/run_tests_locally-telemetry-frontend.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +# python3 kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \ + telemetry/frontend/tests/test_frontend.py diff --git a/scripts/run_tests_locally-telemetry-mgtDB.sh b/scripts/run_tests_locally-telemetry-mgtDB.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b68104eaf343b57ec4953334cda37167cca3529 --- /dev/null +++ b/scripts/run_tests_locally-telemetry-mgtDB.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-cli-level=INFO --verbose \ + telemetry/database/tests/managementDBtests.py diff --git a/scripts/show_logs_kpi_manager.sh b/scripts/show_logs_kpi_manager.sh new file mode 100755 index 0000000000000000000000000000000000000000..86f084f69f6babf5a90957f432b214e35a08c461 --- /dev/null +++ b/scripts/show_logs_kpi_manager.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/kpi-managerservice -c server diff --git a/scripts/show_logs_kpi_value_api.sh b/scripts/show_logs_kpi_value_api.sh new file mode 100755 index 0000000000000000000000000000000000000000..041ad7f1ffb1a218af00d5d142024a5063d109c3 --- /dev/null +++ b/scripts/show_logs_kpi_value_api.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/kpi-value-apiservice -c server diff --git a/scripts/show_logs_kpi_value_writer.sh b/scripts/show_logs_kpi_value_writer.sh new file mode 100755 index 0000000000000000000000000000000000000000..d62f3ea0a1a6961be4a5b6f4841c9ba4e1a89316 --- /dev/null +++ b/scripts/show_logs_kpi_value_writer.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/kpi-value-writerservice -c server diff --git a/scripts/show_logs_telemetry-DB.sh b/scripts/show_logs_telemetry-DB.sh new file mode 100755 index 0000000000000000000000000000000000000000..84fc875d01e18eae9b144edaf220d5cb74017ea4 --- /dev/null +++ b/scripts/show_logs_telemetry-DB.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"crdb"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs cockroachdb-0 diff --git a/src/common/Constants.py b/src/common/Constants.py index a6a6899b0342c8e3efcde8e2e26897c1e4734cd3..767b21343f89e35c2338b522bcdc71c56aca1815 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -61,6 +61,10 @@ class ServiceNameEnum(Enum): E2EORCHESTRATOR = 'e2e-orchestrator' OPTICALCONTROLLER = 'opticalcontroller' BGPLS = 'bgpls-speaker' + KPIMANAGER = 'kpi-manager' + KPIVALUEAPI = 'kpi-value-api' + KPIVALUEWRITER = 'kpi-value-writer' + TELEMETRYFRONTEND = 'telemetry-frontend' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -90,6 +94,10 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.E2EORCHESTRATOR .value : 10050, ServiceNameEnum.OPTICALCONTROLLER .value : 10060, ServiceNameEnum.BGPLS .value : 20030, + ServiceNameEnum.KPIMANAGER .value : 30010, + ServiceNameEnum.KPIVALUEAPI .value : 30020, + ServiceNameEnum.KPIVALUEWRITER .value : 30030, + ServiceNameEnum.TELEMETRYFRONTEND .value : 30050, # Used for test and debugging only ServiceNameEnum.DLT_GATEWAY .value : 50051, diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py new file mode 100644 index 0000000000000000000000000000000000000000..24ae2cff7b5e710e18999eb09029216a4a5d6c8a --- /dev/null +++ b/src/common/tools/kafka/Variables.py @@ -0,0 +1,74 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from enum import Enum +from confluent_kafka import KafkaException +from confluent_kafka.admin import AdminClient, NewTopic + + +LOGGER = logging.getLogger(__name__) + +class KafkaConfig(Enum): + # SERVER_IP = "127.0.0.1:9092" + SERVER_IP = "kafka-service.kafka.svc.cluster.local:9092" + ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_IP}) + +class KafkaTopic(Enum): + REQUEST = 'topic_request' + RESPONSE = 'topic_response' + RAW = 'topic_raw' + LABELED = 'topic_labeled' + VALUE = 'topic_value' + + @staticmethod + def create_all_topics() -> bool: + """ + Method to create Kafka topics defined as class members + """ + all_topics = [member.value for member in KafkaTopic] + if( KafkaTopic.create_new_topic_if_not_exists( all_topics )): + LOGGER.debug("All topics are created sucsessfully") + return True + else: + LOGGER.debug("Error creating all topics") + return False + + @staticmethod + def create_new_topic_if_not_exists(new_topics: list) -> bool: + """ + Method to create Kafka topic if it does not exist. + Args: + list of topic: containing the topic name(s) to be created on Kafka + """ + LOGGER.debug("Topics names to be verified and created: {:}".format(new_topics)) + for topic in new_topics: + try: + topic_metadata = KafkaConfig.ADMIN_CLIENT.value.list_topics(timeout=5) + # LOGGER.debug("Existing topic list: {:}".format(topic_metadata.topics)) + if topic not in topic_metadata.topics: + # If the topic does not exist, create a new topic + print("Topic {:} does not exist. Creating...".format(topic)) + LOGGER.debug("Topic {:} does not exist. Creating...".format(topic)) + new_topic = NewTopic(topic, num_partitions=1, replication_factor=1) + KafkaConfig.ADMIN_CLIENT.value.create_topics([new_topic]) + else: + print("Topic name already exists: {:}".format(topic)) + LOGGER.debug("Topic name already exists: {:}".format(topic)) + except Exception as e: + LOGGER.debug("Failed to create topic: {:}".format(e)) + return False + return True + +# create all topics after the deployments (Telemetry and Analytics) diff --git a/src/kpi_manager/.gitlab-ci.yml b/src/kpi_manager/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..6aef328ea51e835eb06b286478f26d83f9a80f13 --- /dev/null +++ b/src/kpi_manager/.gitlab-ci.yml @@ -0,0 +1,133 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build kpi-manager: + variables: + IMAGE_NAME: 'kpi-manager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test kpi-manager: + variables: + IMAGE_NAME: 'kpi-manager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build kpi-manager + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - docker logs crdb + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $NATS_ADDRESS + - > + docker run --name $IMAGE_NAME -d -p 30010:30010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs $IMAGE_NAME + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + +## Deployment of the service in Kubernetes Cluster +#deploy context: +# variables: +# IMAGE_NAME: 'context' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - unit test context +# # - integ_test execute +# script: +# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# when: manual +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual diff --git a/src/kpi_manager/Dockerfile b/src/kpi_manager/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a57957759a32b45b715e327b54ebe004a6edf265 --- /dev/null +++ b/src/kpi_manager/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/kpi_manager +WORKDIR /var/teraflow/kpi_manager +COPY src/kpi_manager/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/kpi_manager/. kpi_manager/ + +# Start the service +ENTRYPOINT ["python", "-m", "kpi_manager.service"] diff --git a/src/kpi_manager/README.md b/src/kpi_manager/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c1feadcc4843db26a219d1e3b37833ddd80b18dc --- /dev/null +++ b/src/kpi_manager/README.md @@ -0,0 +1,29 @@ +# How to locally run and test KPI manager micro-service + +## --- File links need to be updated. --- +### Pre-requisets +The following requirements should be fulfilled before the execuation of KPI management service. + +1. Verify that [kpi_management.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/kpi_management.proto) file exists and grpcs file are generated sucessfully. +2. Virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/requirements.in) are installed sucessfully. +3. Verify the creation of required database and table. +[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/database/tests/KpiDBtests.py) python file enlist the functions to create tables and database and +[KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/service/database/KpiEngine.py) contains the DB string, update the string as per your deployment. + +### Messages format templates +["Messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_messages.py) python file enlist the basic gRPC messages format used during the testing. + +### Test file +["KPI management test"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_kpi_manager.py) python file enlist different tests conducted during the experiment. + +### Flow of execution (Kpi Maanager Service functions) +1. Call the `create_database()` and `create_tables()` functions from `Kpi_DB` class to create the required database and table if they don't exist. Call `verify_tables` to verify the existence of KPI table. + +2. Call the gRPC method `SetKpiDescriptor(KpiDescriptor)->KpiId` to add the KpiDescriptor in `Kpi` DB. `KpiDescriptor` and `KpiId` are both pre-defined gRPC message types. + +3. Call `GetKpiDescriptor(KpiId)->KpiDescriptor` to read the `KpiDescriptor` from DB and `DeleteKpiDescriptor(KpiId)` to delete the `KpiDescriptor` from DB. + +4. Call `SelectKpiDescriptor(KpiDescriptorFilter)->KpiDescriptorList` to get all `KpiDescriptor` objects that matches the filter criteria. `KpiDescriptorFilter` and `KpiDescriptorList` are pre-defined gRPC message types. + +## For KPI composer and KPI writer +The functionalities of KPI composer and writer is heavily dependent upon Telemetery service. Therfore, these services has other pre-requsites that are mention [here](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/requirements.in). diff --git a/src/kpi_manager/__init__.py b/src/kpi_manager/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_manager/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_manager/client/KpiManagerClient.py b/src/kpi_manager/client/KpiManagerClient.py new file mode 100755 index 0000000000000000000000000000000000000000..672d82f2d78ea8b477429c5ba03fbb4331bae7c7 --- /dev/null +++ b/src/kpi_manager/client/KpiManagerClient.py @@ -0,0 +1,77 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc + +from common.proto.context_pb2 import Empty +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceStub +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 10 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class KpiManagerClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.KPIMANAGER) + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = KpiManagerServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def SetKpiDescriptor(self, request : KpiDescriptor) -> KpiId: + LOGGER.debug('SetKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SetKpiDescriptor(request) + LOGGER.debug('SetKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeleteKpiDescriptor(self,request : KpiId) -> Empty: + LOGGER.debug('DeleteKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeleteKpiDescriptor(request) + LOGGER.debug('DeleteKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetKpiDescriptor(self, request : KpiId) -> KpiDescriptor: + LOGGER.debug('GetKpiDescriptor: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetKpiDescriptor(request) + LOGGER.debug('GetKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def SelectKpiDescriptor(self, filter : KpiDescriptorFilter) -> KpiDescriptorList: + LOGGER.debug('SelectKpiDescriptor: {:s}'.format(grpc_message_to_json_string(filter))) + response = self.stub.SelectKpiDescriptor(filter) + LOGGER.debug('SelectKpiDescriptor result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/kpi_manager/client/__init__.py b/src/kpi_manager/client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..48f7d354a2f3fe6e91bb79b3ca956f68c36ed9e3 --- /dev/null +++ b/src/kpi_manager/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_manager/database/KpiEngine.py b/src/kpi_manager/database/KpiEngine.py new file mode 100644 index 0000000000000000000000000000000000000000..42bda9527dfcb1e5cef0e229d260fd67f4b4a6d5 --- /dev/null +++ b/src/kpi_manager/database/KpiEngine.py @@ -0,0 +1,44 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, sqlalchemy +from common.Settings import get_setting + +LOGGER = logging.getLogger(__name__) + +# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' +CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' + +class KpiEngine: + @staticmethod + def get_engine() -> sqlalchemy.engine.Engine: + crdb_uri = get_setting('CRDB_URI', default=None) + if crdb_uri is None: + CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') + CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') + CRDB_DATABASE = get_setting('CRDB_DATABASE') + CRDB_USERNAME = get_setting('CRDB_USERNAME') + CRDB_PASSWORD = get_setting('CRDB_PASSWORD') + CRDB_SSLMODE = get_setting('CRDB_SSLMODE') + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + # crdb_uri = CRDB_URI_TEMPLATE.format( + # CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + try: + engine = sqlalchemy.create_engine(crdb_uri, echo=False) + LOGGER.info(' KpiDBmanager initalized with DB URL: {:}'.format(crdb_uri)) + except: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) + return None # type: ignore + return engine diff --git a/src/kpi_manager/database/KpiModel.py b/src/kpi_manager/database/KpiModel.py new file mode 100644 index 0000000000000000000000000000000000000000..5c2fdff0664883bcc727096ddeda562fdbe3085d --- /dev/null +++ b/src/kpi_manager/database/KpiModel.py @@ -0,0 +1,84 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import Column, Integer, String, Text +from sqlalchemy.orm import registry +from common.proto.kpi_manager_pb2 import KpiDescriptor + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + +# Create a base class for declarative models +Base = registry().generate_base() + +class Kpi(Base): + __tablename__ = 'kpi' + + kpi_id = Column(UUID(as_uuid=False), primary_key=True) + kpi_description = Column(Text , nullable=False) + kpi_sample_type = Column(Integer , nullable=False) + device_id = Column(String , nullable=False) + endpoint_id = Column(String , nullable=False) + service_id = Column(String , nullable=False) + slice_id = Column(String , nullable=False) + connection_id = Column(String , nullable=False) + link_id = Column(String , nullable=False) + + # helps in logging the information + def __repr__(self): + return (f"") + + @classmethod + def convert_KpiDescriptor_to_row(cls, request): + """ + Create an instance of Kpi from a request object. + Args: request: The request object containing the data. + Returns: An instance of Kpi initialized with data from the request. + """ + return cls( + kpi_id = request.kpi_id.kpi_id.uuid, + kpi_description = request.kpi_description, + kpi_sample_type = request.kpi_sample_type, + device_id = request.device_id.device_uuid.uuid, + endpoint_id = request.endpoint_id.endpoint_uuid.uuid, + service_id = request.service_id.service_uuid.uuid, + slice_id = request.slice_id.slice_uuid.uuid, + connection_id = request.connection_id.connection_uuid.uuid, + link_id = request.link_id.link_uuid.uuid + ) + + @classmethod + def convert_row_to_KpiDescriptor(cls, row): + """ + Create and return a dictionary representation of a Kpi instance. + Args: row: The Kpi instance (row) containing the data. + Returns: KpiDescriptor object + """ + response = KpiDescriptor() + response.kpi_id.kpi_id.uuid = row.kpi_id + response.kpi_description = row.kpi_description + response.kpi_sample_type = row.kpi_sample_type + response.service_id.service_uuid.uuid = row.service_id + response.device_id.device_uuid.uuid = row.device_id + response.slice_id.slice_uuid.uuid = row.slice_id + response.endpoint_id.endpoint_uuid.uuid = row.endpoint_id + response.connection_id.connection_uuid.uuid = row.connection_id + response.link_id.link_uuid.uuid = row.link_id + return response diff --git a/src/kpi_manager/database/Kpi_DB.py b/src/kpi_manager/database/Kpi_DB.py new file mode 100644 index 0000000000000000000000000000000000000000..5b2b586b6d72cf73ff3362f9240d1171e8a9974d --- /dev/null +++ b/src/kpi_manager/database/Kpi_DB.py @@ -0,0 +1,154 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sqlalchemy_utils +from sqlalchemy.orm import sessionmaker +from kpi_manager.database.KpiEngine import KpiEngine +from kpi_manager.database.KpiModel import Kpi as KpiModel +from common.method_wrappers.ServiceExceptions import ( + AlreadyExistsException, OperationFailedException) + +LOGGER = logging.getLogger(__name__) +DB_NAME = "kpi" + +class KpiDB: + def __init__(self): + self.db_engine = KpiEngine.get_engine() + if self.db_engine is None: + LOGGER.error('Unable to get SQLAlchemy DB Engine...') + return False + self.db_name = DB_NAME + self.Session = sessionmaker(bind=self.db_engine) + + def create_database(self) -> None: + if not sqlalchemy_utils.database_exists(self.db_engine.url): + LOGGER.debug("Database created. {:}".format(self.db_engine.url)) + sqlalchemy_utils.create_database(self.db_engine.url) + + def drop_database(self) -> None: + if sqlalchemy_utils.database_exists(self.db_engine.url): + sqlalchemy_utils.drop_database(self.db_engine.url) + + def create_tables(self): + try: + KpiModel.metadata.create_all(self.db_engine) # type: ignore + LOGGER.debug("Tables created in the DB Name: {:}".format(self.db_name)) + except Exception as e: + LOGGER.debug("Tables cannot be created in the kpi database. {:s}".format(str(e))) + raise OperationFailedException ("Tables can't be created", extra_details=["unable to create table {:}".format(e)]) + + def verify_tables(self): + try: + with self.db_engine.connect() as connection: + result = connection.execute("SHOW TABLES;") + tables = result.fetchall() # type: ignore + LOGGER.debug("Tables verified: {:}".format(tables)) + except Exception as e: + LOGGER.debug("Unable to fetch Table names. {:s}".format(str(e))) + + def add_row_to_db(self, row): + session = self.Session() + try: + session.add(row) + session.commit() + LOGGER.debug(f"Row inserted into {row.__class__.__name__} table.") + return True + except Exception as e: + session.rollback() + if "psycopg2.errors.UniqueViolation" in str(e): + LOGGER.error(f"Unique key voilation: {row.__class__.__name__} table. {str(e)}") + raise AlreadyExistsException(row.__class__.__name__, row, + extra_details=["Unique key voilation: {:}".format(e)] ) + else: + LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}") + raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) + finally: + session.close() + + def search_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + entity = session.query(model).filter_by(**{col_name: id_to_search}).first() + if entity: + # LOGGER.debug(f"{model.__name__} ID found: {str(entity)}") + return entity + else: + LOGGER.debug(f"{model.__name__} ID not found: {str(id_to_search)}") + return None + except Exception as e: + session.rollback() + LOGGER.debug(f"Failed to retrieve {model.__name__} ID. {str(e)}") + raise OperationFailedException ("search by column id", extra_details=["unable to search row {:}".format(e)]) + finally: + session.close() + + def delete_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + record = session.query(model).filter_by(**{col_name: id_to_search}).first() + if record: + session.delete(record) + session.commit() + LOGGER.debug("Deleted %s with %s: %s", model.__name__, col_name, id_to_search) + else: + LOGGER.debug("%s with %s %s not found", model.__name__, col_name, id_to_search) + return None + except Exception as e: + session.rollback() + LOGGER.error("Error deleting %s with %s %s: %s", model.__name__, col_name, id_to_search, e) + raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) + finally: + session.close() + + def select_with_filter(self, model, filter_object): + session = self.Session() + try: + query = session.query(KpiModel) + # Apply filters based on the filter_object + if filter_object.kpi_id: + query = query.filter(KpiModel.kpi_id.in_([k.kpi_id.uuid for k in filter_object.kpi_id])) + + if filter_object.kpi_sample_type: + query = query.filter(KpiModel.kpi_sample_type.in_(filter_object.kpi_sample_type)) + + if filter_object.device_id: + query = query.filter(KpiModel.device_id.in_([d.device_uuid.uuid for d in filter_object.device_id])) + + if filter_object.endpoint_id: + query = query.filter(KpiModel.endpoint_id.in_([e.endpoint_uuid.uuid for e in filter_object.endpoint_id])) + + if filter_object.service_id: + query = query.filter(KpiModel.service_id.in_([s.service_uuid.uuid for s in filter_object.service_id])) + + if filter_object.slice_id: + query = query.filter(KpiModel.slice_id.in_([s.slice_uuid.uuid for s in filter_object.slice_id])) + + if filter_object.connection_id: + query = query.filter(KpiModel.connection_id.in_([c.connection_uuid.uuid for c in filter_object.connection_id])) + + if filter_object.link_id: + query = query.filter(KpiModel.link_id.in_([l.link_uuid.uuid for l in filter_object.link_id])) + result = query.all() + + if result: + LOGGER.debug(f"Fetched filtered rows from {model.__name__} table with filters: {filter_object}") # - Results: {result} + else: + LOGGER.debug(f"No matching row found in {model.__name__} table with filters: {filter_object}") + return result + except Exception as e: + LOGGER.error(f"Error fetching filtered rows from {model.__name__} table with filters {filter_object} ::: {e}") + raise OperationFailedException ("Select by filter", extra_details=["unable to apply the filter {:}".format(e)]) + finally: + session.close() diff --git a/src/kpi_manager/database/__init__.py b/src/kpi_manager/database/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_manager/database/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_manager/requirements.in b/src/kpi_manager/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..3e98fef362277dbf60019902e115d1c733bea9e7 --- /dev/null +++ b/src/kpi_manager/requirements.in @@ -0,0 +1,18 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* diff --git a/src/kpi_manager/service/KpiManagerService.py b/src/kpi_manager/service/KpiManagerService.py new file mode 100755 index 0000000000000000000000000000000000000000..b69a926a94c6cf10a680fe1b15d065f6bc073c97 --- /dev/null +++ b/src/kpi_manager/service/KpiManagerService.py @@ -0,0 +1,29 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.proto.kpi_manager_pb2_grpc import add_KpiManagerServiceServicer_to_server +from kpi_manager.service.KpiManagerServiceServicerImpl import KpiManagerServiceServicerImpl + + +class KpiManagerService(GenericGrpcService): + def __init__(self, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) + super().__init__(port, cls_name=cls_name) + self.kpiManagerService_servicer = KpiManagerServiceServicerImpl() + + def install_servicers(self): + add_KpiManagerServiceServicer_to_server(self.kpiManagerService_servicer, self.server) diff --git a/src/kpi_manager/service/KpiManagerServiceServicerImpl.py b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py new file mode 100644 index 0000000000000000000000000000000000000000..05292fc5b14feaf079cc7691c650775965cc9148 --- /dev/null +++ b/src/kpi_manager/service/KpiManagerServiceServicerImpl.py @@ -0,0 +1,94 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty +from common.proto.kpi_manager_pb2_grpc import KpiManagerServiceServicer +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList +from kpi_manager.database.Kpi_DB import KpiDB +from kpi_manager.database.KpiModel import Kpi as KpiModel + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('KpiManager', 'NBIgRPC') + +class KpiManagerServiceServicerImpl(KpiManagerServiceServicer): + def __init__(self): + LOGGER.info('Init KpiManagerService') + self.kpi_db_obj = KpiDB() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SetKpiDescriptor(self, request: KpiDescriptor, grpc_context: grpc.ServicerContext # type: ignore + ) -> KpiId: # type: ignore + response = KpiId() + LOGGER.info("Received gRPC message object: {:}".format(request)) + try: + kpi_to_insert = KpiModel.convert_KpiDescriptor_to_row(request) + if(self.kpi_db_obj.add_row_to_db(kpi_to_insert)): + response.kpi_id.uuid = request.kpi_id.kpi_id.uuid + # LOGGER.info("Added Row: {:}".format(response)) + return response + except Exception as e: + LOGGER.info("Unable to create KpiModel class object. {:}".format(e)) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext # type: ignore + ) -> KpiDescriptor: # type: ignore + response = KpiDescriptor() + print("--> Received gRPC message object: {:}".format(request)) + LOGGER.info("Received gRPC message object: {:}".format(request)) + try: + kpi_id_to_search = request.kpi_id.uuid + row = self.kpi_db_obj.search_db_row_by_id(KpiModel, 'kpi_id', kpi_id_to_search) + if row is not None: + response = KpiModel.convert_row_to_KpiDescriptor(row) + return response + if row is None: + print ('No matching row found for kpi id: {:}'.format(kpi_id_to_search)) + LOGGER.info('No matching row found kpi id: {:}'.format(kpi_id_to_search)) + return Empty() + except Exception as e: + print ('Unable to search kpi id. {:}'.format(e)) + LOGGER.info('Unable to search kpi id. {:}'.format(e)) + raise e + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeleteKpiDescriptor(self, request: KpiId, grpc_context: grpc.ServicerContext # type: ignore + ) -> Empty: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(request)) + try: + kpi_id_to_search = request.kpi_id.uuid + self.kpi_db_obj.delete_db_row_by_id(KpiModel, 'kpi_id', kpi_id_to_search) + except Exception as e: + LOGGER.info('Unable to search kpi id. {:}'.format(e)) + finally: + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SelectKpiDescriptor(self, filter: KpiDescriptorFilter, grpc_context: grpc.ServicerContext # type: ignore + ) -> KpiDescriptorList: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(filter)) + response = KpiDescriptorList() + try: + rows = self.kpi_db_obj.select_with_filter(KpiModel, filter) + except Exception as e: + LOGGER.info('Unable to apply filter on kpi descriptor. {:}'.format(e)) + try: + for row in rows: + kpiDescriptor_obj = KpiModel.convert_row_to_KpiDescriptor(row) + response.kpi_descriptor_list.append(kpiDescriptor_obj) + return response + except Exception as e: + LOGGER.info('Unable to process filter response {:}'.format(e)) diff --git a/src/kpi_manager/service/__init__.py b/src/kpi_manager/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_manager/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_manager/service/__main__.py b/src/kpi_manager/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..244d5afa373a6462a0382a0ed26a588088a689a1 --- /dev/null +++ b/src/kpi_manager/service/__main__.py @@ -0,0 +1,51 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Settings import get_log_level +from .KpiManagerService import KpiManagerService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = KpiManagerService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/kpi_manager/tests/test_kpi_db.py b/src/kpi_manager/tests/test_kpi_db.py new file mode 100644 index 0000000000000000000000000000000000000000..e961c12bacdbac07f111b229435ed3d89d62581f --- /dev/null +++ b/src/kpi_manager/tests/test_kpi_db.py @@ -0,0 +1,28 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from kpi_manager.database.Kpi_DB import KpiDB + +LOGGER = logging.getLogger(__name__) + +def test_verify_databases_and_Tables(): + LOGGER.info('>>> test_verify_Tables : START <<< ') + kpiDBobj = KpiDB() + kpiDBobj.drop_database() + kpiDBobj.verify_tables() + kpiDBobj.create_database() + kpiDBobj.create_tables() + kpiDBobj.verify_tables() diff --git a/src/kpi_manager/tests/test_kpi_manager.py b/src/kpi_manager/tests/test_kpi_manager.py new file mode 100755 index 0000000000000000000000000000000000000000..b41e5139db85cfc462cff1e9545fbc8476c67939 --- /dev/null +++ b/src/kpi_manager/tests/test_kpi_manager.py @@ -0,0 +1,301 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os, pytest +import logging +from typing import Union + +#from common.proto.context_pb2 import Empty +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server + +from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList +from common.tools.service.GenericGrpcService import GenericGrpcService +#from context.client.ContextClient import ContextClient + +# from device.service.driver_api.DriverFactory import DriverFactory +# from device.service.driver_api.DriverInstanceCache import DriverInstanceCache +# from device.service.DeviceService import DeviceService +# from device.client.DeviceClient import DeviceClient + +from kpi_manager.tests.test_messages import create_kpi_descriptor_request, create_kpi_filter_request, create_kpi_descriptor_request_a +from kpi_manager.service.KpiManagerService import KpiManagerService +from kpi_manager.client.KpiManagerClient import KpiManagerClient +from kpi_manager.tests.test_messages import create_kpi_descriptor_request +from kpi_value_writer.tests.test_messages import create_kpi_id_request + + +#from monitoring.service.NameMapping import NameMapping + +#os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' +#from device.service.drivers import DRIVERS + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' + +KPIMANAGER_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # type: ignore +os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(KPIMANAGER_SERVICE_PORT) + +# METRICSDB_HOSTNAME = os.environ.get('METRICSDB_HOSTNAME'){} + +LOGGER = logging.getLogger(__name__) + +class MockContextService(GenericGrpcService): + # Mock Service implementing Context to simplify unitary tests of Monitoring + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +# @pytest.fixture(scope='session') +# def context_service(): +# LOGGER.info('Initializing MockContextService...') +# _service = MockContextService(MOCKSERVICE_PORT) +# _service.start() + +# LOGGER.info('Yielding MockContextService...') +# yield _service + +# LOGGER.info('Terminating MockContextService...') +# _service.context_servicer.msg_broker.terminate() +# _service.stop() + +# LOGGER.info('Terminated MockContextService...') + +# @pytest.fixture(scope='session') +# def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument +# LOGGER.info('Initializing ContextClient...') +# _client = ContextClient() + +# LOGGER.info('Yielding ContextClient...') +# yield _client + +# LOGGER.info('Closing ContextClient...') +# _client.close() + +# LOGGER.info('Closed ContextClient...') + +# @pytest.fixture(scope='session') +# def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument +# LOGGER.info('Initializing DeviceService...') +# driver_factory = DriverFactory(DRIVERS) +# driver_instance_cache = DriverInstanceCache(driver_factory) +# _service = DeviceService(driver_instance_cache) +# _service.start() + +# # yield the server, when test finishes, execution will resume to stop it +# LOGGER.info('Yielding DeviceService...') +# yield _service + +# LOGGER.info('Terminating DeviceService...') +# _service.stop() + +# LOGGER.info('Terminated DeviceService...') + +# @pytest.fixture(scope='session') +# def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument +# LOGGER.info('Initializing DeviceClient...') +# _client = DeviceClient() + +# LOGGER.info('Yielding DeviceClient...') +# yield _client + +# LOGGER.info('Closing DeviceClient...') +# _client.close() + +# LOGGER.info('Closed DeviceClient...') + +# @pytest.fixture(scope='session') +# def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument +# LOGGER.info('Initializing DeviceClient...') +# _client = DeviceClient() + +# LOGGER.info('Yielding DeviceClient...') +# yield _client + +# LOGGER.info('Closing DeviceClient...') +# _client.close() + +# LOGGER.info('Closed DeviceClient...') + +# This fixture will be requested by test cases and last during testing session +@pytest.fixture(scope='session') +def kpi_manager_service(): + LOGGER.info('Initializing KpiManagerService...') + #name_mapping = NameMapping() + # _service = MonitoringService(name_mapping) + # _service = KpiManagerService(name_mapping) + _service = KpiManagerService() + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiManagerService...') + yield _service + + LOGGER.info('Terminating KpiManagerService...') + _service.stop() + + LOGGER.info('Terminated KpiManagerService...') + +# This fixture will be requested by test cases and last during testing session. +# The client requires the server, so client fixture has the server as dependency. +# def monitoring_client(monitoring_service : MonitoringService): (Add for better understanding) +@pytest.fixture(scope='session') +def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing KpiManagerClient...') + _client = KpiManagerClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiManagerClient...') + yield _client + + LOGGER.info('Closing KpiManagerClient...') + _client.close() + + LOGGER.info('Closed KpiManagerClient...') + +################################################## +# Prepare Environment, should be the first test +################################################## + +# # ERROR on this test --- +# def test_prepare_environment( +# context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument +# ): +# context_id = json_context_id(DEFAULT_CONTEXT_NAME) +# context_client.SetContext(Context(**json_context(DEFAULT_CONTEXT_NAME))) +# context_client.SetTopology(Topology(**json_topology(DEFAULT_TOPOLOGY_NAME, context_id=context_id))) + +########################### +# Tests Implementation of Kpi Manager +########################### + +# ---------- 3rd Iteration Tests ---------------- +# def test_SetKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") +# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiId) + +# def test_DeleteKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") +# # adding KPI +# response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# # deleting KPI +# del_response = kpi_manager_client.DeleteKpiDescriptor(response_id) +# # select KPI +# kpi_manager_client.GetKpiDescriptor(response_id) +# LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) +# assert isinstance(del_response, Empty) + +def test_GetKpiDescriptor(kpi_manager_client): + LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") + # adding KPI + response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) + # get KPI + response = kpi_manager_client.GetKpiDescriptor(response_id) + LOGGER.info("Response gRPC message object: {:}".format(response)) + + LOGGER.info(" >>> calling GetKpiDescriptor with random ID") + rand_response = kpi_manager_client.GetKpiDescriptor(create_kpi_id_request()) + LOGGER.info("Response gRPC message object: {:}".format(rand_response)) + + assert isinstance(response, KpiDescriptor) + +# def test_SelectKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") +# # adding KPI +# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# # select KPI(s) +# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiDescriptorList) + +# def test_set_list_of_KPIs(kpi_manager_client): +# LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ") +# KPIs_TO_SEARCH = ["node_in_power_total", "node_in_current_total", "node_out_power_total"] +# # adding KPI +# for kpi in KPIs_TO_SEARCH: +# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi)) + + +# ---------- 2nd Iteration Tests ----------------- +# def test_SetKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") +# with open("kpi_manager/tests/KPI_configs.json", 'r') as file: +# data = json.load(file) +# _descriptors = data.get('KPIs', []) +# for _descritor_name in _descriptors: +# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(_descritor_name)) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiId) + +# def test_GetKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") +# response = kpi_manager_client.GetKpiDescriptor(create_kpi_id_request()) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiDescriptor) + +# def test_DeleteKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") +# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# del_response = kpi_manager_client.DeleteKpiDescriptor(response) +# kpi_manager_client.GetKpiDescriptor(response) +# LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) +# assert isinstance(del_response, Empty) + +# def test_SelectKpiDescriptor(kpi_manager_client): +# LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") +# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a()) +# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request_a()) +# LOGGER.info("Response gRPC message object: {:}".format(response)) +# assert isinstance(response, KpiDescriptorList) + +# ------------- INITIAL TESTs ---------------- +# Test case that makes use of client fixture to test server's CreateKpi method +# def test_set_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name +# # make call to server +# LOGGER.warning('test_create_kpi requesting') +# for i in range(3): +# response = kpi_manager_client.SetKpiDescriptor(create_kpi_request(str(i+1))) +# LOGGER.debug(str(response)) +# assert isinstance(response, KpiId) + +# # Test case that makes use of client fixture to test server's DeleteKpi method +# def test_delete_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name +# # make call to server +# LOGGER.warning('delete_kpi requesting') +# response = kpi_manager_client.SetKpiDescriptor(create_kpi_request('4')) +# response = kpi_manager_client.DeleteKpiDescriptor(response) +# LOGGER.debug(str(response)) +# assert isinstance(response, Empty) + +# # Test case that makes use of client fixture to test server's GetKpiDescriptor method +# def test_select_kpi_descriptor(kpi_manager_client): # pylint: disable=redefined-outer-name +# LOGGER.warning('test_selectkpidescritor begin') +# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) +# LOGGER.debug(str(response)) +# assert isinstance(response, KpiDescriptorList) diff --git a/src/kpi_manager/tests/test_messages.py b/src/kpi_manager/tests/test_messages.py new file mode 100644 index 0000000000000000000000000000000000000000..870660658a2808fc6db2e98a140497980022e5a7 --- /dev/null +++ b/src/kpi_manager/tests/test_messages.py @@ -0,0 +1,72 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid +from common.proto import kpi_manager_pb2 +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.proto.context_pb2 import DeviceId, LinkId, ServiceId, SliceId,\ + ConnectionId, EndPointId + +def create_kpi_descriptor_request(descriptor_name: str = "Test_name"): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_kpi_request.kpi_description = descriptor_name + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV2' + _create_kpi_request.service_id.service_uuid.uuid = 'SERV2' + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC1' + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END1' + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON1' + _create_kpi_request.link_id.link_uuid.uuid = 'LNK1' + return _create_kpi_request + +def create_kpi_descriptor_request_a(description: str = "Test Description"): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_kpi_request.kpi_description = description + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' + _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' + _create_kpi_request.link_id.link_uuid.uuid = 'LNK2' + return _create_kpi_request + +def create_kpi_filter_request(): + _create_kpi_filter_request = kpi_manager_pb2.KpiDescriptorFilter() + _create_kpi_filter_request.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) + + device_id_obj = DeviceId() + service_id_obj = ServiceId() + slice_id_obj = SliceId() + endpoint_id_obj = EndPointId() + connection_id_obj = ConnectionId() + link_id_obj = LinkId() + + device_id_obj.device_uuid.uuid = "DEV2" + service_id_obj.service_uuid.uuid = "SERV2" + slice_id_obj.slice_uuid.uuid = "SLC1" + endpoint_id_obj.endpoint_uuid.uuid = "END1" + connection_id_obj.connection_uuid.uuid = "CON1" + link_id_obj.link_uuid.uuid = "LNK1" + + _create_kpi_filter_request.device_id.append(device_id_obj) + _create_kpi_filter_request.service_id.append(service_id_obj) + _create_kpi_filter_request.slice_id.append(slice_id_obj) + _create_kpi_filter_request.endpoint_id.append(endpoint_id_obj) + _create_kpi_filter_request.connection_id.append(connection_id_obj) + _create_kpi_filter_request.link_id.append(link_id_obj) + + return _create_kpi_filter_request \ No newline at end of file diff --git a/src/kpi_value_api/.gitlab-ci.yml b/src/kpi_value_api/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..c9107abaac83199cc18ea720e4e3e5e14e353189 --- /dev/null +++ b/src/kpi_value_api/.gitlab-ci.yml @@ -0,0 +1,109 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build kpi-value-api: + variables: + IMAGE_NAME: 'kpi-value-api' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test kpi-value-api: + variables: + IMAGE_NAME: 'kpi-value-api' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build kpi-value-api + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker run --name $IMAGE_NAME -d -p 30020:30020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - sleep 5 + - docker ps -a + - docker logs $IMAGE_NAME + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + +## Deployment of the service in Kubernetes Cluster +#deploy context: +# variables: +# IMAGE_NAME: 'context' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - unit test context +# # - integ_test execute +# script: +# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# when: manual +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual diff --git a/src/kpi_value_api/Dockerfile b/src/kpi_value_api/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7dd8d307b8338c4a29e97c742ca12a49c4611e0a --- /dev/null +++ b/src/kpi_value_api/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/kpi_value_api +WORKDIR /var/teraflow/kpi_value_api +COPY src/kpi_value_api/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/kpi_value_api/. kpi_value_api/ + +# Start the service +ENTRYPOINT ["python", "-m", "kpi_value_api.service"] diff --git a/src/kpi_value_api/__init__.py b/src/kpi_value_api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_value_api/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_value_api/client/KpiValueApiClient.py b/src/kpi_value_api/client/KpiValueApiClient.py new file mode 100644 index 0000000000000000000000000000000000000000..f432271cfb7c8136f72156330b25d0b82b934d99 --- /dev/null +++ b/src/kpi_value_api/client/KpiValueApiClient.py @@ -0,0 +1,63 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.tools.grpc.Tools import grpc_message_to_json_string + +from common.proto.context_pb2 import Empty +from common.proto.kpi_value_api_pb2 import KpiValueList, KpiValueFilter +from common.proto.kpi_value_api_pb2_grpc import KpiValueAPIServiceStub + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 10 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class KpiValueApiClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.KPIVALUEAPI) + if not port: port = get_service_port_grpc(ServiceNameEnum.KPIVALUEAPI) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = KpiValueAPIServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def StoreKpiValues(self, request: KpiValueList) -> Empty: + LOGGER.debug('StoreKpiValues: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StoreKpiValues(request) + LOGGER.debug('StoreKpiValues result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def SelectKpiValues(self, request: KpiValueFilter) -> KpiValueList: + LOGGER.debug('SelectKpiValues: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SelectKpiValues(request) + LOGGER.debug('SelectKpiValues result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/kpi_value_api/client/__init__.py b/src/kpi_value_api/client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_value_api/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_value_api/requirements.in b/src/kpi_value_api/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..7e4694109dc4e1d31b86abfc03162494faafcdaf --- /dev/null +++ b/src/kpi_value_api/requirements.in @@ -0,0 +1,16 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +confluent-kafka==2.3.* +requests==2.27.* diff --git a/src/kpi_value_api/service/KpiValueApiService.py b/src/kpi_value_api/service/KpiValueApiService.py new file mode 100644 index 0000000000000000000000000000000000000000..68b6fbdc278a00aa7cf98385bcf8afa573f91445 --- /dev/null +++ b/src/kpi_value_api/service/KpiValueApiService.py @@ -0,0 +1,30 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from .KpiValueApiServiceServicerImpl import KpiValueApiServiceServicerImpl +from common.proto.kpi_value_api_pb2_grpc import add_KpiValueAPIServiceServicer_to_server + + +class KpiValueApiService(GenericGrpcService): + def __init__(self, cls_name : str = __name__ ) -> None: + port = get_service_port_grpc(ServiceNameEnum.KPIVALUEAPI) + super().__init__(port, cls_name=cls_name) + self.kpiValueApiService_servicer = KpiValueApiServiceServicerImpl() + + def install_servicers(self): + add_KpiValueAPIServiceServicer_to_server(self.kpiValueApiService_servicer, self.server) diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py new file mode 100644 index 0000000000000000000000000000000000000000..d27de54f3cddfd0d70d656a89c45adc50e518289 --- /dev/null +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -0,0 +1,118 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, grpc, requests +from typing import Tuple, Any +from datetime import datetime +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic + +from common.proto.context_pb2 import Empty +from common.proto.kpi_value_api_pb2_grpc import KpiValueAPIServiceServicer +from common.proto.kpi_value_api_pb2 import KpiValueList, KpiValueFilter, KpiValue, KpiValueType + +from confluent_kafka import Producer as KafkaProducer + + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('KpiValueAPI', 'NBIgRPC') +PROM_URL = "http://localhost:9090" + +class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): + def __init__(self): + LOGGER.debug('Init KpiValueApiService') + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StoreKpiValues(self, request: KpiValueList, grpc_context: grpc.ServicerContext + ) -> Empty: + LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) + producer_obj = KafkaProducer({ + 'bootstrap.servers' : KafkaConfig.SERVER_IP.value + }) + for kpi_value in request.kpi_value_list: + kpi_value_to_produce : Tuple [str, Any, Any] = ( + kpi_value.kpi_id.kpi_id, + kpi_value.timestamp, + kpi_value.kpi_value_type # kpi_value.kpi_value_type.(many options) how? + ) + LOGGER.debug('KPI to produce is {:}'.format(kpi_value_to_produce)) + msg_key = "gRPC-kpivalueapi" # str(__class__.__name__) can be used + + producer_obj.produce( + KafkaTopic.VALUE.value, + key = msg_key, + value = kpi_value.SerializeToString(), # value = json.dumps(kpi_value_to_produce), + callback = self.delivery_callback + ) + producer_obj.flush() + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SelectKpiValues(self, request: KpiValueFilter, grpc_context: grpc.ServicerContext + ) -> KpiValueList: + LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) + response = KpiValueList() + metrics = [kpi.kpi_id for kpi in request.kpi_id] + start_timestamps = [timestamp for timestamp in request.start_timestamp] + end_timestamps = [timestamp for timestamp in request.end_timestamp] + results = [] + + for start, end in zip(start_timestamps, end_timestamps): + start_str = datetime.fromtimestamp(start.seconds).isoformat() + "Z" + end_str = datetime.fromtimestamp(end.seconds).isoformat() + "Z" + + for metric in metrics: + url = f'{PROM_URL}/api/v1/query_range' + params = { + 'query': metric, + 'start': start_str, + 'end' : end_str, + 'step' : '30s' # or any other step you need + } + response = requests.get(url, params=params) + if response.status_code == 200: + data = response.json() + for result in data['data']['result']: + for value in result['values']: + kpi_value = KpiValue( + kpi_id=metric, + timestamp=str(seconds=value[0]), + kpi_value_type=self._convert_value_to_kpi_value_type(value[1]) + ) + results.append(kpi_value) + + def _convert_value_to_kpi_value_type(self, value): + # Check if the value is an integer (int64) + try: + int64_value = int(value) + return KpiValueType(int64Val=int64_value) + except ValueError: + pass + # Check if the value is a float + try: + float_value = float(value) + return KpiValueType(floatVal=float_value) + except ValueError: + pass + # Check if the value is a boolean + if value.lower() in ['true', 'false']: + bool_value = value.lower() == 'true' + return KpiValueType(boolVal=bool_value) + # If none of the above, treat it as a string + return KpiValueType(stringVal=value) + + + def delivery_callback(self, err, msg): + if err: LOGGER.debug('Message delivery failed: {:}'.format(err)) + else: LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) diff --git a/src/kpi_value_api/service/__init__.py b/src/kpi_value_api/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_value_api/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_value_api/service/__main__.py b/src/kpi_value_api/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b4ebe296e2c4f193aa1fc99aede9364556c2094 --- /dev/null +++ b/src/kpi_value_api/service/__main__.py @@ -0,0 +1,52 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from prometheus_client import start_http_server +from common.Settings import get_log_level +from .KpiValueApiService import KpiValueApiService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = KpiValueApiService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/kpi_value_api/tests/messages.py b/src/kpi_value_api/tests/messages.py new file mode 100644 index 0000000000000000000000000000000000000000..c2a1cbb0b275fb26d6498e4470f3869a105a8d36 --- /dev/null +++ b/src/kpi_value_api/tests/messages.py @@ -0,0 +1,35 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid, time +from common.proto.kpi_value_api_pb2 import KpiValue, KpiValueList + + +def create_kpi_value_list(): + _create_kpi_value_list = KpiValueList() + # To run this experiment sucessfully, already existing UUID in KPI DB in necessary. + # because the UUID is used to get the descriptor form KPI DB. + EXISTING_KPI_IDs = ["725ce3ad-ac67-4373-bd35-8cd9d6a86e09", + str(uuid.uuid4()), + str(uuid.uuid4())] + + for kpi_id_uuid in EXISTING_KPI_IDs: + kpi_value_object = KpiValue() + kpi_value_object.kpi_id.kpi_id.uuid = kpi_id_uuid + kpi_value_object.timestamp.timestamp = float(time.time()) + kpi_value_object.kpi_value_type.floatVal = 100 + + _create_kpi_value_list.kpi_value_list.append(kpi_value_object) + + return _create_kpi_value_list diff --git a/src/kpi_value_api/tests/test_kpi_value_api.py b/src/kpi_value_api/tests/test_kpi_value_api.py new file mode 100644 index 0000000000000000000000000000000000000000..307b5cdad4e6503a774e308f669fc44762f84bf1 --- /dev/null +++ b/src/kpi_value_api/tests/test_kpi_value_api.py @@ -0,0 +1,84 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os, logging, pytest +from common.proto.context_pb2 import Empty +from common.Constants import ServiceNameEnum +from common.tools.kafka.Variables import KafkaTopic +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) +from kpi_value_api.service.KpiValueApiService import KpiValueApiService +from kpi_value_api.client.KpiValueApiClient import KpiValueApiClient +from kpi_value_api.tests.messages import create_kpi_value_list + + +LOCAL_HOST = '127.0.0.1' +KPIVALUEAPI_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.KPIVALUEAPI) # type: ignore +os.environ[get_env_var_name(ServiceNameEnum.KPIVALUEAPI, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.KPIVALUEAPI, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(KPIVALUEAPI_SERVICE_PORT) +LOGGER = logging.getLogger(__name__) + +# This fixture will be requested by test cases and last during testing session +@pytest.fixture(scope='session') +def kpi_value_api_service(): + LOGGER.info('Initializing KpiValueApiService...') + # _service = MonitoringService(name_mapping) + _service = KpiValueApiService() + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiValueApiService...') + yield _service + + LOGGER.info('Terminating KpiValueApiService...') + _service.stop() + + LOGGER.info('Terminated KpiValueApiService...') + +# This fixture will be requested by test cases and last during testing session. +# The client requires the server, so client fixture has the server as dependency. +@pytest.fixture(scope='session') +def kpi_value_api_client(kpi_value_api_service : KpiValueApiService ): + LOGGER.info('Initializing KpiValueApiClient...') + _client = KpiValueApiClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding KpiValueApiClient...') + yield _client + + LOGGER.info('Closing KpiValueApiClient...') + _client.close() + + LOGGER.info('Closed KpiValueApiClient...') + +################################################## +# Prepare Environment, should be the first test +################################################## + +# To be added here + +########################### +# Tests Implementation of Kpi Value Api +########################### + +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) + +def test_store_kpi_values(kpi_value_api_client): + LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ") + response = kpi_value_api_client.StoreKpiValues(create_kpi_value_list()) + assert isinstance(response, Empty) diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..52b1b8fe6fd41c5fd8b43adf4ca8da464c8c08ba --- /dev/null +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -0,0 +1,109 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build kpi-value-writer: + variables: + IMAGE_NAME: 'kpi-value-writer' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test kpi-value-writer: + variables: + IMAGE_NAME: 'kpi-value-writer' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build kpi-value-writer + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker run --name $IMAGE_NAME -d -p 30030:30030 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - sleep 5 + - docker ps -a + - docker logs $IMAGE_NAME + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + +## Deployment of the service in Kubernetes Cluster +#deploy context: +# variables: +# IMAGE_NAME: 'context' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - unit test context +# # - integ_test execute +# script: +# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# when: manual +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual diff --git a/src/kpi_value_writer/Dockerfile b/src/kpi_value_writer/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..70f41128bd8c982f604a3424d2096c918ead080e --- /dev/null +++ b/src/kpi_value_writer/Dockerfile @@ -0,0 +1,70 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/kpi_value_writer +WORKDIR /var/teraflow/kpi_value_writer +COPY src/kpi_value_writer/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/kpi_value_writer/. kpi_value_writer/ +COPY src/kpi_manager/__init__.py kpi_manager/__init__.py +COPY src/kpi_manager/client/. kpi_manager/client/ + +# Start the service +ENTRYPOINT ["python", "-m", "kpi_value_writer.service"] diff --git a/src/kpi_value_writer/README.md b/src/kpi_value_writer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..72ba6e5594adeef4a29d650615716c26273ed115 --- /dev/null +++ b/src/kpi_value_writer/README.md @@ -0,0 +1,29 @@ +# How to locally run and test KPI manager micro-service + +## --- File links need to be updated. --- +### Pre-requisets +The following requirements should be fulfilled before the execuation of KPI management service. + +1. Verify that [kpi_management.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/kpi_management.proto) file exists and grpcs file are generated sucessfully. +2. Virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/requirements.in) are installed sucessfully. +3. Verify the creation of required database and table. +[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/database/tests/KpiDBtests.py) python file enlist the functions to create tables and database and +[KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/service/database/KpiEngine.py) contains the DB string, update the string as per your deployment. + +### Messages format templates +["Messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_messages.py) python file enlist the basic gRPC messages format used during the testing. + +### Test file +["KPI management test"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_kpi_manager.py) python file enlist different tests conducted during the experiment. + +### Flow of execution (Kpi Maanager Service functions) +1. Call the `create_database()` and `create_tables()` functions from `Kpi_DB` class to create the required database and table if they don't exist. Call `verify_tables` to verify the existence of KPI table. + +2. Call the gRPC method `SetKpiDescriptor(KpiDescriptor)->KpiId` to add the KpiDescriptor in `Kpi` DB. `KpiDescriptor` and `KpiId` are both pre-defined gRPC message types. + +3. Call `GetKpiDescriptor(KpiId)->KpiDescriptor` to read the `KpiDescriptor` from DB and `DeleteKpiDescriptor(KpiId)` to delete the `KpiDescriptor` from DB. + +4. Call `SelectKpiDescriptor(KpiDescriptorFilter)->KpiDescriptorList` to get all `KpiDescriptor` objects that matches the filter criteria. `KpiDescriptorFilter` and `KpiDescriptorList` are pre-defined gRPC message types. + +## For KPI composer and KPI writer +The functionalities of KPI composer and writer is heavily dependent upon Telemetery service. Therfore, these services has other pre-requsites that are mention [here](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/requirements.in). \ No newline at end of file diff --git a/src/kpi_value_writer/__init__.py b/src/kpi_value_writer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_value_writer/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_value_writer/requirements.in b/src/kpi_value_writer/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..7e4694109dc4e1d31b86abfc03162494faafcdaf --- /dev/null +++ b/src/kpi_value_writer/requirements.in @@ -0,0 +1,16 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +confluent-kafka==2.3.* +requests==2.27.* diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py new file mode 100644 index 0000000000000000000000000000000000000000..a4b10ed6391c4ff0b0ee45a287ce9f12d77e2dea --- /dev/null +++ b/src/kpi_value_writer/service/KpiValueWriter.py @@ -0,0 +1,98 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic +from common.proto.kpi_value_api_pb2 import KpiValue +from common.proto.kpi_manager_pb2 import KpiDescriptor, KpiId + +from confluent_kafka import KafkaError +from confluent_kafka import Consumer as KafkaConsumer + +from kpi_manager.client.KpiManagerClient import KpiManagerClient +# -- test import -- +from kpi_value_writer.tests.test_messages import create_kpi_descriptor_request +from .MetricWriterToPrometheus import MetricWriterToPrometheus + + +LOGGER = logging.getLogger(__name__) +ACTIVE_CONSUMERS = [] + +class KpiValueWriter: + @staticmethod + def RunKafkaConsumer(): + thread = threading.Thread(target=KpiValueWriter.KafkaConsumer, args=()) + ACTIVE_CONSUMERS.append(thread) + thread.start() + + @staticmethod + def KafkaConsumer(): + kafka_consumer = KafkaConsumer( + { 'bootstrap.servers' : KafkaConfig.SERVER_IP.value, + 'group.id' : __class__, + 'auto.offset.reset' : 'latest'} + ) + + metric_writer_to_prom = MetricWriterToPrometheus() + kpi_manager_client = KpiManagerClient() + print("Kpi manger client created: {:}".format(kpi_manager_client)) + + kafka_consumer.subscribe([KafkaTopic.VALUE.value]) + LOGGER.debug("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) + print("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) + while True: + raw_kpi = kafka_consumer.poll(1.0) + if raw_kpi is None: + continue + elif raw_kpi.error(): + if raw_kpi.error().code() == KafkaError._PARTITION_EOF: + continue + else: + print("Consumer error: {}".format(raw_kpi.error())) + continue + try: + kpi_value = KpiValue() + kpi_value.ParseFromString(raw_kpi.value()) + LOGGER.info("Received KPI : {:}".format(kpi_value)) + print("Received KPI : {:}".format(kpi_value)) + KpiValueWriter.get_kpi_descriptor(kpi_value, kpi_manager_client) + except Exception as e: + print("Error detail: {:}".format(e)) + continue + + @staticmethod + def get_kpi_descriptor(kpi_value: str, kpi_manager_client ): + print("--- START -----") + + kpi_id = KpiId() + kpi_id.kpi_id.uuid = kpi_value.kpi_id.kpi_id.uuid + print("KpiId generated: {:}".format(kpi_id)) + # print("Kpi manger client created: {:}".format(kpi_manager_client)) + + try: + kpi_descriptor_object = KpiDescriptor() + kpi_descriptor_object = kpi_manager_client.GetKpiDescriptor(kpi_id) + + print("kpi descriptor received: {:}".format(kpi_descriptor_object)) + if isinstance (kpi_descriptor_object, KpiDescriptor): + LOGGER.info("Extracted KpiDescriptor: {:}".format(kpi_descriptor_object)) + print("Extracted KpiDescriptor: {:}".format(kpi_descriptor_object)) + MetricWriterToPrometheus.create_and_expose_cooked_kpi(kpi_descriptor_object, kpi_value) + else: + LOGGER.info("Error in extracting KpiDescriptor {:}".format(kpi_descriptor_object)) + print("Error in extracting KpiDescriptor {:}".format(kpi_descriptor_object)) + except Exception as e: + LOGGER.info("Unable to get KpiDescriptor. Error: {:}".format(e)) + print ("Unable to get KpiDescriptor. Error: {:}".format(e)) diff --git a/src/kpi_value_writer/service/KpiWriterOld.py b/src/kpi_value_writer/service/KpiWriterOld.py new file mode 100644 index 0000000000000000000000000000000000000000..b9a4316b0f5decb2364127052ebf0b44edc05fbd --- /dev/null +++ b/src/kpi_value_writer/service/KpiWriterOld.py @@ -0,0 +1,108 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# read Kafka stream from Kafka topic + +import ast +import time +import threading +from confluent_kafka import KafkaError +from prometheus_client import start_http_server, Gauge, CollectorRegistry +from confluent_kafka import Consumer as KafkaConsumer + +KAFKA_SERVER_IP = '127.0.0.1:9092' +KAFKA_TOPICS = {'request' : 'topic_request', 'response': 'topic_response', + 'raw' : 'topic_raw' , 'labeled' : 'topic_labeled'} +CONSUMER_CONFIG = {'bootstrap.servers' : KAFKA_SERVER_IP, + 'group.id' : 'kpi_writer', + 'auto.offset.reset' : 'latest'} +KPIs_TO_SEARCH = ["node_network_receive_packets_total", + "node_network_receive_bytes_total", + "node_network_transmit_bytes_total", + "process_open_fds"] +PROM_METRICS = {} +KAFKA_REGISTERY = CollectorRegistry() + +class KpiWriter: + def __init__(self) -> None: + pass + + @staticmethod + def kpi_writer(): + KpiWriter.create_prom_metrics_name() + threading.Thread(target=KpiWriter.kafka_listener, args=()).start() + + @staticmethod + def kafka_listener(): + """ + listener for events on Kafka topic. + """ + # Start up the server to expose the metrics at port number mention below. + start_http_server(8101, registry=KAFKA_REGISTERY) + kafka_consumer = KafkaConsumer(CONSUMER_CONFIG) + kafka_consumer.subscribe([KAFKA_TOPICS['labeled']]) + while True: + receive_msg = kafka_consumer.poll(2.0) + if receive_msg is None: + # print (" - Telemetry frontend listening on Kafka Topic: ", KAFKA_TOPICS['raw']) # added for debugging purposes + continue + elif receive_msg.error(): + if receive_msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + print("Consumer error: {}".format(receive_msg.error())) + continue + try: + new_event = receive_msg.value().decode('utf-8') + # print("New event on topic '{:}' is {:}".format(KAFKA_TOPICS['raw'], new_event)) + # LOGGER.info("New event on topic '{:}' is {:}".format(KAFKA_TOPICS['raw'], new_event)) + KpiWriter.write_metric_to_promtheus(new_event) + except Exception as e: + print(f"Error to consume event from topic: {KAFKA_TOPICS['labeled']}. Error detail: {str(e)}") + continue + + # send metric to Prometheus + @staticmethod + def write_metric_to_promtheus(event): + event = ast.literal_eval(event) # converted into dict + print("New recevied event: {:}".format(event)) + event_kpi_name = event['kpi_description'] + if event_kpi_name in KPIs_TO_SEARCH: + PROM_METRICS[event_kpi_name].labels( + kpi_id = event['kpi_id'], + kpi_sample_type = event['kpi_sample_type'], + device_id = event['device_id'], + endpoint_id = event['endpoint_id'], + service_id = event['service_id'], + slice_id = event['slice_id'], + connection_id = event['connection_id'], + link_id = event['link_id'] + ).set(float(event['kpi_value'])) + time.sleep(0.05) + + @staticmethod + def create_prom_metrics_name(): + metric_tags = ['kpi_id','kpi_sample_type','device_id', + 'endpoint_id','service_id','slice_id','connection_id','link_id'] + for metric_key in KPIs_TO_SEARCH: + metric_name = metric_key + metric_description = "description of " + str(metric_key) + try: + PROM_METRICS[metric_key] = Gauge ( + metric_name, metric_description, metric_tags, + registry=KAFKA_REGISTERY ) + # print("Metric pushed to Prometheus: {:}".format(PROM_METRICS[metric_key])) + except ValueError as e: + if 'Duplicated timeseries' in str(e): + print("Metric {:} is already registered. Skipping.".format(metric_name)) diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py new file mode 100644 index 0000000000000000000000000000000000000000..b681164786bd310d457998bae55b836522888b94 --- /dev/null +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -0,0 +1,96 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# read Kafka stream from Kafka topic + +import ast +import time +import threading +import logging +from prometheus_client import start_http_server, Gauge, CollectorRegistry +from common.proto.kpi_sample_types_pb2 import KpiSampleType + +from common.proto.kpi_value_api_pb2 import KpiValue +from common.proto.kpi_manager_pb2 import KpiDescriptor + +LOGGER = logging.getLogger(__name__) +PROM_METRICS = {} +PROM_REGISTERY = CollectorRegistry() + +class MetricWriterToPrometheus: + ''' + This class exposes the *cooked KPI* on the endpoint to be scraped by the Prometheus server. + cooked KPI value = KpiDescriptor (gRPC message) + KpiValue (gRPC message) + ''' + def __init__(self): + # prometheus server address and configs + self.start_prometheus_client() + pass + + def start_prometheus_client(self): + start_http_server(10808, registry=PROM_REGISTERY) + LOGGER.debug("Prometheus client is started on port 10808") + + def merge_kpi_descriptor_and_kpi_value(self, kpi_descriptor, kpi_value): + # Creating a dictionary from the kpi_descriptor's attributes + cooked_kpi = { + 'kpi_id' : kpi_descriptor.kpi_id.kpi_id.uuid, + 'kpi_description': kpi_descriptor.kpi_description, + 'kpi_sample_type': KpiSampleType.Name(kpi_descriptor.kpi_sample_type), + 'device_id' : kpi_descriptor.device_id.device_uuid.uuid, + 'endpoint_id' : kpi_descriptor.endpoint_id.endpoint_uuid.uuid, + 'service_id' : kpi_descriptor.service_id.service_uuid.uuid, + 'slice_id' : kpi_descriptor.slice_id.slice_uuid.uuid, + 'connection_id' : kpi_descriptor.connection_id.connection_uuid.uuid, + 'link_id' : kpi_descriptor.link_id.link_uuid.uuid, + 'time_stamp' : kpi_value.timestamp.timestamp, + 'kpi_value' : kpi_value.kpi_value_type.floatVal + } + # LOGGER.debug("Cooked Kpi: {:}".format(cooked_kpi)) + return cooked_kpi + + def create_and_expose_cooked_kpi(self, kpi_descriptor: KpiDescriptor, kpi_value: KpiValue): + # merge both gRPC messages into single varible. + cooked_kpi = self.merge_kpi_descriptor_and_kpi_value(kpi_descriptor, kpi_value) + tags_to_exclude = {'kpi_description', 'kpi_sample_type', 'kpi_value'} # extracted values will be used as metric tag + metric_tags = [tag for tag in cooked_kpi.keys() if tag not in tags_to_exclude] + metric_name = cooked_kpi['kpi_sample_type'] + try: + if metric_name not in PROM_METRICS: # Only register the metric, when it doesn't exists + PROM_METRICS[metric_name] = Gauge ( + metric_name, + cooked_kpi['kpi_description'], + metric_tags, + registry=PROM_REGISTERY + ) + LOGGER.debug("Metric is created with labels: {:}".format(metric_tags)) + PROM_METRICS[metric_name].labels( + kpi_id = cooked_kpi['kpi_id'], + device_id = cooked_kpi['device_id'], + endpoint_id = cooked_kpi['endpoint_id'], + service_id = cooked_kpi['service_id'], + slice_id = cooked_kpi['slice_id'], + connection_id = cooked_kpi['connection_id'], + link_id = cooked_kpi['link_id'], + time_stamp = cooked_kpi['time_stamp'], + ).set(float(cooked_kpi['kpi_value'])) + LOGGER.debug("Metric pushed to the endpoints: {:}".format(PROM_METRICS[metric_name])) + + except ValueError as e: + if 'Duplicated timeseries' in str(e): + LOGGER.debug("Metric {:} is already registered. Skipping.".format(metric_name)) + print("Metric {:} is already registered. Skipping.".format(metric_name)) + else: + LOGGER.error("Error while pushing metric: {}".format(e)) + raise \ No newline at end of file diff --git a/src/kpi_value_writer/service/__init__.py b/src/kpi_value_writer/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/kpi_value_writer/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_value_writer/service/__main__.py b/src/kpi_value_writer/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa67540fb899781297d1235dc2e15bcbb2c38585 --- /dev/null +++ b/src/kpi_value_writer/service/__main__.py @@ -0,0 +1,51 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from kpi_value_writer.service.KpiValueWriter import KpiValueWriter +from common.Settings import get_log_level + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = KpiValueWriter() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/kpi_value_writer/tests/test_kpi_value_writer.py b/src/kpi_value_writer/tests/test_kpi_value_writer.py new file mode 100755 index 0000000000000000000000000000000000000000..572495d48d70cdc40c0ef6bb1efcf877e2a610ee --- /dev/null +++ b/src/kpi_value_writer/tests/test_kpi_value_writer.py @@ -0,0 +1,52 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from kpi_value_writer.service.KpiValueWriter import KpiValueWriter +from common.tools.kafka.Variables import KafkaTopic +from kpi_manager.client.KpiManagerClient import KpiManagerClient +from kpi_manager.tests.test_messages import create_kpi_descriptor_request +from common.proto.kpi_manager_pb2 import KpiDescriptor +from kpi_value_writer.tests.test_messages import create_kpi_id_request + +LOGGER = logging.getLogger(__name__) + +# def test_GetKpiDescriptor(): +# LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") +# kpi_manager_client = KpiManagerClient() +# # adding KPI +# LOGGER.info(" --->>> calling SetKpiDescriptor ") +# response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) +# # get KPI +# LOGGER.info(" --->>> calling GetKpiDescriptor with response ID") +# response = kpi_manager_client.GetKpiDescriptor(response_id) +# LOGGER.info("Response gRPC message object: {:}".format(response)) + +# LOGGER.info(" --->>> calling GetKpiDescriptor with random ID") +# rand_response = kpi_manager_client.GetKpiDescriptor(create_kpi_id_request()) +# LOGGER.info("Response gRPC message object: {:}".format(rand_response)) + +# LOGGER.info("\n------------------ TEST FINISHED ---------------------\n") +# assert isinstance(response, KpiDescriptor) + +# -------- Initial Test ---------------- +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) + +def test_KafkaConsumer(): + LOGGER.debug(" --->>> test_kafka_consumer: START <<<--- ") + KpiValueWriter.RunKafkaConsumer() + diff --git a/src/kpi_value_writer/tests/test_messages.py b/src/kpi_value_writer/tests/test_messages.py new file mode 100755 index 0000000000000000000000000000000000000000..89a41fa08ad37b7d9b305bba6e7c445fea5cd18a --- /dev/null +++ b/src/kpi_value_writer/tests/test_messages.py @@ -0,0 +1,44 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid, time +import random +from common.proto import kpi_manager_pb2 +from common.proto.kpi_value_api_pb2 import KpiValue +from common.proto.kpi_sample_types_pb2 import KpiSampleType + +def create_kpi_id_request(): + _create_kpi_id = kpi_manager_pb2.KpiId() + _create_kpi_id.kpi_id.uuid = str(uuid.uuid4()) + return _create_kpi_id + +def create_kpi_descriptor_request(description: str = "Test Description"): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_kpi_request.kpi_description = description + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.device_id.device_uuid.uuid = 'DEV4' + _create_kpi_request.service_id.service_uuid.uuid = 'SERV3' + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC3' + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END2' + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON2' + _create_kpi_request.link_id.link_uuid.uuid = 'LNK2' + return _create_kpi_request + +def create_kpi_value_request(): + _create_kpi_value_request = KpiValue() + _create_kpi_value_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_kpi_value_request.timestamp.timestamp = time.time() + _create_kpi_value_request.kpi_value_type.floatVal = random.randint(10, 10000) + return _create_kpi_value_request diff --git a/src/kpi_value_writer/tests/test_metric_writer_to_prom.py b/src/kpi_value_writer/tests/test_metric_writer_to_prom.py new file mode 100644 index 0000000000000000000000000000000000000000..f60e96253ae8edb29eedcbe2d6e66aaeb450229c --- /dev/null +++ b/src/kpi_value_writer/tests/test_metric_writer_to_prom.py @@ -0,0 +1,28 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading +import logging +from kpi_value_writer.service.MetricWriterToPrometheus import MetricWriterToPrometheus +from kpi_value_writer.tests.test_messages import create_kpi_descriptor_request, create_kpi_value_request + +LOGGER = logging.getLogger(__name__) + +def test_metric_writer_to_prometheus(): + LOGGER.info(' >>> test_metric_writer_to_prometheus START <<< ') + metric_writer_obj = MetricWriterToPrometheus() + metric_writer_obj.create_and_expose_cooked_kpi( + create_kpi_descriptor_request(), + create_kpi_value_request() + ) diff --git a/src/telemetry/README.md b/src/telemetry/README.md new file mode 100644 index 0000000000000000000000000000000000000000..da43bd471c384ae9133871a097e94043f70ed7de --- /dev/null +++ b/src/telemetry/README.md @@ -0,0 +1,10 @@ +# How to locally run and test Telemetry service + +### Pre-requisets +The following requirements should be fulfilled before the execuation of Telemetry service. + +1. verify that [telmetry_frontend.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/telemetry_frontend.proto) file exists and grpcs file are generated sucessfully. +2. virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/telemetry_virenv.txt) are installed sucessfully. +3. verify the creation of required database and table. +[DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/database/tests/managementDBtests.py) python file enlist the functions to create tables and database. +[KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_manager/service/database/KpiEngine.py) contains the DB string, update the string as per your deployment. diff --git a/src/telemetry/__init__.py b/src/telemetry/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..234a1af6588c91f6a17f3963f69120cd6e2248d9 --- /dev/null +++ b/src/telemetry/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/backend/__init__.py b/src/telemetry/backend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bbfc943b68af13a11e562abbc8680ade71db8f02 --- /dev/null +++ b/src/telemetry/backend/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py new file mode 100755 index 0000000000000000000000000000000000000000..d81be79dbe410ccbf2781816f34735f6bfe5639d --- /dev/null +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -0,0 +1,253 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import time +import random +import logging +import requests +import threading +from typing import Any, Tuple +from common.proto.context_pb2 import Empty +from confluent_kafka import Producer as KafkaProducer +from confluent_kafka import Consumer as KafkaConsumer +from confluent_kafka import KafkaException +from confluent_kafka import KafkaError +from confluent_kafka.admin import AdminClient, NewTopic +from common.proto.telemetry_frontend_pb2 import Collector, CollectorId +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('Telemetry', 'TelemetryBackend') +KAFKA_SERVER_IP = '127.0.0.1:9092' +# KAFKA_SERVER_IP = '10.152.183.175:30092' +ADMIN_KAFKA_CLIENT = AdminClient({'bootstrap.servers': KAFKA_SERVER_IP}) +KAFKA_TOPICS = {'request' : 'topic_request', 'response': 'topic_response', + 'raw' : 'topic_raw' , 'labeled' : 'topic_labeled'} +EXPORTER_ENDPOINT = "http://10.152.183.2:9100/metrics" +PRODUCER_CONFIG = {'bootstrap.servers': KAFKA_SERVER_IP,} + + +class TelemetryBackendService: + """ + Class to listens for request on Kafka topic, fetches metrics and produces measured values to another Kafka topic. + """ + + def __init__(self): + LOGGER.info('Init TelemetryBackendService') + self.running_threads = {} + + def run_kafka_listener(self)->bool: + threading.Thread(target=self.kafka_listener).start() + return True + + def kafka_listener(self): + """ + listener for requests on Kafka topic. + """ + conusmer_configs = { + 'bootstrap.servers' : KAFKA_SERVER_IP, + 'group.id' : 'backend', + 'auto.offset.reset' : 'latest' + } + # topic_request = "topic_request" + consumerObj = KafkaConsumer(conusmer_configs) + # consumerObj.subscribe([topic_request]) + consumerObj.subscribe([KAFKA_TOPICS['request']]) + + while True: + receive_msg = consumerObj.poll(2.0) + if receive_msg is None: + # print (time.time(), " - Telemetry backend is listening on Kafka Topic: ", KAFKA_TOPICS['request']) # added for debugging purposes + continue + elif receive_msg.error(): + if receive_msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + print("Consumer error: {}".format(receive_msg.error())) + break + (kpi_id, duration, interval) = ast.literal_eval(receive_msg.value().decode('utf-8')) + collector_id = receive_msg.key().decode('utf-8') + if duration == -1 and interval == -1: + self.terminate_collector_backend(collector_id) + # threading.Thread(target=self.terminate_collector_backend, args=(collector_id)) + else: + self.run_initiate_collector_backend(collector_id, kpi_id, duration, interval) + + + def run_initiate_collector_backend(self, collector_id: str, kpi_id: str, duration: int, interval: int): + stop_event = threading.Event() + thread = threading.Thread(target=self.initiate_collector_backend, + args=(collector_id, kpi_id, duration, interval, stop_event)) + self.running_threads[collector_id] = (thread, stop_event) + thread.start() + + def initiate_collector_backend(self, collector_id, kpi_id, duration, interval, stop_event + ): # type: ignore + """ + Method to receive collector request attribues and initiates collecter backend. + """ + print("Initiating backend for collector: ", collector_id) + start_time = time.time() + while not stop_event.is_set(): + if time.time() - start_time >= duration: # condition to terminate backend + print("Execuation duration completed: Terminating backend: Collector Id: ", collector_id, " - ", time.time() - start_time) + self.generate_kafka_response(collector_id, "-1", -1) + # write to Kafka to send the termination confirmation. + break + # print ("Received KPI: ", kpi_id, ", Duration: ", duration, ", Fetch Interval: ", interval) + self.extract_kpi_value(collector_id, kpi_id) + # print ("Telemetry Backend running for KPI: ", kpi_id, "after FETCH INTERVAL: ", interval) + time.sleep(interval) + + def extract_kpi_value(self, collector_id: str, kpi_id: str): + """ + Method to extract kpi value. + """ + measured_kpi_value = random.randint(1,100) # Should be extracted from exporter/stream + # measured_kpi_value = self.fetch_node_exporter_metrics() # exporter extracted metric value against default KPI + self.generate_kafka_response(collector_id, kpi_id , measured_kpi_value) + + def generate_kafka_response(self, collector_id: str, kpi_id: str, kpi_value: Any): + """ + Method to write response on Kafka topic + """ + # topic_response = "topic_response" + msg_value : Tuple [str, Any] = (kpi_id, kpi_value) + msg_key = collector_id + producerObj = KafkaProducer(PRODUCER_CONFIG) + # producerObj.produce(topic_response, key=msg_key, value= str(msg_value), callback=self.delivery_callback) + producerObj.produce(KAFKA_TOPICS['response'], key=msg_key, value= str(msg_value), callback=TelemetryBackendService.delivery_callback) + producerObj.flush() + + def terminate_collector_backend(self, collector_id): + if collector_id in self.running_threads: + thread, stop_event = self.running_threads[collector_id] + stop_event.set() + thread.join() + print ("Terminating backend (by StopCollector): Collector Id: ", collector_id) + del self.running_threads[collector_id] + self.generate_kafka_response(collector_id, "-1", -1) + + def create_topic_if_not_exists(self, new_topics: list) -> bool: + """ + Method to create Kafka topic if it does not exist. + Args: + admin_client (AdminClient): Kafka admin client. + """ + for topic in new_topics: + try: + topic_metadata = ADMIN_KAFKA_CLIENT.list_topics(timeout=5) + if topic not in topic_metadata.topics: + # If the topic does not exist, create a new topic + print(f"Topic '{topic}' does not exist. Creating...") + LOGGER.warning("Topic {:} does not exist. Creating...".format(topic)) + new_topic = NewTopic(topic, num_partitions=1, replication_factor=1) + ADMIN_KAFKA_CLIENT.create_topics([new_topic]) + except KafkaException as e: + print(f"Failed to create topic: {e}") + return False + return True + + @staticmethod + def delivery_callback( err, msg): + """ + Callback function to handle message delivery status. + Args: + err (KafkaError): Kafka error object. + msg (Message): Kafka message object. + """ + if err: + print(f'Message delivery failed: {err}') + else: + print(f'Message delivered to topic {msg.topic()}') + +# ----------- BELOW: Actual Implementation of Kafka Producer with Node Exporter ----------- + @staticmethod + def fetch_single_node_exporter_metric(): + """ + Method to fetch metrics from Node Exporter. + Returns: + str: Metrics fetched from Node Exporter. + """ + KPI = "node_network_receive_packets_total" + try: + response = requests.get(EXPORTER_ENDPOINT) # type: ignore + LOGGER.info("Request status {:}".format(response)) + if response.status_code == 200: + # print(f"Metrics fetched sucessfully...") + metrics = response.text + # Check if the desired metric is available in the response + if KPI in metrics: + KPI_VALUE = TelemetryBackendService.extract_metric_value(metrics, KPI) + # Extract the metric value + if KPI_VALUE is not None: + LOGGER.info("Extracted value of {:} is {:}".format(KPI, KPI_VALUE)) + print(f"Extracted value of {KPI} is: {KPI_VALUE}") + return KPI_VALUE + else: + LOGGER.info("Failed to fetch metrics. Status code: {:}".format(response.status_code)) + # print(f"Failed to fetch metrics. Status code: {response.status_code}") + return None + except Exception as e: + LOGGER.info("Failed to fetch metrics. Status code: {:}".format(e)) + # print(f"Failed to fetch metrics: {str(e)}") + return None + + @staticmethod + def extract_metric_value(metrics, metric_name): + """ + Method to extract the value of a metric from the metrics string. + Args: + metrics (str): Metrics string fetched from Exporter. + metric_name (str): Name of the metric to extract. + Returns: + float: Value of the extracted metric, or None if not found. + """ + try: + # Find the metric line containing the desired metric name + metric_line = next(line for line in metrics.split('\n') if line.startswith(metric_name)) + # Split the line to extract the metric value + metric_value = float(metric_line.split()[1]) + return metric_value + except StopIteration: + print(f"Metric '{metric_name}' not found in the metrics.") + return None + + @staticmethod + def stream_node_export_metrics_to_raw_topic(): + try: + while True: + response = requests.get(EXPORTER_ENDPOINT) + # print("Response Status {:} ".format(response)) + # LOGGER.info("Response Status {:} ".format(response)) + try: + if response.status_code == 200: + producerObj = KafkaProducer(PRODUCER_CONFIG) + producerObj.produce(KAFKA_TOPICS['raw'], key="raw", value= str(response.text), callback=TelemetryBackendService.delivery_callback) + producerObj.flush() + LOGGER.info("Produce to topic") + else: + LOGGER.info("Didn't received expected response. Status code: {:}".format(response.status_code)) + print(f"Didn't received expected response. Status code: {response.status_code}") + return None + time.sleep(15) + except Exception as e: + LOGGER.info("Failed to process response. Status code: {:}".format(e)) + return None + except Exception as e: + LOGGER.info("Failed to fetch metrics. Status code: {:}".format(e)) + print(f"Failed to fetch metrics: {str(e)}") + return None +# ----------- ABOVE: Actual Implementation of Kafka Producer with Node Exporter ----------- \ No newline at end of file diff --git a/src/telemetry/backend/service/__init__.py b/src/telemetry/backend/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bbfc943b68af13a11e562abbc8680ade71db8f02 --- /dev/null +++ b/src/telemetry/backend/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/telemetry/backend/tests/__init__.py b/src/telemetry/backend/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bbfc943b68af13a11e562abbc8680ade71db8f02 --- /dev/null +++ b/src/telemetry/backend/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/telemetry/backend/tests/messagesBackend.py b/src/telemetry/backend/tests/messagesBackend.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf553eaaec41de7599b6723e31e4ca3f82cbcae --- /dev/null +++ b/src/telemetry/backend/tests/messagesBackend.py @@ -0,0 +1,15 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/telemetry/backend/tests/testTelemetryBackend.py b/src/telemetry/backend/tests/testTelemetryBackend.py new file mode 100644 index 0000000000000000000000000000000000000000..d832e54e77589ca677682760d19e68b1bd09b1f7 --- /dev/null +++ b/src/telemetry/backend/tests/testTelemetryBackend.py @@ -0,0 +1,53 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +print (sys.path) +sys.path.append('/home/tfs/tfs-ctrl') +import threading +import logging +from typing import Tuple +# from common.proto.context_pb2 import Empty +from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService + +LOGGER = logging.getLogger(__name__) + + +########################### +# Tests Implementation of Telemetry Backend +########################### + +def test_verify_kafka_topics(): + LOGGER.info('test_verify_kafka_topics requesting') + TelemetryBackendServiceObj = TelemetryBackendService() + KafkaTopics = ['topic_request', 'topic_response', 'topic_raw', 'topic_labled'] + response = TelemetryBackendServiceObj.create_topic_if_not_exists(KafkaTopics) + LOGGER.debug(str(response)) + assert isinstance(response, bool) + +# def test_run_kafka_listener(): +# LOGGER.info('test_receive_kafka_request requesting') +# TelemetryBackendServiceObj = TelemetryBackendService() +# response = TelemetryBackendServiceObj.run_kafka_listener() +# LOGGER.debug(str(response)) +# assert isinstance(response, bool) + +# def test_fetch_node_exporter_metrics(): +# LOGGER.info(' >>> test_fetch_node_exporter_metrics START <<< ') +# TelemetryBackendService.fetch_single_node_exporter_metric() + +def test_stream_node_export_metrics_to_raw_topic(): + LOGGER.info(' >>> test_stream_node_export_metrics_to_raw_topic START <<< ') + threading.Thread(target=TelemetryBackendService.stream_node_export_metrics_to_raw_topic, args=()).start() + diff --git a/src/telemetry/database/TelemetryDBmanager.py b/src/telemetry/database/TelemetryDBmanager.py new file mode 100644 index 0000000000000000000000000000000000000000..b558180a9e1fbf85bf523c7faededf58f57e2264 --- /dev/null +++ b/src/telemetry/database/TelemetryDBmanager.py @@ -0,0 +1,248 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, time +import sqlalchemy +from sqlalchemy import inspect, MetaData, Table +from sqlalchemy.orm import sessionmaker +from telemetry.database.TelemetryModel import Collector as CollectorModel +from telemetry.database.TelemetryModel import Kpi as KpiModel +from sqlalchemy.ext.declarative import declarative_base +from telemetry.database.TelemetryEngine import TelemetryEngine +from common.proto.kpi_manager_pb2 import KpiDescriptor, KpiId +from common.proto.telemetry_frontend_pb2 import Collector, CollectorId +from sqlalchemy.exc import SQLAlchemyError +from telemetry.database.TelemetryModel import Base + +LOGGER = logging.getLogger(__name__) +DB_NAME = "telemetryfrontend" + +class TelemetryDBmanager: + def __init__(self): + self.db_engine = TelemetryEngine.get_engine() + if self.db_engine is None: + LOGGER.error('Unable to get SQLAlchemy DB Engine...') + return False + self.db_name = DB_NAME + self.Session = sessionmaker(bind=self.db_engine) + + def create_database(self): + try: + # with self.db_engine.connect() as connection: + # connection.execute(f"CREATE DATABASE {self.db_name};") + TelemetryEngine.create_database(self.db_engine) + LOGGER.info('TelemetryDBmanager initalized DB Name: {:}'.format(self.db_name)) + return True + except Exception as e: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to check/create the database: {:s}'.format(str(e))) + return False + + def create_tables(self): + try: + Base.metadata.create_all(self.db_engine) # type: ignore + LOGGER.info("Tables created in database ({:}) the as per Models".format(self.db_name)) + except Exception as e: + LOGGER.info("Tables cannot be created in the TelemetryFrontend database. {:s}".format(str(e))) + + def verify_tables(self): + try: + with self.db_engine.connect() as connection: + result = connection.execute("SHOW TABLES;") + tables = result.fetchall() + LOGGER.info("Tables in DB: {:}".format(tables)) + except Exception as e: + LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) + + def drop_table(self, table_to_drop: str): + try: + inspector = inspect(self.db_engine) + existing_tables = inspector.get_table_names() + if table_to_drop in existing_tables: + table = Table(table_to_drop, MetaData(), autoload_with=self.db_engine) + table.drop(self.db_engine) + LOGGER.info("Tables delete in the DB Name: {:}".format(self.db_name)) + else: + LOGGER.warning("No table {:} in database {:} ".format(table_to_drop, DB_NAME)) + except Exception as e: + LOGGER.info("Tables cannot be deleted in the {:} database. {:s}".format(DB_NAME, str(e))) + + def list_databases(self): + query = "SHOW DATABASES" + with self.db_engine.connect() as connection: + result = connection.execute(query) + databases = [row[0] for row in result] + LOGGER.info("List of available DBs: {:}".format(databases)) + +# ------------------ INSERT METHODs -------------------------------------- + + def inser_kpi(self, request: KpiDescriptor): + session = self.Session() + try: + # Create a new Kpi instance + kpi_to_insert = KpiModel() + kpi_to_insert.kpi_id = request.kpi_id.kpi_id.uuid + kpi_to_insert.kpi_description = request.kpi_description + kpi_to_insert.kpi_sample_type = request.kpi_sample_type + kpi_to_insert.device_id = request.service_id.service_uuid.uuid + kpi_to_insert.endpoint_id = request.device_id.device_uuid.uuid + kpi_to_insert.service_id = request.slice_id.slice_uuid.uuid + kpi_to_insert.slice_id = request.endpoint_id.endpoint_uuid.uuid + kpi_to_insert.connection_id = request.connection_id.connection_uuid.uuid + # kpi_to_insert.link_id = request.link_id.link_id.uuid + # Add the instance to the session + session.add(kpi_to_insert) + session.commit() + LOGGER.info("Row inserted into kpi table: {:}".format(kpi_to_insert.kpi_id)) + except Exception as e: + session.rollback() + LOGGER.info("Failed to insert new kpi. {:s}".format(str(e))) + finally: + # Close the session + session.close() + + # Function to insert a row into the Collector model + def insert_collector(self, request: Collector): + session = self.Session() + try: + # Create a new Collector instance + collector_to_insert = CollectorModel() + collector_to_insert.collector_id = request.collector_id.collector_id.uuid + collector_to_insert.kpi_id = request.kpi_id.kpi_id.uuid + collector_to_insert.collector = "Test collector description" + collector_to_insert.sampling_duration_s = request.duration_s + collector_to_insert.sampling_interval_s = request.interval_s + collector_to_insert.start_timestamp = time.time() + collector_to_insert.end_timestamp = time.time() + + session.add(collector_to_insert) + session.commit() + LOGGER.info("Row inserted into collector table: {:}".format(collector_to_insert.collector_id)) + except Exception as e: + session.rollback() + LOGGER.info("Failed to insert new collector. {:s}".format(str(e))) + finally: + # Close the session + session.close() + +# ------------------ GET METHODs -------------------------------------- + + def get_kpi_descriptor(self, request: KpiId): + session = self.Session() + try: + kpi_id_to_search = request.kpi_id.uuid + kpi = session.query(KpiModel).filter_by(kpi_id=kpi_id_to_search).first() + if kpi: + LOGGER.info("kpi ID found: {:s}".format(str(kpi))) + return kpi + else: + LOGGER.warning("Kpi ID not found {:s}".format(str(kpi_id_to_search))) + return None + except Exception as e: + session.rollback() + LOGGER.info("Failed to retrieve KPI ID. {:s}".format(str(e))) + raise + finally: + session.close() + + def get_collector(self, request: CollectorId): + session = self.Session() + try: + collector_id_to_search = request.collector_id.uuid + collector = session.query(CollectorModel).filter_by(collector_id=collector_id_to_search).first() + if collector: + LOGGER.info("collector ID found: {:s}".format(str(collector))) + return collector + else: + LOGGER.warning("collector ID not found{:s}".format(str(collector_id_to_search))) + return None + except Exception as e: + session.rollback() + LOGGER.info("Failed to retrieve collector ID. {:s}".format(str(e))) + raise + finally: + session.close() + + # ------------------ SELECT METHODs -------------------------------------- + + def select_kpi_descriptor(self, **filters): + session = self.Session() + try: + query = session.query(KpiModel) + for column, value in filters.items(): + query = query.filter(getattr(KpiModel, column) == value) + result = query.all() + if len(result) != 0: + LOGGER.info("Fetched filtered rows from KPI table with filters : {:s}".format(str(result))) + else: + LOGGER.warning("No matching row found : {:s}".format(str(result))) + return result + except SQLAlchemyError as e: + LOGGER.error("Error fetching filtered rows from KPI table with filters {:}: {:}".format(filters, e)) + return [] + finally: + session.close() + + def select_collector(self, **filters): + session = self.Session() + try: + query = session.query(CollectorModel) + for column, value in filters.items(): + query = query.filter(getattr(CollectorModel, column) == value) + result = query.all() + if len(result) != 0: + LOGGER.info("Fetched filtered rows from KPI table with filters : {:s}".format(str(result))) + else: + LOGGER.warning("No matching row found : {:s}".format(str(result))) + return result + except SQLAlchemyError as e: + LOGGER.error("Error fetching filtered rows from KPI table with filters {:}: {:}".format(filters, e)) + return [] + finally: + session.close() + +# ------------------ DELETE METHODs -------------------------------------- + + def delete_kpi_descriptor(self, request: KpiId): + session = self.Session() + try: + kpi_id_to_delete = request.kpi_id.uuid + kpi = session.query(KpiModel).filter_by(kpi_id=kpi_id_to_delete).first() + if kpi: + session.delete(kpi) + session.commit() + LOGGER.info("Deleted KPI with kpi_id: %s", kpi_id_to_delete) + else: + LOGGER.warning("KPI with kpi_id %s not found", kpi_id_to_delete) + except SQLAlchemyError as e: + session.rollback() + LOGGER.error("Error deleting KPI with kpi_id %s: %s", kpi_id_to_delete, e) + finally: + session.close() + + def delete_collector(self, request: CollectorId): + session = self.Session() + try: + collector_id_to_delete = request.collector_id.uuid + collector = session.query(CollectorModel).filter_by(collector_id=collector_id_to_delete).first() + if collector: + session.delete(collector) + session.commit() + LOGGER.info("Deleted collector with collector_id: %s", collector_id_to_delete) + else: + LOGGER.warning("collector with collector_id %s not found", collector_id_to_delete) + except SQLAlchemyError as e: + session.rollback() + LOGGER.error("Error deleting collector with collector_id %s: %s", collector_id_to_delete, e) + finally: + session.close() \ No newline at end of file diff --git a/src/telemetry/database/TelemetryEngine.py b/src/telemetry/database/TelemetryEngine.py new file mode 100644 index 0000000000000000000000000000000000000000..a563fa09f94c812aed07d0aa3cbd5bc988737fc4 --- /dev/null +++ b/src/telemetry/database/TelemetryEngine.py @@ -0,0 +1,59 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, sqlalchemy, sqlalchemy_utils +# from common.Settings import get_setting + +LOGGER = logging.getLogger(__name__) + +APP_NAME = 'tfs' +ECHO = False # False: No dump SQL commands and transactions executed +CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' +# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' + +class TelemetryEngine: + # def __init__(self): + # self.engine = self.get_engine() + @staticmethod + def get_engine() -> sqlalchemy.engine.Engine: + CRDB_NAMESPACE = "crdb" + CRDB_SQL_PORT = "26257" + CRDB_DATABASE = "telemetryfrontend" + CRDB_USERNAME = "tfs" + CRDB_PASSWORD = "tfs123" + CRDB_SSLMODE = "require" + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + # crdb_uri = CRDB_URI_TEMPLATE.format( + # CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + try: + # engine = sqlalchemy.create_engine( + # crdb_uri, connect_args={'application_name': APP_NAME}, echo=ECHO, future=True) + engine = sqlalchemy.create_engine(crdb_uri, echo=False) + LOGGER.info(' TelemetryDBmanager initalized with DB URL: {:}'.format(crdb_uri)) + except: # pylint: disable=bare-except # pragma: no cover + LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) + return None # type: ignore + return engine # type: ignore + + @staticmethod + def create_database(engine : sqlalchemy.engine.Engine) -> None: + if not sqlalchemy_utils.database_exists(engine.url): + LOGGER.info("Database created. {:}".format(engine.url)) + sqlalchemy_utils.create_database(engine.url) + + @staticmethod + def drop_database(engine : sqlalchemy.engine.Engine) -> None: + if sqlalchemy_utils.database_exists(engine.url): + sqlalchemy_utils.drop_database(engine.url) diff --git a/src/telemetry/database/TelemetryModel.py b/src/telemetry/database/TelemetryModel.py new file mode 100644 index 0000000000000000000000000000000000000000..be4f0969c86638520cf226b8e42db90426165804 --- /dev/null +++ b/src/telemetry/database/TelemetryModel.py @@ -0,0 +1,45 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import Column, Integer, String, Float, Text, ForeignKey +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from sqlalchemy.orm import registry + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + +# Create a base class for declarative models +Base = registry().generate_base() +# Base = declarative_base() + +class Collector(Base): + __tablename__ = 'collector' + + collector_id = Column(UUID(as_uuid=False), primary_key=True) + kpi_id = Column(UUID(as_uuid=False)) + collector_decription = Column(String) + sampling_duration_s = Column(Float) + sampling_interval_s = Column(Float) + start_timestamp = Column(Float) + end_timestamp = Column(Float) + + + def __repr__(self): + return (f"") \ No newline at end of file diff --git a/src/telemetry/database/__init__.py b/src/telemetry/database/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/telemetry/database/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/database/__main__.py b/src/telemetry/database/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf553eaaec41de7599b6723e31e4ca3f82cbcae --- /dev/null +++ b/src/telemetry/database/__main__.py @@ -0,0 +1,15 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/telemetry/database/managementDB.py b/src/telemetry/database/managementDB.py new file mode 100644 index 0000000000000000000000000000000000000000..f79126f279d7bbece6c08ae5eb1cd74e340d1c7d --- /dev/null +++ b/src/telemetry/database/managementDB.py @@ -0,0 +1,138 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, time +import sqlalchemy +import sqlalchemy_utils +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from telemetry.database.TelemetryEngine import TelemetryEngine +from telemetry.database.TelemetryModel import Base + +LOGGER = logging.getLogger(__name__) +DB_NAME = "telemetryfrontend" + +# # Create a base class for declarative models +# Base = declarative_base() + +class managementDB: + def __init__(self): + self.db_engine = TelemetryEngine.get_engine() + if self.db_engine is None: + LOGGER.error('Unable to get SQLAlchemy DB Engine...') + return False + self.db_name = DB_NAME + self.Session = sessionmaker(bind=self.db_engine) + + @staticmethod + def create_database(engine : sqlalchemy.engine.Engine) -> None: + if not sqlalchemy_utils.database_exists(engine.url): + LOGGER.info("Database created. {:}".format(engine.url)) + sqlalchemy_utils.create_database(engine.url) + + @staticmethod + def drop_database(engine : sqlalchemy.engine.Engine) -> None: + if sqlalchemy_utils.database_exists(engine.url): + sqlalchemy_utils.drop_database(engine.url) + + # def create_database(self): + # try: + # with self.db_engine.connect() as connection: + # connection.execute(f"CREATE DATABASE {self.db_name};") + # LOGGER.info('managementDB initalizes database. Name: {self.db_name}') + # return True + # except: + # LOGGER.exception('Failed to check/create the database: {:s}'.format(str(self.db_engine.url))) + # return False + + @staticmethod + def create_tables(engine : sqlalchemy.engine.Engine): + try: + Base.metadata.create_all(engine) # type: ignore + LOGGER.info("Tables created in the DB Name: {:}".format(DB_NAME)) + except Exception as e: + LOGGER.info("Tables cannot be created in the TelemetryFrontend database. {:s}".format(str(e))) + + def verify_tables(self): + try: + with self.db_engine.connect() as connection: + result = connection.execute("SHOW TABLES;") + tables = result.fetchall() # type: ignore + LOGGER.info("Tables verified: {:}".format(tables)) + except Exception as e: + LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) + + @staticmethod + def add_row_to_db(self, row): + session = self.Session() + try: + session.add(row) + session.commit() + LOGGER.info(f"Row inserted into {row.__class__.__name__} table.") + except Exception as e: + session.rollback() + LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}") + finally: + session.close() + + def search_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + entity = session.query(model).filter_by(**{col_name: id_to_search}).first() + if entity: + LOGGER.info(f"{model.__name__} ID found: {str(entity)}") + return entity + else: + LOGGER.warning(f"{model.__name__} ID not found: {str(id_to_search)}") + return None + except Exception as e: + session.rollback() + LOGGER.info(f"Failed to retrieve {model.__name__} ID. {str(e)}") + raise + finally: + session.close() + + def delete_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + record = session.query(model).filter_by(**{col_name: id_to_search}).first() + if record: + session.delete(record) + session.commit() + LOGGER.info("Deleted %s with %s: %s", model.__name__, col_name, id_to_search) + else: + LOGGER.warning("%s with %s %s not found", model.__name__, col_name, id_to_search) + except Exception as e: + session.rollback() + LOGGER.error("Error deleting %s with %s %s: %s", model.__name__, col_name, id_to_search, e) + finally: + session.close() + + def select_with_filter(self, model, **filters): + session = self.Session() + try: + query = session.query(model) + for column, value in filters.items(): + query = query.filter(getattr(model, column) == value) # type: ignore + result = query.all() + if result: + LOGGER.info(f"Fetched filtered rows from {model.__name__} table with filters: {filters}") # - Results: {result} + else: + LOGGER.warning(f"No matching row found in {model.__name__} table with filters: {filters}") + return result + except Exception as e: + LOGGER.error(f"Error fetching filtered rows from {model.__name__} table with filters {filters} ::: {e}") + return [] + finally: + session.close() \ No newline at end of file diff --git a/src/telemetry/database/tests/__init__.py b/src/telemetry/database/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..839e45e3b646bc60de7edd81fcfb91b7b38feadf --- /dev/null +++ b/src/telemetry/database/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/src/telemetry/database/tests/managementDBtests.py b/src/telemetry/database/tests/managementDBtests.py new file mode 100644 index 0000000000000000000000000000000000000000..24138abe42be742bd9b16d7840343f9d7c7fe133 --- /dev/null +++ b/src/telemetry/database/tests/managementDBtests.py @@ -0,0 +1,22 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from telemetry.database.managementDB import managementDB +from telemetry.database.tests.messages import create_collector_model_object + + +def test_add_row_to_db(): + managementDBobj = managementDB() + managementDBobj.add_row_to_db(create_collector_model_object()) \ No newline at end of file diff --git a/src/telemetry/database/tests/messages.py b/src/telemetry/database/tests/messages.py new file mode 100644 index 0000000000000000000000000000000000000000..6919eecc62da0794869f334c4de85cb129fbab14 --- /dev/null +++ b/src/telemetry/database/tests/messages.py @@ -0,0 +1,80 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import uuid +import random +from common.proto import telemetry_frontend_pb2 +from common.proto import kpi_manager_pb2 +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from telemetry.database.TelemetryModel import Collector as CollectorModel + + +def create_collector_request(): + _create_collector_request = telemetry_frontend_pb2.Collector() + _create_collector_request.collector_id.collector_id.uuid = str(uuid.uuid4()) + _create_collector_request.kpi_id.kpi_id.uuid = '71d58648-bf47-49ac-996f-e63a9fbfead4' # must be primary key in kpi table + # _create_collector_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_collector_request.duration_s = float(random.randint(8, 16)) + _create_collector_request.interval_s = float(random.randint(2, 4)) + return _create_collector_request + +def create_kpi_request(): + _create_kpi_request = kpi_manager_pb2.KpiDescriptor() + _create_kpi_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_kpi_request.kpi_description = 'KPI Description Test' + _create_kpi_request.kpi_sample_type = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED + _create_kpi_request.service_id.service_uuid.uuid = 'SERV' + _create_kpi_request.device_id.device_uuid.uuid = 'DEV' + _create_kpi_request.slice_id.slice_uuid.uuid = 'SLC' + _create_kpi_request.endpoint_id.endpoint_uuid.uuid = 'END' + _create_kpi_request.connection_id.connection_uuid.uuid = 'CON' + # _create_kpi_request.link_id.link_id.uuid = 'LNK' + return _create_kpi_request + +def create_kpi_id_request(): + _create_kpi_id_request = kpi_manager_pb2.KpiId() + _create_kpi_id_request.kpi_id.uuid = '71d58648-bf47-49ac-996f-e63a9fbfead4' + return _create_kpi_id_request + +def create_collector_id_request(): + _create_collector_id_request = telemetry_frontend_pb2.CollectorId() + _create_collector_id_request.collector_id.uuid = '71d58648-bf47-49ac-996f-e63a9fbfead4' + return _create_collector_id_request + +def create_kpi_filter_request(): + # create a dict as follows: 'Key' = 'KpiModel' column name and 'Value' = filter to apply. + _create_kpi_filter_request = dict() + _create_kpi_filter_request['kpi_sample_type'] = 102 + _create_kpi_filter_request['kpi_id'] = '3a17230d-8e95-4afb-8b21-6965481aee5a' + return _create_kpi_filter_request + +def create_collector_filter_request(): + # create a dict as follows: 'Key' = 'KpiModel' column name and 'Value' = filter to apply. + _create_kpi_filter_request = dict() + _create_kpi_filter_request['sampling_interval_s'] = 3.0 + # _create_kpi_filter_request['kpi_id'] = '11e2c6c6-b507-40aa-ab3a-ffd41e7125f0' + return _create_kpi_filter_request + +def create_collector_model_object(): + # Create a new Collector instance + collector_to_insert = CollectorModel() + collector_to_insert.collector_id = str(uuid.uuid4()) + collector_to_insert.kpi_id = '3a17230d-8e95-4afb-8b21-6965481aee5a' + collector_to_insert.collector = "Test collector description" + collector_to_insert.sampling_duration_s = 15 + collector_to_insert.sampling_interval_s = 3 + collector_to_insert.start_timestamp = time.time() + collector_to_insert.end_timestamp = time.time() + return collector_to_insert \ No newline at end of file diff --git a/src/telemetry/database/tests/telemetryDBtests.py b/src/telemetry/database/tests/telemetryDBtests.py new file mode 100644 index 0000000000000000000000000000000000000000..0d221106419d6e4ee4b313adf10c90c5e6be7666 --- /dev/null +++ b/src/telemetry/database/tests/telemetryDBtests.py @@ -0,0 +1,86 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Any +from sqlalchemy.ext.declarative import declarative_base +from telemetry.database.TelemetryDBmanager import TelemetryDBmanager +from telemetry.database.TelemetryEngine import TelemetryEngine +from telemetry.database.tests import temp_DB +from .messages import create_kpi_request, create_collector_request, \ + create_kpi_id_request, create_kpi_filter_request, \ + create_collector_id_request, create_collector_filter_request + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + + +# def test_temp_DB(): +# temp_DB.main() + +def test_telemetry_object_creation(): + LOGGER.info('--- test_telemetry_object_creation: START') + + LOGGER.info('>>> Creating TelemetryDBmanager Object <<< ') + TelemetryDBmanagerObj = TelemetryDBmanager() + TelemetryEngine.create_database(TelemetryDBmanagerObj.db_engine) # creates 'frontend' db, if it doesnot exists. + + LOGGER.info('>>> Creating database <<< ') + TelemetryDBmanagerObj.create_database() + + LOGGER.info('>>> verifing database <<< ') + TelemetryDBmanagerObj.list_databases() + + # # LOGGER.info('>>> Droping Tables: ') + # # TelemetryDBmanagerObj.drop_table("table_naem_here") + + LOGGER.info('>>> Creating Tables <<< ') + TelemetryDBmanagerObj.create_tables() + + LOGGER.info('>>> Verifing Table creation <<< ') + TelemetryDBmanagerObj.verify_tables() + + # LOGGER.info('>>> TESTING: Row Insertion Operation: kpi Table <<<') + # kpi_obj = create_kpi_request() + # TelemetryDBmanagerObj.inser_kpi(kpi_obj) + + # LOGGER.info('>>> TESTING: Row Insertion Operation: collector Table <<<') + # collector_obj = create_collector_request() + # TelemetryDBmanagerObj.insert_collector(collector_obj) + + # LOGGER.info('>>> TESTING: Get KpiDescriptor <<<') + # kpi_id_obj = create_kpi_id_request() + # TelemetryDBmanagerObj.get_kpi_descriptor(kpi_id_obj) + + # LOGGER.info('>>> TESTING: Select Collector <<<') + # collector_id_obj = create_collector_id_request() + # TelemetryDBmanagerObj.get_collector(collector_id_obj) + + # LOGGER.info('>>> TESTING: Applying kpi filter <<< ') + # kpi_filter : dict[str, Any] = create_kpi_filter_request() + # TelemetryDBmanagerObj.select_kpi_descriptor(**kpi_filter) + + # LOGGER.info('>>> TESTING: Applying collector filter <<<') + # collector_filter : dict[str, Any] = create_collector_filter_request() + # TelemetryDBmanagerObj.select_collector(**collector_filter) + + # LOGGER.info('>>> TESTING: Delete KpiDescriptor ') + # kpi_id_obj = create_kpi_id_request() + # TelemetryDBmanagerObj.delete_kpi_descriptor(kpi_id_obj) + + # LOGGER.info('>>> TESTING: Delete Collector ') + # collector_id_obj = create_collector_id_request() + # TelemetryDBmanagerObj.delete_collector(collector_id_obj) + \ No newline at end of file diff --git a/src/telemetry/database/tests/temp_DB.py b/src/telemetry/database/tests/temp_DB.py new file mode 100644 index 0000000000000000000000000000000000000000..089d3542492c2da87b839416f7118749bb82caad --- /dev/null +++ b/src/telemetry/database/tests/temp_DB.py @@ -0,0 +1,327 @@ +from sqlalchemy import create_engine, Column, String, Integer, Text, Float, ForeignKey +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from sqlalchemy.dialects.postgresql import UUID +import logging + +LOGGER = logging.getLogger(__name__) +Base = declarative_base() + +class Kpi(Base): + __tablename__ = 'kpi' + + kpi_id = Column(UUID(as_uuid=False), primary_key=True) + kpi_description = Column(Text) + kpi_sample_type = Column(Integer) + device_id = Column(String) + endpoint_id = Column(String) + service_id = Column(String) + slice_id = Column(String) + connection_id = Column(String) + link_id = Column(String) + + collectors = relationship('Collector', back_populates='kpi') + + def __repr__(self): + return (f"") + +class Collector(Base): + __tablename__ = 'collector' + + collector_id = Column(UUID(as_uuid=False), primary_key=True) + kpi_id = Column(UUID(as_uuid=False), ForeignKey('kpi.kpi_id')) + collector = Column(String) + sampling_duration_s = Column(Float) + sampling_interval_s = Column(Float) + start_timestamp = Column(Float) + end_timestamp = Column(Float) + + kpi = relationship('Kpi', back_populates='collectors') + + def __repr__(self): + return (f"") + +class DatabaseManager: + def __init__(self, db_url, db_name): + self.engine = create_engine(db_url) + self.db_name = db_name + self.Session = sessionmaker(bind=self.engine) + LOGGER.info("DatabaseManager initialized with DB URL: %s and DB Name: %s", db_url, db_name) + + def create_database(self): + try: + with self.engine.connect() as connection: + connection.execute(f"CREATE DATABASE {self.db_name};") + LOGGER.info("Database '%s' created successfully.", self.db_name) + except Exception as e: + LOGGER.error("Error creating database '%s': %s", self.db_name, e) + finally: + LOGGER.info("create_database method execution finished.") + + def create_tables(self): + try: + Base.metadata.create_all(self.engine) + LOGGER.info("Tables created successfully.") + except Exception as e: + LOGGER.error("Error creating tables: %s", e) + finally: + LOGGER.info("create_tables method execution finished.") + + def verify_table_creation(self): + try: + with self.engine.connect() as connection: + result = connection.execute("SHOW TABLES;") + tables = result.fetchall() + LOGGER.info("Tables verified: %s", tables) + return tables + except Exception as e: + LOGGER.error("Error verifying table creation: %s", e) + return [] + finally: + LOGGER.info("verify_table_creation method execution finished.") + + def insert_row_kpi(self, kpi_data): + session = self.Session() + try: + new_kpi = Kpi(**kpi_data) + session.add(new_kpi) + session.commit() + LOGGER.info("Inserted row into KPI table: %s", kpi_data) + except Exception as e: + session.rollback() + LOGGER.error("Error inserting row into KPI table: %s", e) + finally: + session.close() + LOGGER.info("insert_row_kpi method execution finished.") + + def insert_row_collector(self, collector_data): + session = self.Session() + try: + new_collector = Collector(**collector_data) + session.add(new_collector) + session.commit() + LOGGER.info("Inserted row into Collector table: %s", collector_data) + except Exception as e: + session.rollback() + LOGGER.error("Error inserting row into Collector table: %s", e) + finally: + session.close() + LOGGER.info("insert_row_collector method execution finished.") + + def verify_insertion_kpi(self, kpi_id): + session = self.Session() + try: + kpi = session.query(Kpi).filter_by(kpi_id=kpi_id).first() + LOGGER.info("Verified insertion in KPI table for kpi_id: %s, Result: %s", kpi_id, kpi) + return kpi + except Exception as e: + LOGGER.error("Error verifying insertion in KPI table for kpi_id %s: %s", kpi_id, e) + return None + finally: + session.close() + LOGGER.info("verify_insertion_kpi method execution finished.") + + def verify_insertion_collector(self, collector_id): + session = self.Session() + try: + collector = session.query(Collector).filter_by(collector_id=collector_id).first() + LOGGER.info("Verified insertion in Collector table for collector_id: %s, Result: %s", collector_id, collector) + return collector + except Exception as e: + LOGGER.error("Error verifying insertion in Collector table for collector_id %s: %s", collector_id, e) + return None + finally: + session.close() + LOGGER.info("verify_insertion_collector method execution finished.") + + def get_all_kpi_rows(self): + session = self.Session() + try: + kpi_rows = session.query(Kpi).all() + LOGGER.info("Fetched all rows from KPI table: %s", kpi_rows) + return kpi_rows + except Exception as e: + LOGGER.error("Error fetching all rows from KPI table: %s", e) + return [] + finally: + session.close() + LOGGER.info("get_all_kpi_rows method execution finished.") + + def get_all_collector_rows(self): + session = self.Session() + try: + collector_rows = session.query(Collector).all() + LOGGER.info("Fetched all rows from Collector table: %s", collector_rows) + return collector_rows + except Exception as e: + LOGGER.error("Error fetching all rows from Collector table: %s", e) + return [] + finally: + session.close() + LOGGER.info("get_all_collector_rows method execution finished.") + + def get_filtered_kpi_rows(self, **filters): + session = self.Session() + try: + query = session.query(Kpi) + for column, value in filters.items(): + query = query.filter(getattr(Kpi, column) == value) + result = query.all() + LOGGER.info("Fetched filtered rows from KPI table with filters ---------- : {:s}".format(str(result))) + return result + except NoResultFound: + LOGGER.warning("No results found in KPI table with filters %s", filters) + return [] + except Exception as e: + LOGGER.error("Error fetching filtered rows from KPI table with filters %s: %s", filters, e) + return [] + finally: + session.close() + LOGGER.info("get_filtered_kpi_rows method execution finished.") + + def get_filtered_collector_rows(self, **filters): + session = self.Session() + try: + query = session.query(Collector) + for column, value in filters.items(): + query = query.filter(getattr(Collector, column) == value) + result = query.all() + LOGGER.info("Fetched filtered rows from Collector table with filters %s: %s", filters, result) + return result + except NoResultFound: + LOGGER.warning("No results found in Collector table with filters %s", filters) + return [] + except Exception as e: + LOGGER.error("Error fetching filtered rows from Collector table with filters %s: %s", filters, e) + return [] + finally: + session.close() + LOGGER.info("get_filtered_collector_rows method execution finished.") + + def delete_kpi_by_id(self, kpi_id): + session = self.Session() + try: + kpi = session.query(Kpi).filter_by(kpi_id=kpi_id).first() + if kpi: + session.delete(kpi) + session.commit() + LOGGER.info("Deleted KPI with kpi_id: %s", kpi_id) + else: + LOGGER.warning("KPI with kpi_id %s not found", kpi_id) + except SQLAlchemyError as e: + session.rollback() + LOGGER.error("Error deleting KPI with kpi_id %s: %s", kpi_id, e) + finally: + session.close() + LOGGER.info("delete_kpi_by_id method execution finished.") + + def delete_collector_by_id(self, collector_id): + session = self.Session() + try: + collector = session.query(Collector).filter_by(collector_id=collector_id).first() + if collector: + session.delete(collector) + session.commit() + LOGGER.info("Deleted Collector with collector_id: %s", collector_id) + else: + LOGGER.warning("Collector with collector_id %s not found", collector_id) + except SQLAlchemyError as e: + session.rollback() + LOGGER.error("Error deleting Collector with collector_id %s: %s", collector_id, e) + finally: + session.close() + LOGGER.info("delete_collector_by_id method execution finished.") + + +# Example Usage +def main(): + CRDB_SQL_PORT = "26257" + CRDB_DATABASE = "telemetryfrontend" + CRDB_USERNAME = "tfs" + CRDB_PASSWORD = "tfs123" + CRDB_SSLMODE = "require" + CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + # db_url = "cockroachdb://username:password@localhost:26257/" + # db_name = "yourdatabase" + db_manager = DatabaseManager(crdb_uri, CRDB_DATABASE) + + # Create database + db_manager.create_database() + + # Update db_url to include the new database name + db_manager.engine = create_engine(f"{crdb_uri}") + db_manager.Session = sessionmaker(bind=db_manager.engine) + + # Create tables + db_manager.create_tables() + + # Verify table creation + tables = db_manager.verify_table_creation() + LOGGER.info('Tables in the database: {:s}'.format(str(tables))) + + # Insert a row into the KPI table + kpi_data = { + 'kpi_id': '123e4567-e89b-12d3-a456-426614174100', + 'kpi_description': 'Sample KPI', + 'kpi_sample_type': 1, + 'device_id': 'device_1', + 'endpoint_id': 'endpoint_1', + 'service_id': 'service_1', + 'slice_id': 'slice_1', + 'connection_id': 'conn_1', + 'link_id': 'link_1' + } + db_manager.insert_row_kpi(kpi_data) + + # Insert a row into the Collector table + collector_data = { + 'collector_id': '123e4567-e89b-12d3-a456-426614174101', + 'kpi_id': '123e4567-e89b-12d3-a456-426614174000', + 'collector': 'Collector 1', + 'sampling_duration_s': 60.0, + 'sampling_interval_s': 10.0, + 'start_timestamp': 1625247600.0, + 'end_timestamp': 1625247660.0 + } + db_manager.insert_row_collector(collector_data) + + # Verify insertion into KPI table + kpi = db_manager.verify_insertion_kpi('123e4567-e89b-12d3-a456-426614174000') + print("Inserted KPI:", kpi) + + # Verify insertion into Collector table + collector = db_manager.verify_insertion_collector('123e4567-e89b-12d3-a456-426614174001') + print("Inserted Collector:", collector) + + # Get all rows from KPI table + all_kpi_rows = db_manager.get_all_kpi_rows() + LOGGER.info("All KPI Rows: %s", all_kpi_rows) + + # Get all rows from Collector table + all_collector_rows = db_manager.get_all_collector_rows() + LOGGER.info("All Collector Rows: %s", all_collector_rows) + + # Get filtered rows from KPI table + filtered_kpi_rows = db_manager.get_filtered_kpi_rows(kpi_description='Sample KPI') + LOGGER.info("Filtered KPI Rows: %s", filtered_kpi_rows) + + # Get filtered rows from Collector table + filtered_collector_rows = db_manager.get_filtered_collector_rows(collector='Collector 1') + LOGGER.info("Filtered Collector Rows: %s", filtered_collector_rows) + + # Delete a KPI by kpi_id + kpi_id_to_delete = '123e4567-e89b-12d3-a456-426614174000' + db_manager.delete_kpi_by_id(kpi_id_to_delete) + + # Delete a Collector by collector_id + collector_id_to_delete = '123e4567-e89b-12d3-a456-426614174001' + db_manager.delete_collector_by_id(collector_id_to_delete) diff --git a/src/telemetry/frontend/__init__.py b/src/telemetry/frontend/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..234a1af6588c91f6a17f3963f69120cd6e2248d9 --- /dev/null +++ b/src/telemetry/frontend/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/frontend/client/TelemetryFrontendClient.py b/src/telemetry/frontend/client/TelemetryFrontendClient.py new file mode 100644 index 0000000000000000000000000000000000000000..cd36ecd45933ad10758e408cf03c1bf834d27ba6 --- /dev/null +++ b/src/telemetry/frontend/client/TelemetryFrontendClient.py @@ -0,0 +1,70 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc + +from common.proto.context_pb2 import Empty +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.client.RetryDecorator import retry, delay_exponential +from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceStub +from common.proto.telemetry_frontend_pb2 import Collector, CollectorId, CollectorFilter, CollectorList + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 10 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class TelemetryFrontendClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.TELEMETRYFRONTEND) + if not port: port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = TelemetryFrontendServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def StartCollector(self, request : Collector) -> CollectorId: # type: ignore + LOGGER.debug('StartCollector: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StartCollector(request) + LOGGER.debug('StartCollector result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def StopCollector(self, request : CollectorId) -> Empty: # type: ignore + LOGGER.debug('StopCollector: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StopCollector(request) + LOGGER.debug('StopCollector result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def SelectCollectors(self, request : CollectorFilter) -> CollectorList: # type: ignore + LOGGER.debug('SelectCollectors: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SelectCollectors(request) + LOGGER.debug('SelectCollectors result: {:s}'.format(grpc_message_to_json_string(response))) + return response + diff --git a/src/telemetry/frontend/client/__init__.py b/src/telemetry/frontend/client/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/telemetry/frontend/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/frontend/service/TelemetryFrontendService.py b/src/telemetry/frontend/service/TelemetryFrontendService.py new file mode 100644 index 0000000000000000000000000000000000000000..dc3f8df363a882db0f0ba3112a38f3bba3921c30 --- /dev/null +++ b/src/telemetry/frontend/service/TelemetryFrontendService.py @@ -0,0 +1,30 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from monitoring.service.NameMapping import NameMapping +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.proto.telemetry_frontend_pb2_grpc import add_TelemetryFrontendServiceServicer_to_server +from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl + + +class TelemetryFrontendService(GenericGrpcService): + def __init__(self, name_mapping : NameMapping, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) + super().__init__(port, cls_name=cls_name) + self.telemetry_frontend_servicer = TelemetryFrontendServiceServicerImpl(name_mapping) + + def install_servicers(self): + add_TelemetryFrontendServiceServicer_to_server(self.telemetry_frontend_servicer, self.server) diff --git a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py new file mode 100644 index 0000000000000000000000000000000000000000..e6830ad676d3934c88b01575ebdd1d0549fb00d1 --- /dev/null +++ b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -0,0 +1,204 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import threading +import time +from typing import Tuple, Any +import grpc +import logging + +from confluent_kafka import Consumer as KafkaConsumer +from common.proto.context_pb2 import Empty +from monitoring.service.NameMapping import NameMapping +from confluent_kafka import Producer as KafkaProducer +from confluent_kafka import KafkaException +from confluent_kafka import KafkaError +from common.proto.telemetry_frontend_pb2 import CollectorId, Collector, CollectorFilter, CollectorList +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceServicer + +from telemetry.database.TelemetryModel import Collector as CollectorModel +from telemetry.database.managementDB import managementDB + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('Monitoring', 'TelemetryFrontend') +KAFKA_SERVER_IP = '127.0.0.1:9092' +ACTIVE_COLLECTORS = [] +KAFKA_TOPICS = {'request' : 'topic_request', + 'response': 'topic_response'} + + +class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): + def __init__(self, name_mapping : NameMapping): + LOGGER.info('Init TelemetryFrontendService') + self.managementDBobj = managementDB() + self.kafka_producer = KafkaProducer({'bootstrap.servers': KAFKA_SERVER_IP,}) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KAFKA_SERVER_IP, + 'group.id' : 'frontend', + 'auto.offset.reset' : 'latest'}) + + def add_collector_to_db(self, request: Collector ): # type: ignore + try: + # Create a new Collector instance + collector_to_insert = CollectorModel() + collector_to_insert.collector_id = request.collector_id.collector_id.uuid + collector_to_insert.kpi_id = request.kpi_id.kpi_id.uuid + # collector_to_insert.collector_decription= request.collector + collector_to_insert.sampling_duration_s = request.duration_s + collector_to_insert.sampling_interval_s = request.interval_s + collector_to_insert.start_timestamp = time.time() + collector_to_insert.end_timestamp = time.time() + managementDB.add_row_to_db(collector_to_insert) + except Exception as e: + LOGGER.info("Unable to create collectorModel class object. {:}".format(e)) + + # @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StartCollector(self, + request : Collector, grpc_context: grpc.ServicerContext # type: ignore + ) -> CollectorId: # type: ignore + # push info to frontend db + LOGGER.info ("gRPC message: {:}".format(request)) + response = CollectorId() + _collector_id = str(request.collector_id.collector_id.uuid) + _collector_kpi_id = str(request.kpi_id.kpi_id.uuid) + _collector_duration = int(request.duration_s) + _collector_interval = int(request.interval_s) + # pushing Collector to DB + self.add_collector_to_db(request) + self.publish_to_kafka_request_topic(_collector_id, _collector_kpi_id, _collector_duration, _collector_interval) + # self.run_publish_to_kafka_request_topic(_collector_id, _collector_kpi_id, _collector_duration, _collector_interval) + response.collector_id.uuid = request.collector_id.collector_id.uuid # type: ignore + return response + + def run_publish_to_kafka_request_topic(self, msg_key: str, kpi: str, duration : int, interval: int): + # Add threading.Thread() response to dictonary and call start() in the next statement + threading.Thread(target=self.publish_to_kafka_request_topic, args=(msg_key, kpi, duration, interval)).start() + + def publish_to_kafka_request_topic(self, + collector_id: str, kpi: str, duration : int, interval: int + ): + """ + Method to generate collector request to Kafka topic. + """ + # time.sleep(5) + # producer_configs = { + # 'bootstrap.servers': KAFKA_SERVER_IP, + # } + # topic_request = "topic_request" + msg_value : Tuple [str, int, int] = (kpi, duration, interval) + # print ("Request generated: ", "Colletcor Id: ", collector_id, \ + # ", \nKPI: ", kpi, ", Duration: ", duration, ", Interval: ", interval) + # producerObj = KafkaProducer(producer_configs) + self.kafka_producer.produce(KAFKA_TOPICS['request'], key=collector_id, value= str(msg_value), callback=self.delivery_callback) + # producerObj.produce(KAFKA_TOPICS['request'], key=collector_id, value= str(msg_value), callback=self.delivery_callback) + LOGGER.info("Collector Request Generated: {:}, {:}, {:}, {:}".format(collector_id, kpi, duration, interval)) + # producerObj.produce(topic_request, key=collector_id, value= str(msg_value), callback=self.delivery_callback) + ACTIVE_COLLECTORS.append(collector_id) + self.kafka_producer.flush() + + def run_kafka_listener(self): + # print ("--- STARTED: run_kafka_listener ---") + threading.Thread(target=self.kafka_listener).start() + return True + + def kafka_listener(self): + """ + listener for response on Kafka topic. + """ + # # print ("--- STARTED: kafka_listener ---") + # conusmer_configs = { + # 'bootstrap.servers' : KAFKA_SERVER_IP, + # 'group.id' : 'frontend', + # 'auto.offset.reset' : 'latest' + # } + # # topic_response = "topic_response" + + # consumerObj = KafkaConsumer(conusmer_configs) + self.kafka_consumer.subscribe([KAFKA_TOPICS['response']]) + # print (time.time()) + while True: + receive_msg = self.kafka_consumer.poll(2.0) + if receive_msg is None: + # print (" - Telemetry frontend listening on Kafka Topic: ", KAFKA_TOPICS['response']) # added for debugging purposes + continue + elif receive_msg.error(): + if receive_msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + print("Consumer error: {}".format(receive_msg.error())) + break + try: + collector_id = receive_msg.key().decode('utf-8') + if collector_id in ACTIVE_COLLECTORS: + (kpi_id, kpi_value) = ast.literal_eval(receive_msg.value().decode('utf-8')) + self.process_response(collector_id, kpi_id, kpi_value) + else: + print(f"collector id does not match.\nRespone ID: '{collector_id}' --- Active IDs: '{ACTIVE_COLLECTORS}' ") + except Exception as e: + print(f"No message key found: {str(e)}") + continue + # return None + + def process_response(self, collector_id: str, kpi_id: str, kpi_value: Any): + if kpi_id == "-1" and kpi_value == -1: + # LOGGER.info("Sucessfully terminated Collector: {:}".format(collector_id)) + print ("Sucessfully terminated Collector: ", collector_id) + else: + print ("Frontend-Received values Collector Id:", collector_id, "-KPI:", kpi_id, "-VALUE:", kpi_value) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def delivery_callback(self, err, msg): + """ + Callback function to handle message delivery status. + Args: + err (KafkaError): Kafka error object. + msg (Message): Kafka message object. + """ + if err: + print(f'Message delivery failed: {err}') + else: + print(f'Message delivered to topic {msg.topic()}') + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StopCollector(self, + request : CollectorId, grpc_context: grpc.ServicerContext # type: ignore + ) -> Empty: # type: ignore + LOGGER.info ("gRPC message: {:}".format(request)) + _collector_id = request.collector_id.uuid + self.publish_to_kafka_request_topic(_collector_id, "", -1, -1) + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SelectCollectors(self, + request : CollectorFilter, contextgrpc_context: grpc.ServicerContext # type: ignore + ) -> CollectorList: # type: ignore + LOGGER.info("gRPC message: {:}".format(request)) + response = CollectorList() + filter_to_apply = dict() + filter_to_apply['kpi_id'] = request.kpi_id[0].kpi_id.uuid + # filter_to_apply['duration_s'] = request.duration_s[0] + try: + rows = self.managementDBobj.select_with_filter(CollectorModel, **filter_to_apply) + except Exception as e: + LOGGER.info('Unable to apply filter on kpi descriptor. {:}'.format(e)) + try: + if len(rows) != 0: + for row in rows: + collector_obj = Collector() + collector_obj.collector_id.collector_id.uuid = row.collector_id + response.collector_list.append(collector_obj) + return response + except Exception as e: + LOGGER.info('Unable to process response {:}'.format(e)) \ No newline at end of file diff --git a/src/telemetry/frontend/service/__init__.py b/src/telemetry/frontend/service/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/telemetry/frontend/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/frontend/service/__main__.py b/src/telemetry/frontend/service/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..3b0263706c3dad3756306d1ba8a3a104d568cd6f --- /dev/null +++ b/src/telemetry/frontend/service/__main__.py @@ -0,0 +1,72 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +import sys +import logging, threading +from prometheus_client import start_http_server +from monitoring.service.NameMapping import NameMapping +from .TelemetryFrontendService import TelemetryFrontendService +from monitoring.service.EventTools import EventsDeviceCollector +from common.Settings import ( + get_log_level, wait_for_environment_variables, get_env_var_name, + get_metrics_port ) + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER + + log_level = get_log_level() + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") + LOGGER = logging.getLogger(__name__) + +# ------- will be added later -------------- + # wait_for_environment_variables([ + # get_env_var_name + + + # ]) +# ------- will be added later -------------- + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + name_mapping = NameMapping() + + grpc_service = TelemetryFrontendService(name_mapping) + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.info('Terminating...') + grpc_service.stop() + + LOGGER.info('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/src/telemetry/frontend/tests/Messages.py b/src/telemetry/frontend/tests/Messages.py new file mode 100644 index 0000000000000000000000000000000000000000..1205898d13a610cd262979242e4f489f5e35cdb8 --- /dev/null +++ b/src/telemetry/frontend/tests/Messages.py @@ -0,0 +1,83 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid +import random +from common.proto import telemetry_frontend_pb2 +from common.proto.kpi_sample_types_pb2 import KpiSampleType + + +# ----------------------- "2nd" Iteration -------------------------------- +def create_collector_id(): + _collector_id = telemetry_frontend_pb2.CollectorId() + _collector_id.collector_id.uuid = uuid.uuid4() + return _collector_id + +# def create_collector_id_a(coll_id_str : str): +# _collector_id = telemetry_frontend_pb2.CollectorId() +# _collector_id.collector_id.uuid = str(coll_id_str) +# return _collector_id + +def create_collector_request(): + _create_collector_request = telemetry_frontend_pb2.Collector() + _create_collector_request.collector_id.collector_id.uuid = str(uuid.uuid4()) + _create_collector_request.kpi_id.kpi_id.uuid = "165d20c5-a446-42fa-812f-e2b7ed283c6f" + # _create_collector_request.collector = "collector description" + _create_collector_request.duration_s = float(random.randint(8, 16)) + _create_collector_request.interval_s = float(random.randint(2, 4)) + return _create_collector_request + +def create_collector_filter(): + _create_collector_filter = telemetry_frontend_pb2.CollectorFilter() + new_kpi_id = _create_collector_filter.kpi_id.add() + new_kpi_id.kpi_id.uuid = "165d20c5-a446-42fa-812f-e2b7ed283c6f" + return _create_collector_filter + +# ----------------------- "First" Iteration -------------------------------- +# def create_collector_request_a(): +# _create_collector_request_a = telemetry_frontend_pb2.Collector() +# _create_collector_request_a.collector_id.collector_id.uuid = "-1" +# return _create_collector_request_a + +# def create_collector_request_b(str_kpi_id, coll_duration_s, coll_interval_s +# ) -> telemetry_frontend_pb2.Collector: +# _create_collector_request_b = telemetry_frontend_pb2.Collector() +# _create_collector_request_b.collector_id.collector_id.uuid = '1' +# _create_collector_request_b.kpi_id.kpi_id.uuid = str_kpi_id +# _create_collector_request_b.duration_s = coll_duration_s +# _create_collector_request_b.interval_s = coll_interval_s +# return _create_collector_request_b + +# def create_collector_filter(): +# _create_collector_filter = telemetry_frontend_pb2.CollectorFilter() +# new_collector_id = _create_collector_filter.collector_id.add() +# new_collector_id.collector_id.uuid = "COLL1" +# new_kpi_id = _create_collector_filter.kpi_id.add() +# new_kpi_id.kpi_id.uuid = "KPI1" +# new_device_id = _create_collector_filter.device_id.add() +# new_device_id.device_uuid.uuid = 'DEV1' +# new_service_id = _create_collector_filter.service_id.add() +# new_service_id.service_uuid.uuid = 'SERV1' +# new_slice_id = _create_collector_filter.slice_id.add() +# new_slice_id.slice_uuid.uuid = 'SLC1' +# new_endpoint_id = _create_collector_filter.endpoint_id.add() +# new_endpoint_id.endpoint_uuid.uuid = 'END1' +# new_connection_id = _create_collector_filter.connection_id.add() +# new_connection_id.connection_uuid.uuid = 'CON1' +# _create_collector_filter.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) +# return _create_collector_filter + +# def create_collector_list(): +# _create_collector_list = telemetry_frontend_pb2.CollectorList() +# return _create_collector_list \ No newline at end of file diff --git a/src/telemetry/frontend/tests/__init__.py b/src/telemetry/frontend/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ee6f7071f145e06c3aeaefc09a43ccd88e619e3 --- /dev/null +++ b/src/telemetry/frontend/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/frontend/tests/test_frontend.py b/src/telemetry/frontend/tests/test_frontend.py new file mode 100644 index 0000000000000000000000000000000000000000..002cc430721845aa5aa18274375e2c22b5d77ff7 --- /dev/null +++ b/src/telemetry/frontend/tests/test_frontend.py @@ -0,0 +1,204 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +import pytest +import logging +from typing import Union + +from common.proto.context_pb2 import Empty +from common.Constants import ServiceNameEnum +from common.proto.telemetry_frontend_pb2 import CollectorId, CollectorList +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server +from context.client.ContextClient import ContextClient +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.Settings import ( + get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) + +from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendClient +from telemetry.frontend.service.TelemetryFrontendService import TelemetryFrontendService +from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl +from telemetry.frontend.tests.Messages import ( create_collector_request, create_collector_filter) +from telemetry.database.managementDB import managementDB +from telemetry.database.TelemetryEngine import TelemetryEngine + +from device.client.DeviceClient import DeviceClient +from device.service.DeviceService import DeviceService +from device.service.driver_api.DriverFactory import DriverFactory +from device.service.driver_api.DriverInstanceCache import DriverInstanceCache + +from monitoring.service.NameMapping import NameMapping + +os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' +from device.service.drivers import DRIVERS + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' +MOCKSERVICE_PORT = 10000 + +TELEMETRY_FRONTEND_PORT = str(MOCKSERVICE_PORT) + str(get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND)) +os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(TELEMETRY_FRONTEND_PORT) + +LOGGER = logging.getLogger(__name__) + +class MockContextService(GenericGrpcService): + # Mock Service implementing Context to simplify unitary tests of Monitoring + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +@pytest.fixture(scope='session') +def context_service(): + LOGGER.info('Initializing MockContextService...') + _service = MockContextService(MOCKSERVICE_PORT) + _service.start() + + LOGGER.info('Yielding MockContextService...') + yield _service + + LOGGER.info('Terminating MockContextService...') + _service.context_servicer.msg_broker.terminate() + _service.stop() + + LOGGER.info('Terminated MockContextService...') + +@pytest.fixture(scope='session') +def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing ContextClient...') + _client = ContextClient() + + LOGGER.info('Yielding ContextClient...') + yield _client + + LOGGER.info('Closing ContextClient...') + _client.close() + + LOGGER.info('Closed ContextClient...') + +@pytest.fixture(scope='session') +def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceService...') + driver_factory = DriverFactory(DRIVERS) + driver_instance_cache = DriverInstanceCache(driver_factory) + _service = DeviceService(driver_instance_cache) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding DeviceService...') + yield _service + + LOGGER.info('Terminating DeviceService...') + _service.stop() + + LOGGER.info('Terminated DeviceService...') + +@pytest.fixture(scope='session') +def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument + LOGGER.info('Initializing DeviceClient...') + _client = DeviceClient() + + LOGGER.info('Yielding DeviceClient...') + yield _client + + LOGGER.info('Closing DeviceClient...') + _client.close() + + LOGGER.info('Closed DeviceClient...') + +@pytest.fixture(scope='session') +def telemetryFrontend_service( + context_service : MockContextService, + device_service : DeviceService + ): + LOGGER.info('Initializing TelemetryFrontendService...') + name_mapping = NameMapping() + + _service = TelemetryFrontendService(name_mapping) + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding TelemetryFrontendService...') + yield _service + + LOGGER.info('Terminating TelemetryFrontendService...') + _service.stop() + + LOGGER.info('Terminated TelemetryFrontendService...') + +@pytest.fixture(scope='session') +def telemetryFrontend_client( + telemetryFrontend_service : TelemetryFrontendService + ): + LOGGER.info('Initializing TelemetryFrontendClient...') + _client = TelemetryFrontendClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding TelemetryFrontendClient...') + yield _client + + LOGGER.info('Closing TelemetryFrontendClient...') + _client.close() + + LOGGER.info('Closed TelemetryFrontendClient...') + + +########################### +# Tests Implementation of Telemetry Frontend +########################### + +def test_verify_db_and_table(): + LOGGER.info(' >>> test_verify_database_and_tables START: <<< ') + _engine = TelemetryEngine.get_engine() + managementDB.create_database(_engine) + managementDB.create_tables(_engine) + +def test_StartCollector(telemetryFrontend_client): + LOGGER.info(' >>> test_StartCollector START: <<< ') + response = telemetryFrontend_client.StartCollector(create_collector_request()) + LOGGER.debug(str(response)) + assert isinstance(response, CollectorId) + +def test_run_kafka_listener(): + LOGGER.info(' >>> test_run_kafka_listener START: <<< ') + name_mapping = NameMapping() + TelemetryFrontendServiceObj = TelemetryFrontendServiceServicerImpl(name_mapping) + response = TelemetryFrontendServiceObj.run_kafka_listener() # Method "run_kafka_listener" is not define in frontend.proto + LOGGER.debug(str(response)) + assert isinstance(response, bool) + +def test_StopCollector(telemetryFrontend_client): + LOGGER.info(' >>> test_StopCollector START: <<< ') + _collector_id = telemetryFrontend_client.StartCollector(create_collector_request()) + time.sleep(3) # wait for small amount before call the stopCollecter() + response = telemetryFrontend_client.StopCollector(_collector_id) + LOGGER.debug(str(response)) + assert isinstance(response, Empty) + +def test_select_collectors(telemetryFrontend_client): + LOGGER.info(' >>> test_select_collector requesting <<< ') + response = telemetryFrontend_client.SelectCollectors(create_collector_filter()) + LOGGER.info('Received Rows after applying Filter: {:} '.format(response)) + LOGGER.debug(str(response)) + assert isinstance(response, CollectorList) \ No newline at end of file diff --git a/src/telemetry/requirements.in b/src/telemetry/requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..a0e78d2bfb7270b9664ad5ba810e2f213d887bf7 --- /dev/null +++ b/src/telemetry/requirements.in @@ -0,0 +1,24 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +anytree==2.8.0 +APScheduler==3.10.1 +influx-line-protocol==0.1.4 +psycopg2-binary==2.9.3 +python-dateutil==2.8.2 +python-json-logger==2.0.2 +pytz==2024.1 +questdb==1.0.1 +requests==2.27.1 +xmltodict==0.12.0 \ No newline at end of file diff --git a/src/telemetry/telemetry_virenv.txt b/src/telemetry/telemetry_virenv.txt new file mode 100644 index 0000000000000000000000000000000000000000..e39f80b6593d6c41411751cdd0ea59ee05344570 --- /dev/null +++ b/src/telemetry/telemetry_virenv.txt @@ -0,0 +1,49 @@ +anytree==2.8.0 +APScheduler==3.10.1 +attrs==23.2.0 +certifi==2024.2.2 +charset-normalizer==2.0.12 +colorama==0.4.6 +confluent-kafka==2.3.0 +coverage==6.3 +future-fstrings==1.2.0 +greenlet==3.0.3 +grpcio==1.47.5 +grpcio-health-checking==1.47.5 +grpcio-tools==1.47.5 +grpclib==0.4.4 +h2==4.1.0 +hpack==4.0.0 +hyperframe==6.0.1 +idna==3.7 +influx-line-protocol==0.1.4 +iniconfig==2.0.0 +kafka-python==2.0.2 +multidict==6.0.5 +networkx==3.3 +packaging==24.0 +pluggy==1.5.0 +prettytable==3.5.0 +prometheus-client==0.13.0 +protobuf==3.20.3 +psycopg2-binary==2.9.3 +py==1.11.0 +py-cpuinfo==9.0.0 +pytest==6.2.5 +pytest-benchmark==3.4.1 +pytest-depends==1.0.1 +python-dateutil==2.8.2 +python-json-logger==2.0.2 +pytz==2024.1 +questdb==1.0.1 +requests==2.27.1 +six==1.16.0 +SQLAlchemy==1.4.52 +sqlalchemy-cockroachdb==1.4.4 +SQLAlchemy-Utils==0.38.3 +toml==0.10.2 +typing_extensions==4.12.0 +tzlocal==5.2 +urllib3==1.26.18 +wcwidth==0.2.13 +xmltodict==0.12.0