From 485d3178322cfc049fb9e12014bb15a5ecd5aa1e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 27 Aug 2024 11:24:33 +0000 Subject: [PATCH 01/70] Initial Version of Analytics Component - Added the Analytics Frontend client and service with no logic implemented yet. - Added enums and ports for the Analytics frontend and backend in the constants. - Added test files and messages. - Added test execution scripts. --- .../run_tests_locally-analytics-frontend.sh | 23 +++++ src/analytics/README.md | 4 + src/analytics/__init__.py | 15 +++ src/analytics/frontend/Dockerfile | 70 ++++++++++++++ src/analytics/frontend/__init__.py | 15 +++ .../client/AnalyticsFrontendClient.py | 68 +++++++++++++ src/analytics/frontend/client/__init__.py | 14 +++ src/analytics/frontend/requirements.in | 21 ++++ .../service/AnalyticsFrontendService.py | 28 ++++++ .../AnalyticsFrontendServiceServicerImpl.py | 58 +++++++++++ src/analytics/frontend/service/__init__.py | 15 +++ src/analytics/frontend/service/__main__.py | 52 ++++++++++ src/analytics/frontend/tests/messages.py | 58 +++++++++++ src/analytics/frontend/tests/test_frontend.py | 95 +++++++++++++++++++ src/common/Constants.py | 6 ++ 15 files changed, 542 insertions(+) create mode 100755 scripts/run_tests_locally-analytics-frontend.sh create mode 100644 src/analytics/README.md create mode 100644 src/analytics/__init__.py create mode 100644 src/analytics/frontend/Dockerfile create mode 100644 src/analytics/frontend/__init__.py create mode 100644 src/analytics/frontend/client/AnalyticsFrontendClient.py create mode 100644 src/analytics/frontend/client/__init__.py create mode 100644 src/analytics/frontend/requirements.in create mode 100644 src/analytics/frontend/service/AnalyticsFrontendService.py create mode 100644 src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py create mode 100644 src/analytics/frontend/service/__init__.py create mode 100644 src/analytics/frontend/service/__main__.py create mode 100644 src/analytics/frontend/tests/messages.py create mode 100644 src/analytics/frontend/tests/test_frontend.py diff --git a/scripts/run_tests_locally-analytics-frontend.sh b/scripts/run_tests_locally-analytics-frontend.sh new file mode 100755 index 000000000..58fd062f2 --- /dev/null +++ b/scripts/run_tests_locally-analytics-frontend.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + analytics/frontend/tests/test_frontend.py diff --git a/src/analytics/README.md b/src/analytics/README.md new file mode 100644 index 000000000..9663e5321 --- /dev/null +++ b/src/analytics/README.md @@ -0,0 +1,4 @@ +# How to locally run and test Analytic service (To be added soon) + +### Pre-requisets +The following requirements should be fulfilled before the execuation of Telemetry service. diff --git a/src/analytics/__init__.py b/src/analytics/__init__.py new file mode 100644 index 000000000..234a1af65 --- /dev/null +++ b/src/analytics/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/analytics/frontend/Dockerfile b/src/analytics/frontend/Dockerfile new file mode 100644 index 000000000..f3b8838b2 --- /dev/null +++ b/src/analytics/frontend/Dockerfile @@ -0,0 +1,70 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/analytics/frontend +WORKDIR /var/analyticstelemetry/frontend +COPY src/analytics/frontend/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/analytics/__init__.py analytics/__init__.py +COPY src/analytics/frontend/. analytics/frontend/ +COPY src/analytics/database/. analytics/database/ + +# Start the service +ENTRYPOINT ["python", "-m", "analytics.frontend.service"] diff --git a/src/analytics/frontend/__init__.py b/src/analytics/frontend/__init__.py new file mode 100644 index 000000000..234a1af65 --- /dev/null +++ b/src/analytics/frontend/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/analytics/frontend/client/AnalyticsFrontendClient.py b/src/analytics/frontend/client/AnalyticsFrontendClient.py new file mode 100644 index 000000000..bfa8cae45 --- /dev/null +++ b/src/analytics/frontend/client/AnalyticsFrontendClient.py @@ -0,0 +1,68 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.proto.context_pb2 import Empty +from common.proto.analytics_frontend_pb2_grpc import AnalyticsFrontendServiceStub +from common.proto.analytics_frontend_pb2 import AnalyzerId, Analyzer, AnalyzerFilter, AnalyzerList +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.client.RetryDecorator import retry, delay_exponential + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 10 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class AnalyticsFrontendClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.ANALYTICSFRONTEND) + if not port: port = get_service_port_grpc(ServiceNameEnum.ANALYTICSFRONTEND) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = AnalyticsFrontendServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def StartAnalyzer (self, request: Analyzer) -> AnalyzerId: #type: ignore + LOGGER.debug('StartAnalyzer: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StartAnalyzer(request) + LOGGER.debug('StartAnalyzer result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def StopAnalyzer(self, request : AnalyzerId) -> Empty: # type: ignore + LOGGER.debug('StopAnalyzer: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.StopAnalyzer(request) + LOGGER.debug('StopAnalyzer result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def SelectAnalyzers(self, request : AnalyzerFilter) -> AnalyzerList: # type: ignore + LOGGER.debug('SelectAnalyzers: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.SelectAnalyzers(request) + LOGGER.debug('SelectAnalyzers result: {:s}'.format(grpc_message_to_json_string(response))) + return response \ No newline at end of file diff --git a/src/analytics/frontend/client/__init__.py b/src/analytics/frontend/client/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/analytics/frontend/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/analytics/frontend/requirements.in b/src/analytics/frontend/requirements.in new file mode 100644 index 000000000..98cf96710 --- /dev/null +++ b/src/analytics/frontend/requirements.in @@ -0,0 +1,21 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +java==11.0.* +pyspark==3.5.2 +confluent-kafka==2.3.* +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* \ No newline at end of file diff --git a/src/analytics/frontend/service/AnalyticsFrontendService.py b/src/analytics/frontend/service/AnalyticsFrontendService.py new file mode 100644 index 000000000..e702c0144 --- /dev/null +++ b/src/analytics/frontend/service/AnalyticsFrontendService.py @@ -0,0 +1,28 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.proto.analytics_frontend_pb2_grpc import add_AnalyticsFrontendServiceServicer_to_server +from analytics.frontend.service.AnalyticsFrontendServiceServicerImpl import AnalyticsFrontendServiceServicerImpl + +class AnalyticsFrontendService(GenericGrpcService): + def __init__(self, cls_name: str = __name__): + port = get_service_port_grpc(ServiceNameEnum.ANALYTICSFRONTEND) + super().__init__(port, cls_name=cls_name) + self.analytics_frontend_servicer = AnalyticsFrontendServiceServicerImpl() + + def install_servicers(self): + add_AnalyticsFrontendServiceServicer_to_server(self.analytics_frontend_servicer, self.server) \ No newline at end of file diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py new file mode 100644 index 000000000..b981c038b --- /dev/null +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -0,0 +1,58 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging, grpc + +from common.proto.context_pb2 import Empty +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method + +from common.proto.analytics_frontend_pb2 import Analyzer, AnalyzerId, AnalyzerFilter, AnalyzerList +from common.proto.analytics_frontend_pb2_grpc import AnalyticsFrontendServiceServicer + + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('AnalyticsFrontend', 'NBIgRPC') + + +class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): + def __init__(self): + LOGGER.info('Init AnalyticsFrontendService') + + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StartAnalyzer(self, + request : Analyzer, grpc_context: grpc.ServicerContext # type: ignore + ) -> AnalyzerId: # type: ignore + LOGGER.info ("At Service gRPC message: {:}".format(request)) + response = AnalyzerId() + + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def StopAnalyzer(self, + request : AnalyzerId, grpc_context: grpc.ServicerContext # type: ignore + ) -> Empty: # type: ignore + LOGGER.info ("At Service gRPC message: {:}".format(request)) + + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def SelectAnalyzers(self, + request : AnalyzerFilter, contextgrpc_context: grpc.ServicerContext # type: ignore + ) -> AnalyzerList: # type: ignore + LOGGER.info("At Service gRPC message: {:}".format(request)) + response = AnalyzerList() + + return response diff --git a/src/analytics/frontend/service/__init__.py b/src/analytics/frontend/service/__init__.py new file mode 100644 index 000000000..234a1af65 --- /dev/null +++ b/src/analytics/frontend/service/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/analytics/frontend/service/__main__.py b/src/analytics/frontend/service/__main__.py new file mode 100644 index 000000000..e33a4c62b --- /dev/null +++ b/src/analytics/frontend/service/__main__.py @@ -0,0 +1,52 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Settings import get_log_level +from .AnalyticsFrontendService import AnalyticsFrontendService + + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = AnalyticsFrontendService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py new file mode 100644 index 000000000..1aaf8dd47 --- /dev/null +++ b/src/analytics/frontend/tests/messages.py @@ -0,0 +1,58 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid +from common.proto.kpi_manager_pb2 import KpiId +from common.proto.analytics_frontend_pb2 import ( AnalyzerOperationMode, AnalyzerId, + Analyzer, AnalyzerFilter ) + +def create_analyzer_id(): + _create_analyzer_id = AnalyzerId() + _create_analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) + return _create_analyzer_id + +def create_analyzer(): + _create_analyzer = Analyzer() + + _create_analyzer.algorithm_name = "some_algo_name" + + _kpi_id = KpiId() + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) # input IDs to analyze + _create_analyzer.input_kpi_ids.append(_kpi_id) + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) # output IDs after analysis + _create_analyzer.output_kpi_ids.append(_kpi_id) + + _create_analyzer.operation_mode = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING + + return _create_analyzer + +def create_analyzer_filter(): + _create_analyzer_filter = AnalyzerFilter() + + _analyzer_id_obj = AnalyzerId() + _analyzer_id_obj.analyzer_id.uuid = str(uuid.uuid4()) + _create_analyzer_filter.analyzer_id.append(_analyzer_id_obj) + + _create_analyzer_filter.algorithm_names.append('Algorithum1') + + _input_kpi_id_obj = KpiId() + _input_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + _create_analyzer_filter.input_kpi_ids.append(_input_kpi_id_obj) + + _output_kpi_id_obj = KpiId() + _output_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + _create_analyzer_filter.output_kpi_ids.append(_output_kpi_id_obj) + + return _create_analyzer_filter + diff --git a/src/analytics/frontend/tests/test_frontend.py b/src/analytics/frontend/tests/test_frontend.py new file mode 100644 index 000000000..ae7875b86 --- /dev/null +++ b/src/analytics/frontend/tests/test_frontend.py @@ -0,0 +1,95 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import logging + +from common.Constants import ServiceNameEnum +from common.proto.context_pb2 import Empty +from common.Settings import ( get_service_port_grpc, get_env_var_name, + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC ) + +from common.proto.analytics_frontend_pb2 import AnalyzerId, AnalyzerList +from analytics.frontend.client.AnalyticsFrontendClient import AnalyticsFrontendClient +from analytics.frontend.service.AnalyticsFrontendService import AnalyticsFrontendService +from analytics.frontend.tests.messages import ( create_analyzer_id, create_analyzer, + create_analyzer_filter ) + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' + +ANALYTICS_FRONTEND_PORT = str(get_service_port_grpc(ServiceNameEnum.ANALYTICSFRONTEND)) +os.environ[get_env_var_name(ServiceNameEnum.ANALYTICSFRONTEND, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.ANALYTICSFRONTEND, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(ANALYTICS_FRONTEND_PORT) + +LOGGER = logging.getLogger(__name__) + +@pytest.fixture(scope='session') +def analyticsFrontend_service(): + LOGGER.info('Initializing AnalyticsFrontendService...') + + _service = AnalyticsFrontendService() + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding AnalyticsFrontendService...') + yield _service + + LOGGER.info('Terminating AnalyticsFrontendService...') + _service.stop() + + LOGGER.info('Terminated AnalyticsFrontendService...') + +@pytest.fixture(scope='session') +def analyticsFrontend_client(analyticsFrontend_service : AnalyticsFrontendService): + LOGGER.info('Initializing AnalyticsFrontendClient...') + + _client = AnalyticsFrontendClient() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding AnalyticsFrontendClient...') + yield _client + + LOGGER.info('Closing AnalyticsFrontendClient...') + _client.close() + + LOGGER.info('Closed AnalyticsFrontendClient...') + + +########################### +# Tests Implementation of Analytics Frontend +########################### + +# ----- core funtionality test ----- +def test_StartAnalytic(analyticsFrontend_client): + LOGGER.info(' >>> test_StartAnalytic START: <<< ') + response = analyticsFrontend_client.StartAnalyzer(create_analyzer()) + LOGGER.debug(str(response)) + assert isinstance(response, AnalyzerId) + +def test_StopAnalytic(analyticsFrontend_client): + LOGGER.info(' >>> test_StopAnalytic START: <<< ') + response = analyticsFrontend_client.StopAnalyzer(create_analyzer_id()) + LOGGER.debug(str(response)) + assert isinstance(response, Empty) + +def test_SelectAnalytics(analyticsFrontend_client): + LOGGER.info(' >>> test_SelectAnalytics START: <<< ') + response = analyticsFrontend_client.SelectAnalyzers(create_analyzer_filter()) + LOGGER.debug(str(response)) + assert isinstance(response, AnalyzerList) \ No newline at end of file diff --git a/src/common/Constants.py b/src/common/Constants.py index 767b21343..74490321f 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -65,6 +65,9 @@ class ServiceNameEnum(Enum): KPIVALUEAPI = 'kpi-value-api' KPIVALUEWRITER = 'kpi-value-writer' TELEMETRYFRONTEND = 'telemetry-frontend' + TELEMETRYBACKEND = 'telemetry-backend' + ANALYTICSFRONTEND = 'analytics-frontend' + ANALYTICSBACKEND = 'analytics-backend' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -98,6 +101,9 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.KPIVALUEAPI .value : 30020, ServiceNameEnum.KPIVALUEWRITER .value : 30030, ServiceNameEnum.TELEMETRYFRONTEND .value : 30050, + ServiceNameEnum.TELEMETRYBACKEND .value : 30060, + ServiceNameEnum.ANALYTICSFRONTEND .value : 30080, + ServiceNameEnum.ANALYTICSBACKEND .value : 30090, # Used for test and debugging only ServiceNameEnum.DLT_GATEWAY .value : 50051, -- GitLab From 8072cdfb1f53b091a5ea43659dcb1b26313247f4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 5 Aug 2024 12:13:34 +0000 Subject: [PATCH 02/70] Changes are made to restructure the Telemetry DB operations. - TelemetryEngine.py is updated. - TelemetryModel.py is refined. - Optimized DB operation were added in TelemetryDB.py --- src/telemetry/database/TelemetryDB.py | 137 ++++++++++ src/telemetry/database/TelemetryDBmanager.py | 248 ------------------- src/telemetry/database/TelemetryEngine.py | 39 +-- src/telemetry/database/TelemetryModel.py | 23 +- src/telemetry/database/managementDB.py | 138 ----------- 5 files changed, 160 insertions(+), 425 deletions(-) create mode 100644 src/telemetry/database/TelemetryDB.py delete mode 100644 src/telemetry/database/TelemetryDBmanager.py delete mode 100644 src/telemetry/database/managementDB.py diff --git a/src/telemetry/database/TelemetryDB.py b/src/telemetry/database/TelemetryDB.py new file mode 100644 index 000000000..b5b0c4c7e --- /dev/null +++ b/src/telemetry/database/TelemetryDB.py @@ -0,0 +1,137 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sqlalchemy_utils +from sqlalchemy.orm import sessionmaker +from telemetry.database.TelemetryModel import Collector as CollectorModel +from telemetry.database.TelemetryEngine import TelemetryEngine +from common.method_wrappers.ServiceExceptions import ( + OperationFailedException, AlreadyExistsException ) + +LOGGER = logging.getLogger(__name__) +DB_NAME = "tfs_telemetry" + +class TelemetryDBmanager: + def __init__(self): + self.db_engine = TelemetryEngine.get_engine() + if self.db_engine is None: + LOGGER.error('Unable to get SQLAlchemy DB Engine...') + return False + self.db_name = DB_NAME + self.Session = sessionmaker(bind=self.db_engine) + + def create_database(self): + if not sqlalchemy_utils.database_exists(self.db_engine.url): + LOGGER.debug("Database created. {:}".format(self.db_engine.url)) + sqlalchemy_utils.create_database(self.db_engine.url) + + def drop_database(self) -> None: + if sqlalchemy_utils.database_exists(self.db_engine.url): + sqlalchemy_utils.drop_database(self.db_engine.url) + + def create_tables(self): + try: + CollectorModel.metadata.create_all(self.db_engine) # type: ignore + LOGGER.debug("Tables created in the database: {:}".format(self.db_name)) + except Exception as e: + LOGGER.debug("Tables cannot be created in the database. {:s}".format(str(e))) + raise OperationFailedException ("Tables can't be created", extra_details=["unable to create table {:}".format(e)]) + + def verify_tables(self): + try: + with self.db_engine.connect() as connection: + result = connection.execute("SHOW TABLES;") + tables = result.fetchall() + LOGGER.info("Tables in DB: {:}".format(tables)) + except Exception as e: + LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) + +# ----------------- CURD METHODs --------------------- + + def add_row_to_db(self, row): + session = self.Session() + try: + session.add(row) + session.commit() + LOGGER.debug(f"Row inserted into {row.__class__.__name__} table.") + return True + except Exception as e: + session.rollback() + if "psycopg2.errors.UniqueViolation" in str(e): + LOGGER.error(f"Unique key voilation: {row.__class__.__name__} table. {str(e)}") + raise AlreadyExistsException(row.__class__.__name__, row, + extra_details=["Unique key voilation: {:}".format(e)] ) + else: + LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}") + raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) + finally: + session.close() + + def search_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + entity = session.query(model).filter_by(**{col_name: id_to_search}).first() + if entity: + # LOGGER.debug(f"{model.__name__} ID found: {str(entity)}") + return entity + else: + LOGGER.debug(f"{model.__name__} ID not found, No matching row: {str(id_to_search)}") + print("{:} ID not found, No matching row: {:}".format(model.__name__, id_to_search)) + return None + except Exception as e: + session.rollback() + LOGGER.debug(f"Failed to retrieve {model.__name__} ID. {str(e)}") + raise OperationFailedException ("search by column id", extra_details=["unable to search row {:}".format(e)]) + finally: + session.close() + + def delete_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + record = session.query(model).filter_by(**{col_name: id_to_search}).first() + if record: + session.delete(record) + session.commit() + LOGGER.debug("Deleted %s with %s: %s", model.__name__, col_name, id_to_search) + else: + LOGGER.debug("%s with %s %s not found", model.__name__, col_name, id_to_search) + return None + except Exception as e: + session.rollback() + LOGGER.error("Error deleting %s with %s %s: %s", model.__name__, col_name, id_to_search, e) + raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) + finally: + session.close() + + def select_with_filter(self, model, filter_object): + session = self.Session() + try: + query = session.query(CollectorModel) + # Apply filters based on the filter_object + if filter_object.kpi_id: + query = query.filter(CollectorModel.kpi_id.in_([k.kpi_id.uuid for k in filter_object.kpi_id])) + result = query.all() + + if result: + LOGGER.debug(f"Fetched filtered rows from {model.__name__} table with filters: {filter_object}") # - Results: {result} + else: + LOGGER.debug(f"No matching row found in {model.__name__} table with filters: {filter_object}") + return result + except Exception as e: + LOGGER.error(f"Error fetching filtered rows from {model.__name__} table with filters {filter_object} ::: {e}") + raise OperationFailedException ("Select by filter", extra_details=["unable to apply the filter {:}".format(e)]) + finally: + session.close() + diff --git a/src/telemetry/database/TelemetryDBmanager.py b/src/telemetry/database/TelemetryDBmanager.py deleted file mode 100644 index b558180a9..000000000 --- a/src/telemetry/database/TelemetryDBmanager.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, time -import sqlalchemy -from sqlalchemy import inspect, MetaData, Table -from sqlalchemy.orm import sessionmaker -from telemetry.database.TelemetryModel import Collector as CollectorModel -from telemetry.database.TelemetryModel import Kpi as KpiModel -from sqlalchemy.ext.declarative import declarative_base -from telemetry.database.TelemetryEngine import TelemetryEngine -from common.proto.kpi_manager_pb2 import KpiDescriptor, KpiId -from common.proto.telemetry_frontend_pb2 import Collector, CollectorId -from sqlalchemy.exc import SQLAlchemyError -from telemetry.database.TelemetryModel import Base - -LOGGER = logging.getLogger(__name__) -DB_NAME = "telemetryfrontend" - -class TelemetryDBmanager: - def __init__(self): - self.db_engine = TelemetryEngine.get_engine() - if self.db_engine is None: - LOGGER.error('Unable to get SQLAlchemy DB Engine...') - return False - self.db_name = DB_NAME - self.Session = sessionmaker(bind=self.db_engine) - - def create_database(self): - try: - # with self.db_engine.connect() as connection: - # connection.execute(f"CREATE DATABASE {self.db_name};") - TelemetryEngine.create_database(self.db_engine) - LOGGER.info('TelemetryDBmanager initalized DB Name: {:}'.format(self.db_name)) - return True - except Exception as e: # pylint: disable=bare-except # pragma: no cover - LOGGER.exception('Failed to check/create the database: {:s}'.format(str(e))) - return False - - def create_tables(self): - try: - Base.metadata.create_all(self.db_engine) # type: ignore - LOGGER.info("Tables created in database ({:}) the as per Models".format(self.db_name)) - except Exception as e: - LOGGER.info("Tables cannot be created in the TelemetryFrontend database. {:s}".format(str(e))) - - def verify_tables(self): - try: - with self.db_engine.connect() as connection: - result = connection.execute("SHOW TABLES;") - tables = result.fetchall() - LOGGER.info("Tables in DB: {:}".format(tables)) - except Exception as e: - LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) - - def drop_table(self, table_to_drop: str): - try: - inspector = inspect(self.db_engine) - existing_tables = inspector.get_table_names() - if table_to_drop in existing_tables: - table = Table(table_to_drop, MetaData(), autoload_with=self.db_engine) - table.drop(self.db_engine) - LOGGER.info("Tables delete in the DB Name: {:}".format(self.db_name)) - else: - LOGGER.warning("No table {:} in database {:} ".format(table_to_drop, DB_NAME)) - except Exception as e: - LOGGER.info("Tables cannot be deleted in the {:} database. {:s}".format(DB_NAME, str(e))) - - def list_databases(self): - query = "SHOW DATABASES" - with self.db_engine.connect() as connection: - result = connection.execute(query) - databases = [row[0] for row in result] - LOGGER.info("List of available DBs: {:}".format(databases)) - -# ------------------ INSERT METHODs -------------------------------------- - - def inser_kpi(self, request: KpiDescriptor): - session = self.Session() - try: - # Create a new Kpi instance - kpi_to_insert = KpiModel() - kpi_to_insert.kpi_id = request.kpi_id.kpi_id.uuid - kpi_to_insert.kpi_description = request.kpi_description - kpi_to_insert.kpi_sample_type = request.kpi_sample_type - kpi_to_insert.device_id = request.service_id.service_uuid.uuid - kpi_to_insert.endpoint_id = request.device_id.device_uuid.uuid - kpi_to_insert.service_id = request.slice_id.slice_uuid.uuid - kpi_to_insert.slice_id = request.endpoint_id.endpoint_uuid.uuid - kpi_to_insert.connection_id = request.connection_id.connection_uuid.uuid - # kpi_to_insert.link_id = request.link_id.link_id.uuid - # Add the instance to the session - session.add(kpi_to_insert) - session.commit() - LOGGER.info("Row inserted into kpi table: {:}".format(kpi_to_insert.kpi_id)) - except Exception as e: - session.rollback() - LOGGER.info("Failed to insert new kpi. {:s}".format(str(e))) - finally: - # Close the session - session.close() - - # Function to insert a row into the Collector model - def insert_collector(self, request: Collector): - session = self.Session() - try: - # Create a new Collector instance - collector_to_insert = CollectorModel() - collector_to_insert.collector_id = request.collector_id.collector_id.uuid - collector_to_insert.kpi_id = request.kpi_id.kpi_id.uuid - collector_to_insert.collector = "Test collector description" - collector_to_insert.sampling_duration_s = request.duration_s - collector_to_insert.sampling_interval_s = request.interval_s - collector_to_insert.start_timestamp = time.time() - collector_to_insert.end_timestamp = time.time() - - session.add(collector_to_insert) - session.commit() - LOGGER.info("Row inserted into collector table: {:}".format(collector_to_insert.collector_id)) - except Exception as e: - session.rollback() - LOGGER.info("Failed to insert new collector. {:s}".format(str(e))) - finally: - # Close the session - session.close() - -# ------------------ GET METHODs -------------------------------------- - - def get_kpi_descriptor(self, request: KpiId): - session = self.Session() - try: - kpi_id_to_search = request.kpi_id.uuid - kpi = session.query(KpiModel).filter_by(kpi_id=kpi_id_to_search).first() - if kpi: - LOGGER.info("kpi ID found: {:s}".format(str(kpi))) - return kpi - else: - LOGGER.warning("Kpi ID not found {:s}".format(str(kpi_id_to_search))) - return None - except Exception as e: - session.rollback() - LOGGER.info("Failed to retrieve KPI ID. {:s}".format(str(e))) - raise - finally: - session.close() - - def get_collector(self, request: CollectorId): - session = self.Session() - try: - collector_id_to_search = request.collector_id.uuid - collector = session.query(CollectorModel).filter_by(collector_id=collector_id_to_search).first() - if collector: - LOGGER.info("collector ID found: {:s}".format(str(collector))) - return collector - else: - LOGGER.warning("collector ID not found{:s}".format(str(collector_id_to_search))) - return None - except Exception as e: - session.rollback() - LOGGER.info("Failed to retrieve collector ID. {:s}".format(str(e))) - raise - finally: - session.close() - - # ------------------ SELECT METHODs -------------------------------------- - - def select_kpi_descriptor(self, **filters): - session = self.Session() - try: - query = session.query(KpiModel) - for column, value in filters.items(): - query = query.filter(getattr(KpiModel, column) == value) - result = query.all() - if len(result) != 0: - LOGGER.info("Fetched filtered rows from KPI table with filters : {:s}".format(str(result))) - else: - LOGGER.warning("No matching row found : {:s}".format(str(result))) - return result - except SQLAlchemyError as e: - LOGGER.error("Error fetching filtered rows from KPI table with filters {:}: {:}".format(filters, e)) - return [] - finally: - session.close() - - def select_collector(self, **filters): - session = self.Session() - try: - query = session.query(CollectorModel) - for column, value in filters.items(): - query = query.filter(getattr(CollectorModel, column) == value) - result = query.all() - if len(result) != 0: - LOGGER.info("Fetched filtered rows from KPI table with filters : {:s}".format(str(result))) - else: - LOGGER.warning("No matching row found : {:s}".format(str(result))) - return result - except SQLAlchemyError as e: - LOGGER.error("Error fetching filtered rows from KPI table with filters {:}: {:}".format(filters, e)) - return [] - finally: - session.close() - -# ------------------ DELETE METHODs -------------------------------------- - - def delete_kpi_descriptor(self, request: KpiId): - session = self.Session() - try: - kpi_id_to_delete = request.kpi_id.uuid - kpi = session.query(KpiModel).filter_by(kpi_id=kpi_id_to_delete).first() - if kpi: - session.delete(kpi) - session.commit() - LOGGER.info("Deleted KPI with kpi_id: %s", kpi_id_to_delete) - else: - LOGGER.warning("KPI with kpi_id %s not found", kpi_id_to_delete) - except SQLAlchemyError as e: - session.rollback() - LOGGER.error("Error deleting KPI with kpi_id %s: %s", kpi_id_to_delete, e) - finally: - session.close() - - def delete_collector(self, request: CollectorId): - session = self.Session() - try: - collector_id_to_delete = request.collector_id.uuid - collector = session.query(CollectorModel).filter_by(collector_id=collector_id_to_delete).first() - if collector: - session.delete(collector) - session.commit() - LOGGER.info("Deleted collector with collector_id: %s", collector_id_to_delete) - else: - LOGGER.warning("collector with collector_id %s not found", collector_id_to_delete) - except SQLAlchemyError as e: - session.rollback() - LOGGER.error("Error deleting collector with collector_id %s: %s", collector_id_to_delete, e) - finally: - session.close() \ No newline at end of file diff --git a/src/telemetry/database/TelemetryEngine.py b/src/telemetry/database/TelemetryEngine.py index a563fa09f..bd7cda599 100644 --- a/src/telemetry/database/TelemetryEngine.py +++ b/src/telemetry/database/TelemetryEngine.py @@ -12,34 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, sqlalchemy, sqlalchemy_utils -# from common.Settings import get_setting +import logging, sqlalchemy +from common.Settings import get_setting LOGGER = logging.getLogger(__name__) -APP_NAME = 'tfs' -ECHO = False # False: No dump SQL commands and transactions executed CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' # CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' class TelemetryEngine: - # def __init__(self): - # self.engine = self.get_engine() @staticmethod def get_engine() -> sqlalchemy.engine.Engine: - CRDB_NAMESPACE = "crdb" - CRDB_SQL_PORT = "26257" - CRDB_DATABASE = "telemetryfrontend" - CRDB_USERNAME = "tfs" - CRDB_PASSWORD = "tfs123" - CRDB_SSLMODE = "require" - crdb_uri = CRDB_URI_TEMPLATE.format( - CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) - # crdb_uri = CRDB_URI_TEMPLATE.format( - # CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + crdb_uri = get_setting('CRDB_URI', default=None) + if crdb_uri is None: + CRDB_NAMESPACE = "crdb" + CRDB_SQL_PORT = "26257" + CRDB_DATABASE = "telemetryfrontend" + CRDB_USERNAME = "tfs" + CRDB_PASSWORD = "tfs123" + CRDB_SSLMODE = "require" + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: - # engine = sqlalchemy.create_engine( - # crdb_uri, connect_args={'application_name': APP_NAME}, echo=ECHO, future=True) engine = sqlalchemy.create_engine(crdb_uri, echo=False) LOGGER.info(' TelemetryDBmanager initalized with DB URL: {:}'.format(crdb_uri)) except: # pylint: disable=bare-except # pragma: no cover @@ -47,13 +41,4 @@ class TelemetryEngine: return None # type: ignore return engine # type: ignore - @staticmethod - def create_database(engine : sqlalchemy.engine.Engine) -> None: - if not sqlalchemy_utils.database_exists(engine.url): - LOGGER.info("Database created. {:}".format(engine.url)) - sqlalchemy_utils.create_database(engine.url) - @staticmethod - def drop_database(engine : sqlalchemy.engine.Engine) -> None: - if sqlalchemy_utils.database_exists(engine.url): - sqlalchemy_utils.drop_database(engine.url) diff --git a/src/telemetry/database/TelemetryModel.py b/src/telemetry/database/TelemetryModel.py index be4f0969c..95f692e4b 100644 --- a/src/telemetry/database/TelemetryModel.py +++ b/src/telemetry/database/TelemetryModel.py @@ -14,9 +14,7 @@ import logging from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy import Column, Integer, String, Float, Text, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker, relationship +from sqlalchemy import Column, String, Float from sqlalchemy.orm import registry logging.basicConfig(level=logging.INFO) @@ -24,22 +22,23 @@ LOGGER = logging.getLogger(__name__) # Create a base class for declarative models Base = registry().generate_base() -# Base = declarative_base() class Collector(Base): __tablename__ = 'collector' collector_id = Column(UUID(as_uuid=False), primary_key=True) - kpi_id = Column(UUID(as_uuid=False)) - collector_decription = Column(String) - sampling_duration_s = Column(Float) - sampling_interval_s = Column(Float) - start_timestamp = Column(Float) - end_timestamp = Column(Float) - + kpi_id = Column(UUID(as_uuid=False), nullable=False) + collector_decription = Column(String , nullable=False) + sampling_duration_s = Column(Float , nullable=False) + sampling_interval_s = Column(Float , nullable=False) + start_timestamp = Column(Float , nullable=False) + end_timestamp = Column(Float , nullable=False) + # helps in logging the information def __repr__(self): return (f"") \ No newline at end of file + f"end_timestamp='{self.end_timestamp}')>") + +# add method to convert gRPC requests to rows if necessary... diff --git a/src/telemetry/database/managementDB.py b/src/telemetry/database/managementDB.py deleted file mode 100644 index f79126f27..000000000 --- a/src/telemetry/database/managementDB.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, time -import sqlalchemy -import sqlalchemy_utils -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.declarative import declarative_base -from telemetry.database.TelemetryEngine import TelemetryEngine -from telemetry.database.TelemetryModel import Base - -LOGGER = logging.getLogger(__name__) -DB_NAME = "telemetryfrontend" - -# # Create a base class for declarative models -# Base = declarative_base() - -class managementDB: - def __init__(self): - self.db_engine = TelemetryEngine.get_engine() - if self.db_engine is None: - LOGGER.error('Unable to get SQLAlchemy DB Engine...') - return False - self.db_name = DB_NAME - self.Session = sessionmaker(bind=self.db_engine) - - @staticmethod - def create_database(engine : sqlalchemy.engine.Engine) -> None: - if not sqlalchemy_utils.database_exists(engine.url): - LOGGER.info("Database created. {:}".format(engine.url)) - sqlalchemy_utils.create_database(engine.url) - - @staticmethod - def drop_database(engine : sqlalchemy.engine.Engine) -> None: - if sqlalchemy_utils.database_exists(engine.url): - sqlalchemy_utils.drop_database(engine.url) - - # def create_database(self): - # try: - # with self.db_engine.connect() as connection: - # connection.execute(f"CREATE DATABASE {self.db_name};") - # LOGGER.info('managementDB initalizes database. Name: {self.db_name}') - # return True - # except: - # LOGGER.exception('Failed to check/create the database: {:s}'.format(str(self.db_engine.url))) - # return False - - @staticmethod - def create_tables(engine : sqlalchemy.engine.Engine): - try: - Base.metadata.create_all(engine) # type: ignore - LOGGER.info("Tables created in the DB Name: {:}".format(DB_NAME)) - except Exception as e: - LOGGER.info("Tables cannot be created in the TelemetryFrontend database. {:s}".format(str(e))) - - def verify_tables(self): - try: - with self.db_engine.connect() as connection: - result = connection.execute("SHOW TABLES;") - tables = result.fetchall() # type: ignore - LOGGER.info("Tables verified: {:}".format(tables)) - except Exception as e: - LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) - - @staticmethod - def add_row_to_db(self, row): - session = self.Session() - try: - session.add(row) - session.commit() - LOGGER.info(f"Row inserted into {row.__class__.__name__} table.") - except Exception as e: - session.rollback() - LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}") - finally: - session.close() - - def search_db_row_by_id(self, model, col_name, id_to_search): - session = self.Session() - try: - entity = session.query(model).filter_by(**{col_name: id_to_search}).first() - if entity: - LOGGER.info(f"{model.__name__} ID found: {str(entity)}") - return entity - else: - LOGGER.warning(f"{model.__name__} ID not found: {str(id_to_search)}") - return None - except Exception as e: - session.rollback() - LOGGER.info(f"Failed to retrieve {model.__name__} ID. {str(e)}") - raise - finally: - session.close() - - def delete_db_row_by_id(self, model, col_name, id_to_search): - session = self.Session() - try: - record = session.query(model).filter_by(**{col_name: id_to_search}).first() - if record: - session.delete(record) - session.commit() - LOGGER.info("Deleted %s with %s: %s", model.__name__, col_name, id_to_search) - else: - LOGGER.warning("%s with %s %s not found", model.__name__, col_name, id_to_search) - except Exception as e: - session.rollback() - LOGGER.error("Error deleting %s with %s %s: %s", model.__name__, col_name, id_to_search, e) - finally: - session.close() - - def select_with_filter(self, model, **filters): - session = self.Session() - try: - query = session.query(model) - for column, value in filters.items(): - query = query.filter(getattr(model, column) == value) # type: ignore - result = query.all() - if result: - LOGGER.info(f"Fetched filtered rows from {model.__name__} table with filters: {filters}") # - Results: {result} - else: - LOGGER.warning(f"No matching row found in {model.__name__} table with filters: {filters}") - return result - except Exception as e: - LOGGER.error(f"Error fetching filtered rows from {model.__name__} table with filters {filters} ::: {e}") - return [] - finally: - session.close() \ No newline at end of file -- GitLab From 35ef2d9d5970923d90eebfe60fd6da43744ec745 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 5 Aug 2024 13:05:25 +0000 Subject: [PATCH 03/70] chanegs to sucessfully executes the DB test. - updated test file to log DEBUG cmd - change the class name of "TelemetryDBmanager" to "TelemeterDB" - corrected the DB name - move messages.py and test files to correcte location. --- scripts/run_tests_locally-telemetry-DB.sh | 4 +- src/telemetry/database/TelemetryEngine.py | 4 +- .../{TelemetryDB.py => Telemetry_DB.py} | 10 +- src/telemetry/database/tests/__init__.py | 13 - .../database/tests/telemetryDBtests.py | 86 ----- src/telemetry/database/tests/temp_DB.py | 327 ------------------ .../{database => }/tests/messages.py | 0 .../test_telemetryDB.py} | 16 +- 8 files changed, 20 insertions(+), 440 deletions(-) rename src/telemetry/database/{TelemetryDB.py => Telemetry_DB.py} (96%) delete mode 100644 src/telemetry/database/tests/__init__.py delete mode 100644 src/telemetry/database/tests/telemetryDBtests.py delete mode 100644 src/telemetry/database/tests/temp_DB.py rename src/telemetry/{database => }/tests/messages.py (100%) rename src/telemetry/{database/tests/managementDBtests.py => tests/test_telemetryDB.py} (59%) diff --git a/scripts/run_tests_locally-telemetry-DB.sh b/scripts/run_tests_locally-telemetry-DB.sh index bb1c48b76..4b9a41760 100755 --- a/scripts/run_tests_locally-telemetry-DB.sh +++ b/scripts/run_tests_locally-telemetry-DB.sh @@ -22,5 +22,5 @@ cd $PROJECTDIR/src # kpi_manager/tests/test_unitary.py RCFILE=$PROJECTDIR/coverage/.coveragerc -python3 -m pytest --log-cli-level=INFO --verbose \ - telemetry/database/tests/telemetryDBtests.py +python3 -m pytest --log-level=DEBUG --log-cli-level=debug --verbose \ + telemetry/tests/test_telemetryDB.py diff --git a/src/telemetry/database/TelemetryEngine.py b/src/telemetry/database/TelemetryEngine.py index bd7cda599..965b7c38d 100644 --- a/src/telemetry/database/TelemetryEngine.py +++ b/src/telemetry/database/TelemetryEngine.py @@ -27,7 +27,7 @@ class TelemetryEngine: if crdb_uri is None: CRDB_NAMESPACE = "crdb" CRDB_SQL_PORT = "26257" - CRDB_DATABASE = "telemetryfrontend" + CRDB_DATABASE = "tfs-telemetry" CRDB_USERNAME = "tfs" CRDB_PASSWORD = "tfs123" CRDB_SSLMODE = "require" @@ -35,7 +35,7 @@ class TelemetryEngine: CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: engine = sqlalchemy.create_engine(crdb_uri, echo=False) - LOGGER.info(' TelemetryDBmanager initalized with DB URL: {:}'.format(crdb_uri)) + LOGGER.info(' TelemetryDB initalized with DB URL: {:}'.format(crdb_uri)) except: # pylint: disable=bare-except # pragma: no cover LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) return None # type: ignore diff --git a/src/telemetry/database/TelemetryDB.py b/src/telemetry/database/Telemetry_DB.py similarity index 96% rename from src/telemetry/database/TelemetryDB.py rename to src/telemetry/database/Telemetry_DB.py index b5b0c4c7e..ec7da9e40 100644 --- a/src/telemetry/database/TelemetryDB.py +++ b/src/telemetry/database/Telemetry_DB.py @@ -14,6 +14,7 @@ import logging import sqlalchemy_utils +from sqlalchemy import inspect from sqlalchemy.orm import sessionmaker from telemetry.database.TelemetryModel import Collector as CollectorModel from telemetry.database.TelemetryEngine import TelemetryEngine @@ -23,7 +24,7 @@ from common.method_wrappers.ServiceExceptions import ( LOGGER = logging.getLogger(__name__) DB_NAME = "tfs_telemetry" -class TelemetryDBmanager: +class TelemetryDB: def __init__(self): self.db_engine = TelemetryEngine.get_engine() if self.db_engine is None: @@ -51,10 +52,9 @@ class TelemetryDBmanager: def verify_tables(self): try: - with self.db_engine.connect() as connection: - result = connection.execute("SHOW TABLES;") - tables = result.fetchall() - LOGGER.info("Tables in DB: {:}".format(tables)) + inspect_object = inspect(self.db_engine) + if(inspect_object.has_table('collector', None)): + LOGGER.info("Table exists in DB: {:}".format(self.db_name)) except Exception as e: LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) diff --git a/src/telemetry/database/tests/__init__.py b/src/telemetry/database/tests/__init__.py deleted file mode 100644 index 839e45e3b..000000000 --- a/src/telemetry/database/tests/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/src/telemetry/database/tests/telemetryDBtests.py b/src/telemetry/database/tests/telemetryDBtests.py deleted file mode 100644 index 0d2211064..000000000 --- a/src/telemetry/database/tests/telemetryDBtests.py +++ /dev/null @@ -1,86 +0,0 @@ - -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from typing import Any -from sqlalchemy.ext.declarative import declarative_base -from telemetry.database.TelemetryDBmanager import TelemetryDBmanager -from telemetry.database.TelemetryEngine import TelemetryEngine -from telemetry.database.tests import temp_DB -from .messages import create_kpi_request, create_collector_request, \ - create_kpi_id_request, create_kpi_filter_request, \ - create_collector_id_request, create_collector_filter_request - -logging.basicConfig(level=logging.INFO) -LOGGER = logging.getLogger(__name__) - - -# def test_temp_DB(): -# temp_DB.main() - -def test_telemetry_object_creation(): - LOGGER.info('--- test_telemetry_object_creation: START') - - LOGGER.info('>>> Creating TelemetryDBmanager Object <<< ') - TelemetryDBmanagerObj = TelemetryDBmanager() - TelemetryEngine.create_database(TelemetryDBmanagerObj.db_engine) # creates 'frontend' db, if it doesnot exists. - - LOGGER.info('>>> Creating database <<< ') - TelemetryDBmanagerObj.create_database() - - LOGGER.info('>>> verifing database <<< ') - TelemetryDBmanagerObj.list_databases() - - # # LOGGER.info('>>> Droping Tables: ') - # # TelemetryDBmanagerObj.drop_table("table_naem_here") - - LOGGER.info('>>> Creating Tables <<< ') - TelemetryDBmanagerObj.create_tables() - - LOGGER.info('>>> Verifing Table creation <<< ') - TelemetryDBmanagerObj.verify_tables() - - # LOGGER.info('>>> TESTING: Row Insertion Operation: kpi Table <<<') - # kpi_obj = create_kpi_request() - # TelemetryDBmanagerObj.inser_kpi(kpi_obj) - - # LOGGER.info('>>> TESTING: Row Insertion Operation: collector Table <<<') - # collector_obj = create_collector_request() - # TelemetryDBmanagerObj.insert_collector(collector_obj) - - # LOGGER.info('>>> TESTING: Get KpiDescriptor <<<') - # kpi_id_obj = create_kpi_id_request() - # TelemetryDBmanagerObj.get_kpi_descriptor(kpi_id_obj) - - # LOGGER.info('>>> TESTING: Select Collector <<<') - # collector_id_obj = create_collector_id_request() - # TelemetryDBmanagerObj.get_collector(collector_id_obj) - - # LOGGER.info('>>> TESTING: Applying kpi filter <<< ') - # kpi_filter : dict[str, Any] = create_kpi_filter_request() - # TelemetryDBmanagerObj.select_kpi_descriptor(**kpi_filter) - - # LOGGER.info('>>> TESTING: Applying collector filter <<<') - # collector_filter : dict[str, Any] = create_collector_filter_request() - # TelemetryDBmanagerObj.select_collector(**collector_filter) - - # LOGGER.info('>>> TESTING: Delete KpiDescriptor ') - # kpi_id_obj = create_kpi_id_request() - # TelemetryDBmanagerObj.delete_kpi_descriptor(kpi_id_obj) - - # LOGGER.info('>>> TESTING: Delete Collector ') - # collector_id_obj = create_collector_id_request() - # TelemetryDBmanagerObj.delete_collector(collector_id_obj) - \ No newline at end of file diff --git a/src/telemetry/database/tests/temp_DB.py b/src/telemetry/database/tests/temp_DB.py deleted file mode 100644 index 089d35424..000000000 --- a/src/telemetry/database/tests/temp_DB.py +++ /dev/null @@ -1,327 +0,0 @@ -from sqlalchemy import create_engine, Column, String, Integer, Text, Float, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker, relationship -from sqlalchemy.dialects.postgresql import UUID -import logging - -LOGGER = logging.getLogger(__name__) -Base = declarative_base() - -class Kpi(Base): - __tablename__ = 'kpi' - - kpi_id = Column(UUID(as_uuid=False), primary_key=True) - kpi_description = Column(Text) - kpi_sample_type = Column(Integer) - device_id = Column(String) - endpoint_id = Column(String) - service_id = Column(String) - slice_id = Column(String) - connection_id = Column(String) - link_id = Column(String) - - collectors = relationship('Collector', back_populates='kpi') - - def __repr__(self): - return (f"") - -class Collector(Base): - __tablename__ = 'collector' - - collector_id = Column(UUID(as_uuid=False), primary_key=True) - kpi_id = Column(UUID(as_uuid=False), ForeignKey('kpi.kpi_id')) - collector = Column(String) - sampling_duration_s = Column(Float) - sampling_interval_s = Column(Float) - start_timestamp = Column(Float) - end_timestamp = Column(Float) - - kpi = relationship('Kpi', back_populates='collectors') - - def __repr__(self): - return (f"") - -class DatabaseManager: - def __init__(self, db_url, db_name): - self.engine = create_engine(db_url) - self.db_name = db_name - self.Session = sessionmaker(bind=self.engine) - LOGGER.info("DatabaseManager initialized with DB URL: %s and DB Name: %s", db_url, db_name) - - def create_database(self): - try: - with self.engine.connect() as connection: - connection.execute(f"CREATE DATABASE {self.db_name};") - LOGGER.info("Database '%s' created successfully.", self.db_name) - except Exception as e: - LOGGER.error("Error creating database '%s': %s", self.db_name, e) - finally: - LOGGER.info("create_database method execution finished.") - - def create_tables(self): - try: - Base.metadata.create_all(self.engine) - LOGGER.info("Tables created successfully.") - except Exception as e: - LOGGER.error("Error creating tables: %s", e) - finally: - LOGGER.info("create_tables method execution finished.") - - def verify_table_creation(self): - try: - with self.engine.connect() as connection: - result = connection.execute("SHOW TABLES;") - tables = result.fetchall() - LOGGER.info("Tables verified: %s", tables) - return tables - except Exception as e: - LOGGER.error("Error verifying table creation: %s", e) - return [] - finally: - LOGGER.info("verify_table_creation method execution finished.") - - def insert_row_kpi(self, kpi_data): - session = self.Session() - try: - new_kpi = Kpi(**kpi_data) - session.add(new_kpi) - session.commit() - LOGGER.info("Inserted row into KPI table: %s", kpi_data) - except Exception as e: - session.rollback() - LOGGER.error("Error inserting row into KPI table: %s", e) - finally: - session.close() - LOGGER.info("insert_row_kpi method execution finished.") - - def insert_row_collector(self, collector_data): - session = self.Session() - try: - new_collector = Collector(**collector_data) - session.add(new_collector) - session.commit() - LOGGER.info("Inserted row into Collector table: %s", collector_data) - except Exception as e: - session.rollback() - LOGGER.error("Error inserting row into Collector table: %s", e) - finally: - session.close() - LOGGER.info("insert_row_collector method execution finished.") - - def verify_insertion_kpi(self, kpi_id): - session = self.Session() - try: - kpi = session.query(Kpi).filter_by(kpi_id=kpi_id).first() - LOGGER.info("Verified insertion in KPI table for kpi_id: %s, Result: %s", kpi_id, kpi) - return kpi - except Exception as e: - LOGGER.error("Error verifying insertion in KPI table for kpi_id %s: %s", kpi_id, e) - return None - finally: - session.close() - LOGGER.info("verify_insertion_kpi method execution finished.") - - def verify_insertion_collector(self, collector_id): - session = self.Session() - try: - collector = session.query(Collector).filter_by(collector_id=collector_id).first() - LOGGER.info("Verified insertion in Collector table for collector_id: %s, Result: %s", collector_id, collector) - return collector - except Exception as e: - LOGGER.error("Error verifying insertion in Collector table for collector_id %s: %s", collector_id, e) - return None - finally: - session.close() - LOGGER.info("verify_insertion_collector method execution finished.") - - def get_all_kpi_rows(self): - session = self.Session() - try: - kpi_rows = session.query(Kpi).all() - LOGGER.info("Fetched all rows from KPI table: %s", kpi_rows) - return kpi_rows - except Exception as e: - LOGGER.error("Error fetching all rows from KPI table: %s", e) - return [] - finally: - session.close() - LOGGER.info("get_all_kpi_rows method execution finished.") - - def get_all_collector_rows(self): - session = self.Session() - try: - collector_rows = session.query(Collector).all() - LOGGER.info("Fetched all rows from Collector table: %s", collector_rows) - return collector_rows - except Exception as e: - LOGGER.error("Error fetching all rows from Collector table: %s", e) - return [] - finally: - session.close() - LOGGER.info("get_all_collector_rows method execution finished.") - - def get_filtered_kpi_rows(self, **filters): - session = self.Session() - try: - query = session.query(Kpi) - for column, value in filters.items(): - query = query.filter(getattr(Kpi, column) == value) - result = query.all() - LOGGER.info("Fetched filtered rows from KPI table with filters ---------- : {:s}".format(str(result))) - return result - except NoResultFound: - LOGGER.warning("No results found in KPI table with filters %s", filters) - return [] - except Exception as e: - LOGGER.error("Error fetching filtered rows from KPI table with filters %s: %s", filters, e) - return [] - finally: - session.close() - LOGGER.info("get_filtered_kpi_rows method execution finished.") - - def get_filtered_collector_rows(self, **filters): - session = self.Session() - try: - query = session.query(Collector) - for column, value in filters.items(): - query = query.filter(getattr(Collector, column) == value) - result = query.all() - LOGGER.info("Fetched filtered rows from Collector table with filters %s: %s", filters, result) - return result - except NoResultFound: - LOGGER.warning("No results found in Collector table with filters %s", filters) - return [] - except Exception as e: - LOGGER.error("Error fetching filtered rows from Collector table with filters %s: %s", filters, e) - return [] - finally: - session.close() - LOGGER.info("get_filtered_collector_rows method execution finished.") - - def delete_kpi_by_id(self, kpi_id): - session = self.Session() - try: - kpi = session.query(Kpi).filter_by(kpi_id=kpi_id).first() - if kpi: - session.delete(kpi) - session.commit() - LOGGER.info("Deleted KPI with kpi_id: %s", kpi_id) - else: - LOGGER.warning("KPI with kpi_id %s not found", kpi_id) - except SQLAlchemyError as e: - session.rollback() - LOGGER.error("Error deleting KPI with kpi_id %s: %s", kpi_id, e) - finally: - session.close() - LOGGER.info("delete_kpi_by_id method execution finished.") - - def delete_collector_by_id(self, collector_id): - session = self.Session() - try: - collector = session.query(Collector).filter_by(collector_id=collector_id).first() - if collector: - session.delete(collector) - session.commit() - LOGGER.info("Deleted Collector with collector_id: %s", collector_id) - else: - LOGGER.warning("Collector with collector_id %s not found", collector_id) - except SQLAlchemyError as e: - session.rollback() - LOGGER.error("Error deleting Collector with collector_id %s: %s", collector_id, e) - finally: - session.close() - LOGGER.info("delete_collector_by_id method execution finished.") - - -# Example Usage -def main(): - CRDB_SQL_PORT = "26257" - CRDB_DATABASE = "telemetryfrontend" - CRDB_USERNAME = "tfs" - CRDB_PASSWORD = "tfs123" - CRDB_SSLMODE = "require" - CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' - crdb_uri = CRDB_URI_TEMPLATE.format( - CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) - # db_url = "cockroachdb://username:password@localhost:26257/" - # db_name = "yourdatabase" - db_manager = DatabaseManager(crdb_uri, CRDB_DATABASE) - - # Create database - db_manager.create_database() - - # Update db_url to include the new database name - db_manager.engine = create_engine(f"{crdb_uri}") - db_manager.Session = sessionmaker(bind=db_manager.engine) - - # Create tables - db_manager.create_tables() - - # Verify table creation - tables = db_manager.verify_table_creation() - LOGGER.info('Tables in the database: {:s}'.format(str(tables))) - - # Insert a row into the KPI table - kpi_data = { - 'kpi_id': '123e4567-e89b-12d3-a456-426614174100', - 'kpi_description': 'Sample KPI', - 'kpi_sample_type': 1, - 'device_id': 'device_1', - 'endpoint_id': 'endpoint_1', - 'service_id': 'service_1', - 'slice_id': 'slice_1', - 'connection_id': 'conn_1', - 'link_id': 'link_1' - } - db_manager.insert_row_kpi(kpi_data) - - # Insert a row into the Collector table - collector_data = { - 'collector_id': '123e4567-e89b-12d3-a456-426614174101', - 'kpi_id': '123e4567-e89b-12d3-a456-426614174000', - 'collector': 'Collector 1', - 'sampling_duration_s': 60.0, - 'sampling_interval_s': 10.0, - 'start_timestamp': 1625247600.0, - 'end_timestamp': 1625247660.0 - } - db_manager.insert_row_collector(collector_data) - - # Verify insertion into KPI table - kpi = db_manager.verify_insertion_kpi('123e4567-e89b-12d3-a456-426614174000') - print("Inserted KPI:", kpi) - - # Verify insertion into Collector table - collector = db_manager.verify_insertion_collector('123e4567-e89b-12d3-a456-426614174001') - print("Inserted Collector:", collector) - - # Get all rows from KPI table - all_kpi_rows = db_manager.get_all_kpi_rows() - LOGGER.info("All KPI Rows: %s", all_kpi_rows) - - # Get all rows from Collector table - all_collector_rows = db_manager.get_all_collector_rows() - LOGGER.info("All Collector Rows: %s", all_collector_rows) - - # Get filtered rows from KPI table - filtered_kpi_rows = db_manager.get_filtered_kpi_rows(kpi_description='Sample KPI') - LOGGER.info("Filtered KPI Rows: %s", filtered_kpi_rows) - - # Get filtered rows from Collector table - filtered_collector_rows = db_manager.get_filtered_collector_rows(collector='Collector 1') - LOGGER.info("Filtered Collector Rows: %s", filtered_collector_rows) - - # Delete a KPI by kpi_id - kpi_id_to_delete = '123e4567-e89b-12d3-a456-426614174000' - db_manager.delete_kpi_by_id(kpi_id_to_delete) - - # Delete a Collector by collector_id - collector_id_to_delete = '123e4567-e89b-12d3-a456-426614174001' - db_manager.delete_collector_by_id(collector_id_to_delete) diff --git a/src/telemetry/database/tests/messages.py b/src/telemetry/tests/messages.py similarity index 100% rename from src/telemetry/database/tests/messages.py rename to src/telemetry/tests/messages.py diff --git a/src/telemetry/database/tests/managementDBtests.py b/src/telemetry/tests/test_telemetryDB.py similarity index 59% rename from src/telemetry/database/tests/managementDBtests.py rename to src/telemetry/tests/test_telemetryDB.py index 24138abe4..c4976f8c2 100644 --- a/src/telemetry/database/tests/managementDBtests.py +++ b/src/telemetry/tests/test_telemetryDB.py @@ -13,10 +13,16 @@ # limitations under the License. -from telemetry.database.managementDB import managementDB -from telemetry.database.tests.messages import create_collector_model_object +import logging +from telemetry.database.Telemetry_DB import TelemetryDB +LOGGER = logging.getLogger(__name__) -def test_add_row_to_db(): - managementDBobj = managementDB() - managementDBobj.add_row_to_db(create_collector_model_object()) \ No newline at end of file +def test_verify_databases_and_tables(): + LOGGER.info('>>> test_verify_databases_and_tables : START <<< ') + TelemetryDBobj = TelemetryDB() + TelemetryDBobj.drop_database() + TelemetryDBobj.verify_tables() + TelemetryDBobj.create_database() + TelemetryDBobj.create_tables() + TelemetryDBobj.verify_tables() \ No newline at end of file -- GitLab From 614e448337bcf51948bf6475d3c7e4821596b5dd Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Jul 2024 09:45:49 +0000 Subject: [PATCH 04/70] Foreced changes in KpiValueWriter to handle gRPC empty return message. --- src/kpi_value_writer/service/KpiValueWriter.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py index 26bab4465..022126fd0 100644 --- a/src/kpi_value_writer/service/KpiValueWriter.py +++ b/src/kpi_value_writer/service/KpiValueWriter.py @@ -33,7 +33,6 @@ from .MetricWriterToPrometheus import MetricWriterToPrometheus LOGGER = logging.getLogger(__name__) ACTIVE_CONSUMERS = [] -METRIC_WRITER = MetricWriterToPrometheus() class KpiValueWriter(GenericGrpcService): def __init__(self, cls_name : str = __name__) -> None: @@ -48,12 +47,14 @@ class KpiValueWriter(GenericGrpcService): @staticmethod def KafkaConsumer(): + kpi_manager_client = KpiManagerClient() + metric_writer = MetricWriterToPrometheus() + kafka_consumer = KafkaConsumer( { 'bootstrap.servers' : KafkaConfig.SERVER_IP.value, 'group.id' : __class__, 'auto.offset.reset' : 'latest'} - ) - kpi_manager_client = KpiManagerClient() + ) kafka_consumer.subscribe([KafkaTopic.VALUE.value]) LOGGER.debug("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) print("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) @@ -72,13 +73,13 @@ class KpiValueWriter(GenericGrpcService): kpi_value.ParseFromString(raw_kpi.value()) LOGGER.info("Received KPI : {:}".format(kpi_value)) print("Received KPI : {:}".format(kpi_value)) - KpiValueWriter.get_kpi_descriptor(kpi_value, kpi_manager_client) + KpiValueWriter.get_kpi_descriptor(kpi_value, kpi_manager_client, metric_writer) except Exception as e: print("Error detail: {:}".format(e)) continue @staticmethod - def get_kpi_descriptor(kpi_value: str, kpi_manager_client ): + def get_kpi_descriptor(kpi_value: str, kpi_manager_client, metric_writer): print("--- START -----") kpi_id = KpiId() @@ -89,12 +90,11 @@ class KpiValueWriter(GenericGrpcService): try: kpi_descriptor_object = KpiDescriptor() kpi_descriptor_object = kpi_manager_client.GetKpiDescriptor(kpi_id) + # TODO: why kpi_descriptor_object recevies a KpiDescriptor type object not Empty type object??? if kpi_descriptor_object.kpi_id.kpi_id.uuid == kpi_id.kpi_id.uuid: - # print("kpi descriptor received: {:}".format(kpi_descriptor_object)) - # if isinstance (kpi_descriptor_object, KpiDescriptor): LOGGER.info("Extracted KpiDescriptor: {:}".format(kpi_descriptor_object)) print("Extracted KpiDescriptor: {:}".format(kpi_descriptor_object)) - METRIC_WRITER.create_and_expose_cooked_kpi(kpi_descriptor_object, kpi_value) + metric_writer.create_and_expose_cooked_kpi(kpi_descriptor_object, kpi_value) else: LOGGER.info("No KPI Descriptor found in DB for Kpi ID: {:}".format(kpi_id)) print("No KPI Descriptor found in DB for Kpi ID: {:}".format(kpi_id)) -- GitLab From a66af97b0404aa5815f3cda0bf1d8cf9240018ae Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Jul 2024 09:55:51 +0000 Subject: [PATCH 05/70] updated imports to resolve error generated by unit test. - Imports are updated in test_kpi_value_writer.py --- src/kpi_value_writer/tests/test_kpi_value_writer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/kpi_value_writer/tests/test_kpi_value_writer.py b/src/kpi_value_writer/tests/test_kpi_value_writer.py index 572495d48..40593af97 100755 --- a/src/kpi_value_writer/tests/test_kpi_value_writer.py +++ b/src/kpi_value_writer/tests/test_kpi_value_writer.py @@ -14,11 +14,12 @@ import logging from kpi_value_writer.service.KpiValueWriter import KpiValueWriter +from kpi_value_writer.tests.test_messages import create_kpi_id_request, create_kpi_descriptor_request + from common.tools.kafka.Variables import KafkaTopic -from kpi_manager.client.KpiManagerClient import KpiManagerClient -from kpi_manager.tests.test_messages import create_kpi_descriptor_request from common.proto.kpi_manager_pb2 import KpiDescriptor -from kpi_value_writer.tests.test_messages import create_kpi_id_request +from kpi_manager.client.KpiManagerClient import KpiManagerClient + LOGGER = logging.getLogger(__name__) -- GitLab From befae34c31d9394884ad6858db8ef11e6f934dbd Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Jul 2024 14:14:18 +0000 Subject: [PATCH 06/70] Kafka deployment script is integrated with TFS deplyment script. - Kafka env variables are created in my_deply.sh, kafka.sh and all.sh --- deploy/all.sh | 3 ++ deploy/kafka.sh | 83 ++++++++++++++++++++++++++++++------------------- deploy/tfs.sh | 20 +++++++++++- 3 files changed, 73 insertions(+), 33 deletions(-) diff --git a/deploy/all.sh b/deploy/all.sh index f93cd92ac..e9b33b469 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -215,6 +215,9 @@ export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} # Deploy QuestDB ./deploy/qdb.sh +# Deploy Apache Kafka +./deploy/kafka.sh + # Expose Dashboard ./deploy/expose_dashboard.sh diff --git a/deploy/kafka.sh b/deploy/kafka.sh index 4a91bfc9e..b2f2f1f9e 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -20,50 +20,69 @@ # If not already set, set the namespace where Apache Kafka will be deployed. export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} +# If not already set, set the port Apache Kafka server will be exposed to. +export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"} + +# If not already set, if flag is YES, Apache Kafka will be redeployed and all topics will be lost. +export KFK_REDEPLOY=${KFK_REDEPLOY:-""} + ######################################################################################################################## # Automated steps start here ######################################################################################################################## -# Constants -TMP_FOLDER="./tmp" -KFK_MANIFESTS_PATH="manifests/kafka" -KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml" -KFK_MANIFEST="02-kafka.yaml" + # Constants + TMP_FOLDER="./tmp" + KFK_MANIFESTS_PATH="manifests/kafka" + KFK_ZOOKEEPER_MANIFEST="01-zookeeper.yaml" + KFK_MANIFEST="02-kafka.yaml" + + # Create a tmp folder for files modified during the deployment + TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" + mkdir -p ${TMP_MANIFESTS_FOLDER} -# Create a tmp folder for files modified during the deployment -TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${KFK_NAMESPACE}/manifests" -mkdir -p ${TMP_MANIFESTS_FOLDER} +function kafka_deploy() { + # copy zookeeper and kafka manifest files to temporary manifest location + cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" + cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" -# copy zookeeper and kafka manifest files to temporary manifest location -cp "${KFK_MANIFESTS_PATH}/${KFK_ZOOKEEPER_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" -cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}" + # echo "Apache Kafka Namespace" + echo ">>> Delete Apache Kafka Namespace" + kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found -echo "Apache Kafka Namespace" -echo ">>> Delete Apache Kafka Namespace" -kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found + echo ">>> Create Apache Kafka Namespace" + kubectl create namespace ${KFK_NAMESPACE} -echo ">>> Create Apache Kafka Namespace" -kubectl create namespace ${KFK_NAMESPACE} + # echo ">>> Deplying Apache Kafka Zookeeper" + # Kafka zookeeper service should be deployed before the kafka service + kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" -echo ">>> Deplying Apache Kafka Zookeeper" -# Kafka zookeeper service should be deployed before the kafka service -kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/${KFK_ZOOKEEPER_MANIFEST}" + KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically + KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') -KFK_ZOOKEEPER_SERVICE="zookeeper-service" # this command may be replaced with command to extract service name automatically -KFK_ZOOKEEPER_IP=$(kubectl --namespace ${KFK_NAMESPACE} get service ${KFK_ZOOKEEPER_SERVICE} -o 'jsonpath={.spec.clusterIP}') + # Kafka service should be deployed after the zookeeper service + sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" -# Kafka service should be deployed after the zookeeper service -sed -i "s//${KFK_ZOOKEEPER_IP}/" "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" + # echo ">>> Deploying Apache Kafka Broker" + kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" -echo ">>> Deploying Apache Kafka Broker" -kubectl --namespace ${KFK_NAMESPACE} apply -f "${TMP_MANIFESTS_FOLDER}/$KFK_MANIFEST" + # echo ">>> Verifing Apache Kafka deployment" + sleep 5 + # KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods) + # if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then + # echo "Deployment Error: \n $KFK_PODS_STATUS" + # else + # echo "$KFK_PODS_STATUS" + # fi +} -echo ">>> Verifing Apache Kafka deployment" -sleep 10 -KFK_PODS_STATUS=$(kubectl --namespace ${KFK_NAMESPACE} get pods) -if echo "$KFK_PODS_STATUS" | grep -qEv 'STATUS|Running'; then - echo "Deployment Error: \n $KFK_PODS_STATUS" +echo "Apache Kafka" +echo ">>> Checking if Apache Kafka is deployed ... " +if [ "$KFK_REDEPLOY" == "YES" ]; then + kafka_deploy +elif kubectl get --namespace ${KFK_NAMESPACE} deployments.apps &> /dev/null; then + echo ">>> Apache Kafka already present; skipping step..." else - echo "$KFK_PODS_STATUS" -fi \ No newline at end of file + kafka_deploy +fi +echo diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 62f36a2c1..d92d789e3 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -115,6 +115,17 @@ export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"} export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"} +# ----- Apache Kafka ------------------------------------------------------ + +# If not already set, set the namespace where Apache Kafka will be deployed. +export KFK_NAMESPACE=${KFK_NAMESPACE:-"kafka"} + +# If not already set, set the port Apache Kafka server will be exposed to. +export KFK_SERVER_PORT=${KFK_SERVER_PORT:-"9092"} + +# If not already set, if flag is YES, Apache Kafka will be redeployed and topic will be lost. +export KFK_REDEPLOY=${KFK_REDEPLOY:-""} + ######################################################################################################################## # Automated steps start here ######################################################################################################################## @@ -147,7 +158,7 @@ kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type= --from-literal=CRDB_SSLMODE=require printf "\n" -echo "Create secret with CockroachDB data for KPI Management" +echo "Create secret with CockroachDB data for KPI Management microservices" CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') CRDB_DATABASE_KPI_MGMT="tfs_kpi_mgmt" # TODO: change by specific configurable environment variable kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ @@ -159,6 +170,13 @@ kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --t --from-literal=CRDB_SSLMODE=require printf "\n" +echo "Create secret with Apache Kafka kfk-kpi-data for KPI and Telemetry microservices" +KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}') +kubectl create secret generic kfk-kpi-data --namespace ${KFK_NAMESPACE} --type='Opaque' \ + --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \ + --from-literal=KFK_SERVER_PORT=${KFK_NAMESPACE} +printf "\n" + echo "Create secret with NATS data" NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}') if [ -z "$NATS_CLIENT_PORT" ]; then -- GitLab From bde43f260a7c7abdc658bc77755cb4c78633d9a4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Jul 2024 14:43:57 +0000 Subject: [PATCH 07/70] Kafka secret added to kpi_value_api/kpi_value_writer - improvements to accuratly read the env variables --- deploy/kafka.sh | 2 +- deploy/tfs.sh | 4 ++-- manifests/kpi_value_apiservice.yaml | 3 +++ manifests/kpi_value_writerservice.yaml | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/deploy/kafka.sh b/deploy/kafka.sh index b2f2f1f9e..21ba89408 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -81,7 +81,7 @@ echo ">>> Checking if Apache Kafka is deployed ... " if [ "$KFK_REDEPLOY" == "YES" ]; then kafka_deploy elif kubectl get --namespace ${KFK_NAMESPACE} deployments.apps &> /dev/null; then - echo ">>> Apache Kafka already present; skipping step..." + echo ">>> Apache Kafka already present; skipping step." else kafka_deploy fi diff --git a/deploy/tfs.sh b/deploy/tfs.sh index d92d789e3..4ecfaae99 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -172,9 +172,9 @@ printf "\n" echo "Create secret with Apache Kafka kfk-kpi-data for KPI and Telemetry microservices" KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}') -kubectl create secret generic kfk-kpi-data --namespace ${KFK_NAMESPACE} --type='Opaque' \ +kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \ - --from-literal=KFK_SERVER_PORT=${KFK_NAMESPACE} + --from-literal=KFK_SERVER_PORT=${KFK_SERVER_PORT} printf "\n" echo "Create secret with NATS data" diff --git a/manifests/kpi_value_apiservice.yaml b/manifests/kpi_value_apiservice.yaml index 74eb90f67..e4dcb0054 100644 --- a/manifests/kpi_value_apiservice.yaml +++ b/manifests/kpi_value_apiservice.yaml @@ -39,6 +39,9 @@ spec: env: - name: LOG_LEVEL value: "INFO" + envFrom: + - secretRef: + name: kfk-kpi-data readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:30020"] diff --git a/manifests/kpi_value_writerservice.yaml b/manifests/kpi_value_writerservice.yaml index 8a8e44ec2..e21e36f48 100644 --- a/manifests/kpi_value_writerservice.yaml +++ b/manifests/kpi_value_writerservice.yaml @@ -39,6 +39,9 @@ spec: env: - name: LOG_LEVEL value: "INFO" + envFrom: + - secretRef: + name: kfk-kpi-data readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:30030"] -- GitLab From 5f8f4d5ec041bf8f1221881448eb5724743b252b Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Jul 2024 14:49:57 +0000 Subject: [PATCH 08/70] dynamically creates the kafka server address with env variables. - get the values of KAFKA_NAMESPACE and KFK_SERVER_PORT to create KAFKA server address. --- src/common/tools/kafka/Variables.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 24ae2cff7..89ac42f90 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -16,14 +16,18 @@ import logging from enum import Enum from confluent_kafka import KafkaException from confluent_kafka.admin import AdminClient, NewTopic +from common.Settings import get_setting LOGGER = logging.getLogger(__name__) +KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' class KafkaConfig(Enum): + KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = get_setting('KFK_SERVER_PORT') # SERVER_IP = "127.0.0.1:9092" - SERVER_IP = "kafka-service.kafka.svc.cluster.local:9092" - ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_IP}) + server_address = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) + ADMIN_CLIENT = AdminClient({'bootstrap.servers': server_address}) class KafkaTopic(Enum): REQUEST = 'topic_request' -- GitLab From daadb074bb3f003450d9a3a4de5cfe5ba2eed86b Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 30 Jul 2024 15:14:40 +0000 Subject: [PATCH 09/70] Some improvements to Kpi Manager test and messages file. - comment is added in Kpi DB file for future reference. --- src/kpi_manager/database/Kpi_DB.py | 3 +- src/kpi_manager/tests/test_kpi_db.py | 4 +- src/kpi_manager/tests/test_kpi_manager.py | 148 +++++----------------- 3 files changed, 34 insertions(+), 121 deletions(-) diff --git a/src/kpi_manager/database/Kpi_DB.py b/src/kpi_manager/database/Kpi_DB.py index 4b6064070..530abe457 100644 --- a/src/kpi_manager/database/Kpi_DB.py +++ b/src/kpi_manager/database/Kpi_DB.py @@ -34,14 +34,15 @@ class KpiDB: def create_database(self) -> None: if not sqlalchemy_utils.database_exists(self.db_engine.url): - LOGGER.debug("Database created. {:}".format(self.db_engine.url)) sqlalchemy_utils.create_database(self.db_engine.url) + LOGGER.debug("Database created. {:}".format(self.db_engine.url)) def drop_database(self) -> None: if sqlalchemy_utils.database_exists(self.db_engine.url): sqlalchemy_utils.drop_database(self.db_engine.url) def create_tables(self): + # TODO: use "get_tables(declatrative class obj)" method of "sqlalchemy_utils" to verify tables. try: KpiModel.metadata.create_all(self.db_engine) # type: ignore LOGGER.debug("Tables created in the DB Name: {:}".format(self.db_name)) diff --git a/src/kpi_manager/tests/test_kpi_db.py b/src/kpi_manager/tests/test_kpi_db.py index e961c12ba..d4a57f836 100644 --- a/src/kpi_manager/tests/test_kpi_db.py +++ b/src/kpi_manager/tests/test_kpi_db.py @@ -21,8 +21,8 @@ LOGGER = logging.getLogger(__name__) def test_verify_databases_and_Tables(): LOGGER.info('>>> test_verify_Tables : START <<< ') kpiDBobj = KpiDB() - kpiDBobj.drop_database() - kpiDBobj.verify_tables() + # kpiDBobj.drop_database() + # kpiDBobj.verify_tables() kpiDBobj.create_database() kpiDBobj.create_tables() kpiDBobj.verify_tables() diff --git a/src/kpi_manager/tests/test_kpi_manager.py b/src/kpi_manager/tests/test_kpi_manager.py index f0d9526d3..da149e3fe 100755 --- a/src/kpi_manager/tests/test_kpi_manager.py +++ b/src/kpi_manager/tests/test_kpi_manager.py @@ -17,7 +17,7 @@ import os, pytest import logging from typing import Union -#from common.proto.context_pb2 import Empty +from common.proto.context_pb2 import Empty from common.Constants import ServiceNameEnum from common.Settings import ( ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc) @@ -26,12 +26,6 @@ from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server from common.proto.kpi_manager_pb2 import KpiId, KpiDescriptor, KpiDescriptorFilter, KpiDescriptorList from common.tools.service.GenericGrpcService import GenericGrpcService -#from context.client.ContextClient import ContextClient - -# from device.service.driver_api.DriverFactory import DriverFactory -# from device.service.driver_api.DriverInstanceCache import DriverInstanceCache -# from device.service.DeviceService import DeviceService -# from device.client.DeviceClient import DeviceClient from kpi_manager.tests.test_messages import create_kpi_descriptor_request, create_kpi_filter_request, create_kpi_descriptor_request_a from kpi_manager.service.KpiManagerService import KpiManagerService @@ -39,12 +33,6 @@ from kpi_manager.client.KpiManagerClient import KpiManagerClient from kpi_manager.tests.test_messages import create_kpi_descriptor_request from kpi_manager.tests.test_messages import create_kpi_id_request - -#from monitoring.service.NameMapping import NameMapping - -#os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' -#from device.service.drivers import DRIVERS - ########################### # Tests Setup ########################### @@ -55,8 +43,6 @@ KPIMANAGER_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.KPIMANAGER) # t os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) os.environ[get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(KPIMANAGER_SERVICE_PORT) -# METRICSDB_HOSTNAME = os.environ.get('METRICSDB_HOSTNAME'){} - LOGGER = logging.getLogger(__name__) class MockContextService(GenericGrpcService): @@ -70,84 +56,10 @@ class MockContextService(GenericGrpcService): self.context_servicer = MockServicerImpl_Context() add_ContextServiceServicer_to_server(self.context_servicer, self.server) -# @pytest.fixture(scope='session') -# def context_service(): -# LOGGER.info('Initializing MockContextService...') -# _service = MockContextService(MOCKSERVICE_PORT) -# _service.start() - -# LOGGER.info('Yielding MockContextService...') -# yield _service - -# LOGGER.info('Terminating MockContextService...') -# _service.context_servicer.msg_broker.terminate() -# _service.stop() - -# LOGGER.info('Terminated MockContextService...') - -# @pytest.fixture(scope='session') -# def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument -# LOGGER.info('Initializing ContextClient...') -# _client = ContextClient() - -# LOGGER.info('Yielding ContextClient...') -# yield _client - -# LOGGER.info('Closing ContextClient...') -# _client.close() - -# LOGGER.info('Closed ContextClient...') - -# @pytest.fixture(scope='session') -# def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument -# LOGGER.info('Initializing DeviceService...') -# driver_factory = DriverFactory(DRIVERS) -# driver_instance_cache = DriverInstanceCache(driver_factory) -# _service = DeviceService(driver_instance_cache) -# _service.start() - -# # yield the server, when test finishes, execution will resume to stop it -# LOGGER.info('Yielding DeviceService...') -# yield _service - -# LOGGER.info('Terminating DeviceService...') -# _service.stop() - -# LOGGER.info('Terminated DeviceService...') - -# @pytest.fixture(scope='session') -# def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument -# LOGGER.info('Initializing DeviceClient...') -# _client = DeviceClient() - -# LOGGER.info('Yielding DeviceClient...') -# yield _client - -# LOGGER.info('Closing DeviceClient...') -# _client.close() - -# LOGGER.info('Closed DeviceClient...') - -# @pytest.fixture(scope='session') -# def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument -# LOGGER.info('Initializing DeviceClient...') -# _client = DeviceClient() - -# LOGGER.info('Yielding DeviceClient...') -# yield _client - -# LOGGER.info('Closing DeviceClient...') -# _client.close() - -# LOGGER.info('Closed DeviceClient...') - # This fixture will be requested by test cases and last during testing session @pytest.fixture(scope='session') def kpi_manager_service(): LOGGER.info('Initializing KpiManagerService...') - #name_mapping = NameMapping() - # _service = MonitoringService(name_mapping) - # _service = KpiManagerService(name_mapping) _service = KpiManagerService() _service.start() @@ -194,22 +106,22 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab ########################### # ---------- 3rd Iteration Tests ---------------- -# def test_SetKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") -# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) -# LOGGER.info("Response gRPC message object: {:}".format(response)) -# assert isinstance(response, KpiId) +def test_SetKpiDescriptor(kpi_manager_client): + LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") + response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) + LOGGER.info("Response gRPC message object: {:}".format(response)) + assert isinstance(response, KpiId) -# def test_DeleteKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") -# # adding KPI -# response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) -# # deleting KPI -# del_response = kpi_manager_client.DeleteKpiDescriptor(response_id) -# # select KPI -# kpi_manager_client.GetKpiDescriptor(response_id) -# LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) -# assert isinstance(del_response, Empty) +def test_DeleteKpiDescriptor(kpi_manager_client): + LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") + # adding KPI + response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) + # deleting KPI + del_response = kpi_manager_client.DeleteKpiDescriptor(response_id) + # select KPI + kpi_manager_client.GetKpiDescriptor(response_id) + LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) + assert isinstance(del_response, Empty) def test_GetKpiDescriptor(kpi_manager_client): LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") @@ -225,21 +137,21 @@ def test_GetKpiDescriptor(kpi_manager_client): assert isinstance(response, KpiDescriptor) -# def test_SelectKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") -# # adding KPI -# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) -# # select KPI(s) -# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) -# LOGGER.info("Response gRPC message object: {:}".format(response)) -# assert isinstance(response, KpiDescriptorList) +def test_SelectKpiDescriptor(kpi_manager_client): + LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") + # adding KPI + kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) + # select KPI(s) + response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) + LOGGER.info("Response gRPC message object: {:}".format(response)) + assert isinstance(response, KpiDescriptorList) -# def test_set_list_of_KPIs(kpi_manager_client): -# LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ") -# KPIs_TO_SEARCH = ["node_in_power_total", "node_in_current_total", "node_out_power_total"] -# # adding KPI -# for kpi in KPIs_TO_SEARCH: -# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi)) +def test_set_list_of_KPIs(kpi_manager_client): + LOGGER.debug(" >>> test_set_list_of_KPIs: START <<< ") + KPIs_TO_SEARCH = ["node_in_power_total", "node_in_current_total", "node_out_power_total"] + # adding KPI + for kpi in KPIs_TO_SEARCH: + kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi)) # ---------- 2nd Iteration Tests ----------------- -- GitLab From e418d0861f191593de51b1f79475d5fae900cd65 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 31 Jul 2024 12:30:39 +0000 Subject: [PATCH 10/70] Changes to resolve Kafka server error - KFK_SERVER_ADDRESS_TEMPLATE now defined inside the class KafkaConfig. - variable renamed to "SERVER_ADDRESS" from "server_address" --- src/common/tools/kafka/Variables.py | 13 +++++++------ .../service/KpiValueApiServiceServicerImpl.py | 2 +- src/kpi_value_writer/service/KpiValueWriter.py | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 89ac42f90..9abc32b3e 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -20,14 +20,14 @@ from common.Settings import get_setting LOGGER = logging.getLogger(__name__) -KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' class KafkaConfig(Enum): - KFK_NAMESPACE = get_setting('KFK_NAMESPACE') - KFK_PORT = get_setting('KFK_SERVER_PORT') - # SERVER_IP = "127.0.0.1:9092" - server_address = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) - ADMIN_CLIENT = AdminClient({'bootstrap.servers': server_address}) + KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' + KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = get_setting('KFK_SERVER_PORT') + # SERVER_ADDRESS = "127.0.0.1:9092" + SERVER_ADDRESS = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) + ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) class KafkaTopic(Enum): REQUEST = 'topic_request' @@ -42,6 +42,7 @@ class KafkaTopic(Enum): Method to create Kafka topics defined as class members """ all_topics = [member.value for member in KafkaTopic] + LOGGER.debug("Kafka server address is: {:} ".format(KafkaConfig.SERVER_ADDRESS.value)) if( KafkaTopic.create_new_topic_if_not_exists( all_topics )): LOGGER.debug("All topics are created sucsessfully") return True diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py index d27de54f3..1559457d7 100644 --- a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -38,7 +38,7 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): ) -> Empty: LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) producer_obj = KafkaProducer({ - 'bootstrap.servers' : KafkaConfig.SERVER_IP.value + 'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value }) for kpi_value in request.kpi_value_list: kpi_value_to_produce : Tuple [str, Any, Any] = ( diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py index 022126fd0..5e2b6babe 100644 --- a/src/kpi_value_writer/service/KpiValueWriter.py +++ b/src/kpi_value_writer/service/KpiValueWriter.py @@ -51,10 +51,10 @@ class KpiValueWriter(GenericGrpcService): metric_writer = MetricWriterToPrometheus() kafka_consumer = KafkaConsumer( - { 'bootstrap.servers' : KafkaConfig.SERVER_IP.value, + { 'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, 'group.id' : __class__, 'auto.offset.reset' : 'latest'} - ) + ) kafka_consumer.subscribe([KafkaTopic.VALUE.value]) LOGGER.debug("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) print("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) -- GitLab From 4dad45318c5640b6496918e06fc16ae8f4d4e8e5 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 1 Aug 2024 12:53:52 +0000 Subject: [PATCH 11/70] changes to manage Kafka enviornment variable efficiently - KFK_SERVER_PORT and KFK_REDOPLY added into my_deploy.sh file. - refines kafka env variables import --- my_deploy.sh | 5 +++++ src/common/tools/kafka/Variables.py | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/my_deploy.sh b/my_deploy.sh index b89df7481..45e0d1301 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -181,3 +181,8 @@ export GRAF_EXT_PORT_HTTP="3000" # Set the namespace where Apache Kafka will be deployed. export KFK_NAMESPACE="kafka" +# Set the port Apache Kafka server will be exposed to. +export KFK_SERVER_PORT="9092" + +# Set the flag to YES for redeploying of Apache Kafka +export KFK_REDEPLOY="" diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 9abc32b3e..e3ee2016a 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -23,11 +23,11 @@ LOGGER = logging.getLogger(__name__) class KafkaConfig(Enum): KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' - KFK_NAMESPACE = get_setting('KFK_NAMESPACE') - KFK_PORT = get_setting('KFK_SERVER_PORT') - # SERVER_ADDRESS = "127.0.0.1:9092" + KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = get_setting('KFK_SERVER_PORT') + # SERVER_ADDRESS = "127.0.0.1:9092" SERVER_ADDRESS = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) - ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) + ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) class KafkaTopic(Enum): REQUEST = 'topic_request' -- GitLab From 72c6e0e28edb051bdb34b0caa52f5a0eef86b683 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 2 Aug 2024 09:41:59 +0000 Subject: [PATCH 12/70] Improvement in SelectKpiValues method. - Added "GetKpiSampleType" method to extract KpiSampleType based on KpiId. - Added PromtheusConnect method to query Prometheus from prometheus_api_client library. - KpiManagerClient added in DockerFile - prometheus_api_client added in requirement file. --- src/kpi_value_api/Dockerfile | 2 + src/kpi_value_api/requirements.in | 1 + .../service/KpiValueApiServiceServicerImpl.py | 93 ++++++++++++------- .../service/MetricWriterToPrometheus.py | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/src/kpi_value_api/Dockerfile b/src/kpi_value_api/Dockerfile index 7dd8d307b..25b8da931 100644 --- a/src/kpi_value_api/Dockerfile +++ b/src/kpi_value_api/Dockerfile @@ -63,6 +63,8 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow COPY src/kpi_value_api/. kpi_value_api/ +COPY src/kpi_manager/__init__.py kpi_manager/__init__.py +COPY src/kpi_manager/client/. kpi_manager/client/ # Start the service ENTRYPOINT ["python", "-m", "kpi_value_api.service"] diff --git a/src/kpi_value_api/requirements.in b/src/kpi_value_api/requirements.in index 7e4694109..f5695906a 100644 --- a/src/kpi_value_api/requirements.in +++ b/src/kpi_value_api/requirements.in @@ -14,3 +14,4 @@ confluent-kafka==2.3.* requests==2.27.* +prometheus-api-client==0.5.3 \ No newline at end of file diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py index 1559457d7..b2ebecad0 100644 --- a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -12,18 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, grpc, requests +import logging, grpc from typing import Tuple, Any -from datetime import datetime from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from common.proto.context_pb2 import Empty +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.proto.kpi_manager_pb2 import KpiDescriptor, KpiId from common.proto.kpi_value_api_pb2_grpc import KpiValueAPIServiceServicer from common.proto.kpi_value_api_pb2 import KpiValueList, KpiValueFilter, KpiValue, KpiValueType from confluent_kafka import Producer as KafkaProducer +from prometheus_api_client import PrometheusConnect +from prometheus_api_client.utils import parse_datetime + +from kpi_manager.client.KpiManagerClient import KpiManagerClient LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('KpiValueAPI', 'NBIgRPC') @@ -63,40 +68,67 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): ) -> KpiValueList: LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) response = KpiValueList() - metrics = [kpi.kpi_id for kpi in request.kpi_id] - start_timestamps = [timestamp for timestamp in request.start_timestamp] - end_timestamps = [timestamp for timestamp in request.end_timestamp] - results = [] + + kpi_manager_client = KpiManagerClient() + prom_connect = PrometheusConnect(url=PROM_URL) - for start, end in zip(start_timestamps, end_timestamps): - start_str = datetime.fromtimestamp(start.seconds).isoformat() + "Z" - end_str = datetime.fromtimestamp(end.seconds).isoformat() + "Z" + metrics = [self.GetKpiSampleType(kpi, kpi_manager_client) for kpi in request.kpi_id] + start_timestamps = [parse_datetime(timestamp) for timestamp in request.start_timestamp] + end_timestamps = [parse_datetime(timestamp) for timestamp in request.end_timestamp] + prom_response = [] + for start_time, end_time in zip(start_timestamps, end_timestamps): for metric in metrics: - url = f'{PROM_URL}/api/v1/query_range' - params = { - 'query': metric, - 'start': start_str, - 'end' : end_str, - 'step' : '30s' # or any other step you need - } - response = requests.get(url, params=params) - if response.status_code == 200: - data = response.json() - for result in data['data']['result']: - for value in result['values']: - kpi_value = KpiValue( - kpi_id=metric, - timestamp=str(seconds=value[0]), - kpi_value_type=self._convert_value_to_kpi_value_type(value[1]) - ) - results.append(kpi_value) + # print(start_time, end_time, metric) + prom_response.append( + prom_connect.custom_query_range( + query = metric, # this is the metric name and label config + start_time = start_time, + end_time = end_time, + step = 30, # or any other step value (missing in gRPC Filter request) + ) + ) + + for single_resposne in prom_response: + # print ("{:}".format(single_resposne)) + for record in single_resposne: + # print("Record >>> kpi: {:} >>> time & values set: {:}".format(record['metric']['__name__'], record['values'])) + for value in record['values']: + # print("{:} - {:}".format(record['metric']['__name__'], value)) + kpi_value = KpiValue() + kpi_value.kpi_id.kpi_id = record['metric']['__name__'], + kpi_value.timestamp = value[0], + kpi_value.kpi_value_type = self.ConverValueToKpiValueType(value[1]) + response.kpi_value_list.append(kpi_value) + return response + + def GetKpiSampleType(self, kpi_value: str, kpi_manager_client): + print("--- START -----") - def _convert_value_to_kpi_value_type(self, value): + kpi_id = KpiId() + kpi_id.kpi_id.uuid = kpi_value.kpi_id.kpi_id.uuid + # print("KpiId generated: {:}".format(kpi_id)) + + try: + kpi_descriptor_object = KpiDescriptor() + kpi_descriptor_object = kpi_manager_client.GetKpiDescriptor(kpi_id) + # TODO: why kpi_descriptor_object recevies a KpiDescriptor type object not Empty type object??? + if kpi_descriptor_object.kpi_id.kpi_id.uuid == kpi_id.kpi_id.uuid: + LOGGER.info("Extracted KpiDescriptor: {:}".format(kpi_descriptor_object)) + print("Extracted KpiDescriptor: {:}".format(kpi_descriptor_object)) + return KpiSampleType.Name(kpi_descriptor_object.kpi_sample_type) # extract and return the name of KpiSampleType + else: + LOGGER.info("No KPI Descriptor found in DB for Kpi ID: {:}".format(kpi_id)) + print("No KPI Descriptor found in DB for Kpi ID: {:}".format(kpi_id)) + except Exception as e: + LOGGER.info("Unable to get KpiDescriptor. Error: {:}".format(e)) + print ("Unable to get KpiDescriptor. Error: {:}".format(e)) + + def ConverValueToKpiValueType(self, value): # Check if the value is an integer (int64) try: - int64_value = int(value) - return KpiValueType(int64Val=int64_value) + int_value = int(value) + return KpiValueType(int64Val=int_value) except ValueError: pass # Check if the value is a float @@ -112,7 +144,6 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): # If none of the above, treat it as a string return KpiValueType(stringVal=value) - def delivery_callback(self, err, msg): if err: LOGGER.debug('Message delivery failed: {:}'.format(err)) else: LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py index b68116478..40bffa06e 100644 --- a/src/kpi_value_writer/service/MetricWriterToPrometheus.py +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -93,4 +93,4 @@ class MetricWriterToPrometheus: print("Metric {:} is already registered. Skipping.".format(metric_name)) else: LOGGER.error("Error while pushing metric: {}".format(e)) - raise \ No newline at end of file + raise -- GitLab From 8c5c12866f06410d765516981e5cfb4fb3c49063 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 2 Aug 2024 09:58:58 +0000 Subject: [PATCH 13/70] Temporarly defines the static value of env variables to test the working of microservice. - KFK_NAMESPACE and KFK_PORT --- src/common/tools/kafka/Variables.py | 6 ++++-- src/kpi_value_writer/service/MetricWriterToPrometheus.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index e3ee2016a..168957a26 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -23,8 +23,10 @@ LOGGER = logging.getLogger(__name__) class KafkaConfig(Enum): KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' - KFK_NAMESPACE = get_setting('KFK_NAMESPACE') - KFK_PORT = get_setting('KFK_SERVER_PORT') + KFK_NAMESPACE = 'kafka' + # KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = '9092' + # KFK_PORT = get_setting('KFK_SERVER_PORT') # SERVER_ADDRESS = "127.0.0.1:9092" SERVER_ADDRESS = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py index 40bffa06e..81324b759 100644 --- a/src/kpi_value_writer/service/MetricWriterToPrometheus.py +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -94,3 +94,4 @@ class MetricWriterToPrometheus: else: LOGGER.error("Error while pushing metric: {}".format(e)) raise + -- GitLab From 05978993285c076c603dc7c8876e0a65557dbb74 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 2 Aug 2024 10:04:31 +0000 Subject: [PATCH 14/70] minor changes in code. - refine Kpi_DB.py methods. - improve description of messages. - imporve the text description. --- src/kpi_manager/database/Kpi_DB.py | 4 +--- src/kpi_value_api/tests/messages.py | 5 +++-- src/kpi_value_writer/service/MetricWriterToPrometheus.py | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/kpi_manager/database/Kpi_DB.py b/src/kpi_manager/database/Kpi_DB.py index 530abe457..49ad9c9b5 100644 --- a/src/kpi_manager/database/Kpi_DB.py +++ b/src/kpi_manager/database/Kpi_DB.py @@ -70,8 +70,7 @@ class KpiDB: session.rollback() if "psycopg2.errors.UniqueViolation" in str(e): LOGGER.error(f"Unique key voilation: {row.__class__.__name__} table. {str(e)}") - raise AlreadyExistsException(row.__class__.__name__, row, - extra_details=["Unique key voilation: {:}".format(e)] ) + raise AlreadyExistsException(row.__class__.__name__, row, extra_details=["Unique key voilation: {:}".format(e)] ) else: LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}") raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) @@ -90,7 +89,6 @@ class KpiDB: print("{:} ID not found, No matching row: {:}".format(model.__name__, id_to_search)) return None except Exception as e: - session.rollback() LOGGER.debug(f"Failed to retrieve {model.__name__} ID. {str(e)}") raise OperationFailedException ("search by column id", extra_details=["unable to search row {:}".format(e)]) finally: diff --git a/src/kpi_value_api/tests/messages.py b/src/kpi_value_api/tests/messages.py index c2a1cbb0b..d8ad14bd4 100644 --- a/src/kpi_value_api/tests/messages.py +++ b/src/kpi_value_api/tests/messages.py @@ -18,8 +18,9 @@ from common.proto.kpi_value_api_pb2 import KpiValue, KpiValueList def create_kpi_value_list(): _create_kpi_value_list = KpiValueList() - # To run this experiment sucessfully, already existing UUID in KPI DB in necessary. - # because the UUID is used to get the descriptor form KPI DB. + # To run this experiment sucessfully, add an existing UUID of a KPI Descriptor from the KPI DB. + # This UUID is used to get the descriptor form the KPI DB. If the Kpi ID does not exists, + # some part of the code won't execute. EXISTING_KPI_IDs = ["725ce3ad-ac67-4373-bd35-8cd9d6a86e09", str(uuid.uuid4()), str(uuid.uuid4())] diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py index 81324b759..f1d079783 100644 --- a/src/kpi_value_writer/service/MetricWriterToPrometheus.py +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -63,8 +63,8 @@ class MetricWriterToPrometheus: def create_and_expose_cooked_kpi(self, kpi_descriptor: KpiDescriptor, kpi_value: KpiValue): # merge both gRPC messages into single varible. cooked_kpi = self.merge_kpi_descriptor_and_kpi_value(kpi_descriptor, kpi_value) - tags_to_exclude = {'kpi_description', 'kpi_sample_type', 'kpi_value'} # extracted values will be used as metric tag - metric_tags = [tag for tag in cooked_kpi.keys() if tag not in tags_to_exclude] + tags_to_exclude = {'kpi_description', 'kpi_sample_type', 'kpi_value'} + metric_tags = [tag for tag in cooked_kpi.keys() if tag not in tags_to_exclude] # These values will be used as metric tags metric_name = cooked_kpi['kpi_sample_type'] try: if metric_name not in PROM_METRICS: # Only register the metric, when it doesn't exists -- GitLab From 108618942d60f463efddade3a13e502e3d1e8352 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 2 Aug 2024 14:07:29 +0000 Subject: [PATCH 15/70] Cleaning unit test and messages files. - unused imports and functions are removed --- src/kpi_manager/tests/test_kpi_manager.py | 66 ------------------- .../tests/test_kpi_value_writer.py | 23 +------ 2 files changed, 1 insertion(+), 88 deletions(-) diff --git a/src/kpi_manager/tests/test_kpi_manager.py b/src/kpi_manager/tests/test_kpi_manager.py index da149e3fe..219fdadee 100755 --- a/src/kpi_manager/tests/test_kpi_manager.py +++ b/src/kpi_manager/tests/test_kpi_manager.py @@ -93,13 +93,6 @@ def kpi_manager_client(kpi_manager_service : KpiManagerService): # pylint: disab # Prepare Environment, should be the first test ################################################## -# # ERROR on this test --- -# def test_prepare_environment( -# context_client : ContextClient, # pylint: disable=redefined-outer-name,unused-argument -# ): -# context_id = json_context_id(DEFAULT_CONTEXT_NAME) -# context_client.SetContext(Context(**json_context(DEFAULT_CONTEXT_NAME))) -# context_client.SetTopology(Topology(**json_topology(DEFAULT_TOPOLOGY_NAME, context_id=context_id))) ########################### # Tests Implementation of Kpi Manager @@ -152,62 +145,3 @@ def test_set_list_of_KPIs(kpi_manager_client): # adding KPI for kpi in KPIs_TO_SEARCH: kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(kpi)) - - -# ---------- 2nd Iteration Tests ----------------- -# def test_SetKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_SetKpiDescriptor: START <<< ") -# with open("kpi_manager/tests/KPI_configs.json", 'r') as file: -# data = json.load(file) -# _descriptors = data.get('KPIs', []) -# for _descritor_name in _descriptors: -# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a(_descritor_name)) -# LOGGER.info("Response gRPC message object: {:}".format(response)) -# assert isinstance(response, KpiId) - -# def test_GetKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") -# response = kpi_manager_client.GetKpiDescriptor(create_kpi_id_request()) -# LOGGER.info("Response gRPC message object: {:}".format(response)) -# assert isinstance(response, KpiDescriptor) - -# def test_DeleteKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_DeleteKpiDescriptor: START <<< ") -# response = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) -# del_response = kpi_manager_client.DeleteKpiDescriptor(response) -# kpi_manager_client.GetKpiDescriptor(response) -# LOGGER.info("Response of delete method gRPC message object: {:}".format(del_response)) -# assert isinstance(del_response, Empty) - -# def test_SelectKpiDescriptor(kpi_manager_client): -# LOGGER.info(" >>> test_SelectKpiDescriptor: START <<< ") -# kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request_a()) -# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request_a()) -# LOGGER.info("Response gRPC message object: {:}".format(response)) -# assert isinstance(response, KpiDescriptorList) - -# ------------- INITIAL TESTs ---------------- -# Test case that makes use of client fixture to test server's CreateKpi method -# def test_set_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name -# # make call to server -# LOGGER.warning('test_create_kpi requesting') -# for i in range(3): -# response = kpi_manager_client.SetKpiDescriptor(create_kpi_request(str(i+1))) -# LOGGER.debug(str(response)) -# assert isinstance(response, KpiId) - -# # Test case that makes use of client fixture to test server's DeleteKpi method -# def test_delete_kpi(kpi_manager_client): # pylint: disable=redefined-outer-name -# # make call to server -# LOGGER.warning('delete_kpi requesting') -# response = kpi_manager_client.SetKpiDescriptor(create_kpi_request('4')) -# response = kpi_manager_client.DeleteKpiDescriptor(response) -# LOGGER.debug(str(response)) -# assert isinstance(response, Empty) - -# # Test case that makes use of client fixture to test server's GetKpiDescriptor method -# def test_select_kpi_descriptor(kpi_manager_client): # pylint: disable=redefined-outer-name -# LOGGER.warning('test_selectkpidescritor begin') -# response = kpi_manager_client.SelectKpiDescriptor(create_kpi_filter_request()) -# LOGGER.debug(str(response)) -# assert isinstance(response, KpiDescriptorList) diff --git a/src/kpi_value_writer/tests/test_kpi_value_writer.py b/src/kpi_value_writer/tests/test_kpi_value_writer.py index 40593af97..fce043d7f 100755 --- a/src/kpi_value_writer/tests/test_kpi_value_writer.py +++ b/src/kpi_value_writer/tests/test_kpi_value_writer.py @@ -14,32 +14,12 @@ import logging from kpi_value_writer.service.KpiValueWriter import KpiValueWriter -from kpi_value_writer.tests.test_messages import create_kpi_id_request, create_kpi_descriptor_request from common.tools.kafka.Variables import KafkaTopic -from common.proto.kpi_manager_pb2 import KpiDescriptor -from kpi_manager.client.KpiManagerClient import KpiManagerClient -LOGGER = logging.getLogger(__name__) - -# def test_GetKpiDescriptor(): -# LOGGER.info(" >>> test_GetKpiDescriptor: START <<< ") -# kpi_manager_client = KpiManagerClient() -# # adding KPI -# LOGGER.info(" --->>> calling SetKpiDescriptor ") -# response_id = kpi_manager_client.SetKpiDescriptor(create_kpi_descriptor_request()) -# # get KPI -# LOGGER.info(" --->>> calling GetKpiDescriptor with response ID") -# response = kpi_manager_client.GetKpiDescriptor(response_id) -# LOGGER.info("Response gRPC message object: {:}".format(response)) - -# LOGGER.info(" --->>> calling GetKpiDescriptor with random ID") -# rand_response = kpi_manager_client.GetKpiDescriptor(create_kpi_id_request()) -# LOGGER.info("Response gRPC message object: {:}".format(rand_response)) -# LOGGER.info("\n------------------ TEST FINISHED ---------------------\n") -# assert isinstance(response, KpiDescriptor) +LOGGER = logging.getLogger(__name__) # -------- Initial Test ---------------- def test_validate_kafka_topics(): @@ -50,4 +30,3 @@ def test_validate_kafka_topics(): def test_KafkaConsumer(): LOGGER.debug(" --->>> test_kafka_consumer: START <<<--- ") KpiValueWriter.RunKafkaConsumer() - -- GitLab From 4cd64ed72284b7ac07dd20cdfcd97a6e4f6d0606 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 2 Aug 2024 14:09:52 +0000 Subject: [PATCH 16/70] Updated Promtheus URL - PROM_URL variable is updated with FQDN of Prometheus. --- src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py index b2ebecad0..5e7c3d139 100644 --- a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -32,7 +32,7 @@ from kpi_manager.client.KpiManagerClient import KpiManagerClient LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('KpiValueAPI', 'NBIgRPC') -PROM_URL = "http://localhost:9090" +PROM_URL = "http://prometheus-k8s.monitoring.svc.cluster.local:9090" # TODO: updated with the env variables class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): def __init__(self): @@ -79,7 +79,8 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): prom_response = [] for start_time, end_time in zip(start_timestamps, end_timestamps): for metric in metrics: - # print(start_time, end_time, metric) + print(start_time, end_time, metric) + LOGGER.debug(">>> Query: {:}".format(metric)) prom_response.append( prom_connect.custom_query_range( query = metric, # this is the metric name and label config -- GitLab From 37e2bfc82d82e47c89f0783cc748f1d404640e56 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 2 Aug 2024 15:15:03 +0000 Subject: [PATCH 17/70] changes in README files. - README files of kpi manager/value writer/api are updated to reflect new changes. --- src/kpi_manager/README.md | 27 +++++++++++---------------- src/kpi_value_api/README.md | 23 +++++++++++++++++++++++ src/kpi_value_writer/README.md | 32 ++++++++++---------------------- 3 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 src/kpi_value_api/README.md diff --git a/src/kpi_manager/README.md b/src/kpi_manager/README.md index c1feadcc4..6e9b56d93 100644 --- a/src/kpi_manager/README.md +++ b/src/kpi_manager/README.md @@ -1,29 +1,24 @@ # How to locally run and test KPI manager micro-service -## --- File links need to be updated. --- ### Pre-requisets -The following requirements should be fulfilled before the execuation of KPI management service. +Ensure the following requirements are met before executing the KPI management service: -1. Verify that [kpi_management.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/kpi_management.proto) file exists and grpcs file are generated sucessfully. -2. Virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/requirements.in) are installed sucessfully. -3. Verify the creation of required database and table. -[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/database/tests/KpiDBtests.py) python file enlist the functions to create tables and database and -[KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/service/database/KpiEngine.py) contains the DB string, update the string as per your deployment. +1. A virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_manager/requirements.in) sucessfully installed. +2. Verify the creation of required database and table. The +[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_manager/tests/test_kpi_db.py) python file lists the functions to create tables and the database. The +[KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_manager/database/KpiEngine.py) file contains the DB string. ### Messages format templates -["Messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_messages.py) python file enlist the basic gRPC messages format used during the testing. +The ["messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_manager/tests/test_messages.py) python file contains templates for creating gRPC messages. -### Test file -["KPI management test"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_kpi_manager.py) python file enlist different tests conducted during the experiment. +### Unit test file +The ["KPI manager test"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_manager/tests/test_kpi_manager.py) python file lists various tests conducted to validate functionality. ### Flow of execution (Kpi Maanager Service functions) 1. Call the `create_database()` and `create_tables()` functions from `Kpi_DB` class to create the required database and table if they don't exist. Call `verify_tables` to verify the existence of KPI table. -2. Call the gRPC method `SetKpiDescriptor(KpiDescriptor)->KpiId` to add the KpiDescriptor in `Kpi` DB. `KpiDescriptor` and `KpiId` are both pre-defined gRPC message types. +2. Call the gRPC method `SetKpiDescriptor(KpiDescriptor)->KpiId` to add the KpiDescriptor to the `Kpi` DB. `KpiDescriptor` and `KpiId` are both pre-defined gRPC message types. -3. Call `GetKpiDescriptor(KpiId)->KpiDescriptor` to read the `KpiDescriptor` from DB and `DeleteKpiDescriptor(KpiId)` to delete the `KpiDescriptor` from DB. +3. Call `GetKpiDescriptor(KpiId)->KpiDescriptor` to read the `KpiDescriptor` from the DB and `DeleteKpiDescriptor(KpiId)` to delete the `KpiDescriptor` from the DB. -4. Call `SelectKpiDescriptor(KpiDescriptorFilter)->KpiDescriptorList` to get all `KpiDescriptor` objects that matches the filter criteria. `KpiDescriptorFilter` and `KpiDescriptorList` are pre-defined gRPC message types. - -## For KPI composer and KPI writer -The functionalities of KPI composer and writer is heavily dependent upon Telemetery service. Therfore, these services has other pre-requsites that are mention [here](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/requirements.in). +4. Call `SelectKpiDescriptor(KpiDescriptorFilter)->KpiDescriptorList` to get all `KpiDescriptor` objects that matches filter criteria. `KpiDescriptorFilter` and `KpiDescriptorList` are pre-defined gRPC message types. diff --git a/src/kpi_value_api/README.md b/src/kpi_value_api/README.md new file mode 100644 index 000000000..70ba2c5e7 --- /dev/null +++ b/src/kpi_value_api/README.md @@ -0,0 +1,23 @@ +# How to locally run and test KPI Value API micro-service + +### Pre-requisets +Ensure the following requirements are met before executing the KPI Value API service. + +1. The KPI Manger service is running and Apache Kafka is running. + +2. A virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_value_api/requirements.in) file sucessfully installed. + +3. Call the ["create_all_topics()"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/common/tools/kafka/Variables.py) function to verify the existence of all required topics on kafka. + +### Messages format templates +The ["messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_value_api/tests/messages.py) python file contains templates for creating gRPC messages. + +### Unit test file +The ["KPI Value API test"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_value_api/tests/test_kpi_value_api.py) python file enlist various tests conducted to validate functionality. + +### Flow of execution (Kpi Maanager Service functions) +1. Call the `create_new_topic_if_not_exists()` method to create any new topics if needed. + +2. Call `StoreKpiValues(KpiValueList)` to produce `Kpi Value` on a Kafka Topic. (The `KpiValueWriter` microservice will consume and process the `Kpi Value`) + +3. Call `SelectKpiValues(KpiValueFilter) -> KpiValueList` to read metric from the Prometheus DB. diff --git a/src/kpi_value_writer/README.md b/src/kpi_value_writer/README.md index 72ba6e559..c45a0e395 100644 --- a/src/kpi_value_writer/README.md +++ b/src/kpi_value_writer/README.md @@ -1,29 +1,17 @@ -# How to locally run and test KPI manager micro-service +# How to locally run and test the KPI Value Writer micro-service -## --- File links need to be updated. --- ### Pre-requisets -The following requirements should be fulfilled before the execuation of KPI management service. +Ensure the following requirements are meet before executing the KPI Value Writer service> -1. Verify that [kpi_management.proto](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/proto/kpi_management.proto) file exists and grpcs file are generated sucessfully. -2. Virtual enviornment exist with all the required packages listed in ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/requirements.in) are installed sucessfully. -3. Verify the creation of required database and table. -[KPI DB test](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/database/tests/KpiDBtests.py) python file enlist the functions to create tables and database and -[KPI Engine](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/service/database/KpiEngine.py) contains the DB string, update the string as per your deployment. +1. The KPI Manger and KPI Value API services are running and Apache Kafka is running. -### Messages format templates -["Messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_messages.py) python file enlist the basic gRPC messages format used during the testing. - -### Test file -["KPI management test"](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/kpi_management/kpi_manager/tests/test_kpi_manager.py) python file enlist different tests conducted during the experiment. - -### Flow of execution (Kpi Maanager Service functions) -1. Call the `create_database()` and `create_tables()` functions from `Kpi_DB` class to create the required database and table if they don't exist. Call `verify_tables` to verify the existence of KPI table. +2. A Virtual enviornment exist with all the required packages listed in the ["requirements.in"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_value_writer/requirements.in) file installed sucessfully. -2. Call the gRPC method `SetKpiDescriptor(KpiDescriptor)->KpiId` to add the KpiDescriptor in `Kpi` DB. `KpiDescriptor` and `KpiId` are both pre-defined gRPC message types. - -3. Call `GetKpiDescriptor(KpiId)->KpiDescriptor` to read the `KpiDescriptor` from DB and `DeleteKpiDescriptor(KpiId)` to delete the `KpiDescriptor` from DB. +### Messages format templates +The ["messages"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_value_writer/tests/test_messages.py) python file contains the templates to create gRPC messages. -4. Call `SelectKpiDescriptor(KpiDescriptorFilter)->KpiDescriptorList` to get all `KpiDescriptor` objects that matches the filter criteria. `KpiDescriptorFilter` and `KpiDescriptorList` are pre-defined gRPC message types. +### Unit test file +The ["KPI Value API test"](https://labs.etsi.org/rep/tfs/controller/-/blob/develop/src/kpi_value_writer/tests/test_kpi_value_writer.py) python file enlist various tests conducted to validate functionality. -## For KPI composer and KPI writer -The functionalities of KPI composer and writer is heavily dependent upon Telemetery service. Therfore, these services has other pre-requsites that are mention [here](https://labs.etsi.org/rep/tfs/controller/-/blob/feat/71-cttc-separation-of-monitoring/src/telemetry/requirements.in). \ No newline at end of file +### Flow of execution +1. Call the `RunKafkaConsumer` method from the `KpiValueWriter` class to start consuming the `KPI Value` generated by the `KPI Value API` or `Telemetry`. For every valid `KPI Value` consumer from Kafka, it invokes the `PrometheusWriter` class to prepare and push the metric to the Promethues DB. -- GitLab From f4b9fa91d9de8ae3f89e89c2265686733cd8c00f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 6 Aug 2024 14:57:33 +0000 Subject: [PATCH 18/70] updated Kafka deployment script. - --- deploy/kafka.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/kafka.sh b/deploy/kafka.sh index 21ba89408..f86108011 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -78,11 +78,13 @@ function kafka_deploy() { echo "Apache Kafka" echo ">>> Checking if Apache Kafka is deployed ... " -if [ "$KFK_REDEPLOY" == "YES" ]; then +if [ "$KFK_REDEPLOY" = "YES" ]; then + echo ">>> Redeploying kafka namespace" kafka_deploy -elif kubectl get --namespace ${KFK_NAMESPACE} deployments.apps &> /dev/null; then - echo ">>> Apache Kafka already present; skipping step." +elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then + echo ">>> Apache Kafka already present; skipping step." else + echo ">>> Kafka namespace doesn't exists. Deploying kafka namespace" kafka_deploy fi echo -- GitLab From dcdceee0222acad9c2a909e1a681554262c4f7bf Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 6 Aug 2024 15:04:27 +0000 Subject: [PATCH 19/70] updated FrontendTelemetery.proto file. - added start_time and end_time in proto. --- proto/telemetry_frontend.proto | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/proto/telemetry_frontend.proto b/proto/telemetry_frontend.proto index dbc1e8bf6..614d10cf0 100644 --- a/proto/telemetry_frontend.proto +++ b/proto/telemetry_frontend.proto @@ -19,9 +19,9 @@ import "context.proto"; import "kpi_manager.proto"; service TelemetryFrontendService { - rpc StartCollector (Collector ) returns (CollectorId ) {} - rpc StopCollector (CollectorId ) returns (context.Empty) {} - rpc SelectCollectors(CollectorFilter) returns (CollectorList) {} + rpc StartCollector (Collector ) returns (CollectorId ) {} + rpc StopCollector (CollectorId ) returns (context.Empty) {} + rpc SelectCollectors (CollectorFilter) returns (CollectorList) {} } message CollectorId { @@ -29,10 +29,12 @@ message CollectorId { } message Collector { - CollectorId collector_id = 1; // The Collector ID - kpi_manager.KpiId kpi_id = 2; // The KPI Id to be associated to the collected samples - float duration_s = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely - float interval_s = 4; // Interval between collected samples + CollectorId collector_id = 1; // The Collector ID + kpi_manager.KpiId kpi_id = 2; // The KPI Id to be associated to the collected samples + float duration_s = 3; // Terminate data collection after duration[seconds]; duration==0 means indefinitely + float interval_s = 4; // Interval between collected samples + context.Timestamp start_time = 5; // Timestamp when Collector start execution + context.Timestamp end_time = 6; // Timestamp when Collector stop execution } message CollectorFilter { -- GitLab From 9e741088fe875b8574c3a772c111d8caf8d62f08 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 6 Aug 2024 15:10:17 +0000 Subject: [PATCH 20/70] changes in Telemetry Frontend service and client. - collector description is removed from TelemetryModel. - "ConvertCollectorToRow" is added in Telemetry Model class. - NameMapping is removed from service client and service. - TelemetryDB object name and import is updated with correct class name. - StartCollector is restructured. - "PublishRequestOnKafka" is restructured. --- src/telemetry/database/TelemetryModel.py | 29 +++-- .../service/TelemetryFrontendService.py | 5 +- .../TelemetryFrontendServiceServicerImpl.py | 107 +++++++----------- src/telemetry/frontend/tests/Messages.py | 3 +- 4 files changed, 66 insertions(+), 78 deletions(-) diff --git a/src/telemetry/database/TelemetryModel.py b/src/telemetry/database/TelemetryModel.py index 95f692e4b..1faf16e1a 100644 --- a/src/telemetry/database/TelemetryModel.py +++ b/src/telemetry/database/TelemetryModel.py @@ -28,17 +28,32 @@ class Collector(Base): collector_id = Column(UUID(as_uuid=False), primary_key=True) kpi_id = Column(UUID(as_uuid=False), nullable=False) - collector_decription = Column(String , nullable=False) sampling_duration_s = Column(Float , nullable=False) sampling_interval_s = Column(Float , nullable=False) - start_timestamp = Column(Float , nullable=False) - end_timestamp = Column(Float , nullable=False) + start_timestamp = Column(String , nullable=False) + end_timestamp = Column(String , nullable=False) # helps in logging the information def __repr__(self): - return (f"") + return (f"") + + @classmethod + def ConvertCollectorToRow(cls, request): + """ + Create an instance of collector rows from a request object. + Args: request: The request object containing collector gRPC message. + Returns: A row (an instance of Collector table) initialized with content of the request. + """ + return cls( + collector_id = request.collector_id.collector_id.uuid, + kpi_id = request.kpi_id.kpi_id.uuid, + sampling_duration_s = request.duration_s, + sampling_interval_s = request.interval_s, + start_timestamp = request.start_time.timestamp, + end_timestamp = request.end_time.timestamp + ) # add method to convert gRPC requests to rows if necessary... + diff --git a/src/telemetry/frontend/service/TelemetryFrontendService.py b/src/telemetry/frontend/service/TelemetryFrontendService.py index dc3f8df36..abd361aa0 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendService.py +++ b/src/telemetry/frontend/service/TelemetryFrontendService.py @@ -14,17 +14,16 @@ from common.Constants import ServiceNameEnum from common.Settings import get_service_port_grpc -from monitoring.service.NameMapping import NameMapping from common.tools.service.GenericGrpcService import GenericGrpcService from common.proto.telemetry_frontend_pb2_grpc import add_TelemetryFrontendServiceServicer_to_server from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl class TelemetryFrontendService(GenericGrpcService): - def __init__(self, name_mapping : NameMapping, cls_name: str = __name__) -> None: + def __init__(self, cls_name: str = __name__) -> None: port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) super().__init__(port, cls_name=cls_name) - self.telemetry_frontend_servicer = TelemetryFrontendServiceServicerImpl(name_mapping) + self.telemetry_frontend_servicer = TelemetryFrontendServiceServicerImpl() def install_servicers(self): add_TelemetryFrontendServiceServicer_to_server(self.telemetry_frontend_servicer, self.server) diff --git a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py index e6830ad67..49641aae1 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -14,100 +14,74 @@ import ast import threading -import time from typing import Tuple, Any import grpc import logging +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from confluent_kafka import Consumer as KafkaConsumer -from common.proto.context_pb2 import Empty -from monitoring.service.NameMapping import NameMapping from confluent_kafka import Producer as KafkaProducer -from confluent_kafka import KafkaException from confluent_kafka import KafkaError + +from common.proto.context_pb2 import Empty + from common.proto.telemetry_frontend_pb2 import CollectorId, Collector, CollectorFilter, CollectorList -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceServicer - from telemetry.database.TelemetryModel import Collector as CollectorModel -from telemetry.database.managementDB import managementDB +from telemetry.database.Telemetry_DB import TelemetryDB + LOGGER = logging.getLogger(__name__) -METRICS_POOL = MetricsPool('Monitoring', 'TelemetryFrontend') -KAFKA_SERVER_IP = '127.0.0.1:9092' +METRICS_POOL = MetricsPool('TelemetryFrontend', 'NBIgRPC') ACTIVE_COLLECTORS = [] -KAFKA_TOPICS = {'request' : 'topic_request', - 'response': 'topic_response'} class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): - def __init__(self, name_mapping : NameMapping): + def __init__(self): LOGGER.info('Init TelemetryFrontendService') - self.managementDBobj = managementDB() - self.kafka_producer = KafkaProducer({'bootstrap.servers': KAFKA_SERVER_IP,}) - self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KAFKA_SERVER_IP, - 'group.id' : 'frontend', - 'auto.offset.reset' : 'latest'}) + self.DBobj = TelemetryDB() + self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value}) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, + 'group.id' : 'frontend', + 'auto.offset.reset' : 'latest'}) - def add_collector_to_db(self, request: Collector ): # type: ignore - try: - # Create a new Collector instance - collector_to_insert = CollectorModel() - collector_to_insert.collector_id = request.collector_id.collector_id.uuid - collector_to_insert.kpi_id = request.kpi_id.kpi_id.uuid - # collector_to_insert.collector_decription= request.collector - collector_to_insert.sampling_duration_s = request.duration_s - collector_to_insert.sampling_interval_s = request.interval_s - collector_to_insert.start_timestamp = time.time() - collector_to_insert.end_timestamp = time.time() - managementDB.add_row_to_db(collector_to_insert) - except Exception as e: - LOGGER.info("Unable to create collectorModel class object. {:}".format(e)) - - # @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StartCollector(self, request : Collector, grpc_context: grpc.ServicerContext # type: ignore ) -> CollectorId: # type: ignore - # push info to frontend db LOGGER.info ("gRPC message: {:}".format(request)) response = CollectorId() - _collector_id = str(request.collector_id.collector_id.uuid) - _collector_kpi_id = str(request.kpi_id.kpi_id.uuid) - _collector_duration = int(request.duration_s) - _collector_interval = int(request.interval_s) - # pushing Collector to DB - self.add_collector_to_db(request) - self.publish_to_kafka_request_topic(_collector_id, _collector_kpi_id, _collector_duration, _collector_interval) - # self.run_publish_to_kafka_request_topic(_collector_id, _collector_kpi_id, _collector_duration, _collector_interval) + + # TODO: Verify the presence of Kpi ID in KpiDB or assume that KPI ID already exists. + self.DBobj.add_row_to_db( + CollectorModel.ConvertCollectorToRow(request) + ) + self.PublishRequestOnKafka(request) + response.collector_id.uuid = request.collector_id.collector_id.uuid # type: ignore return response - - def run_publish_to_kafka_request_topic(self, msg_key: str, kpi: str, duration : int, interval: int): - # Add threading.Thread() response to dictonary and call start() in the next statement - threading.Thread(target=self.publish_to_kafka_request_topic, args=(msg_key, kpi, duration, interval)).start() - - def publish_to_kafka_request_topic(self, - collector_id: str, kpi: str, duration : int, interval: int - ): + + def PublishRequestOnKafka(self, collector_obj): """ - Method to generate collector request to Kafka topic. + Method to generate collector request on Kafka. """ - # time.sleep(5) - # producer_configs = { - # 'bootstrap.servers': KAFKA_SERVER_IP, - # } - # topic_request = "topic_request" - msg_value : Tuple [str, int, int] = (kpi, duration, interval) - # print ("Request generated: ", "Colletcor Id: ", collector_id, \ - # ", \nKPI: ", kpi, ", Duration: ", duration, ", Interval: ", interval) - # producerObj = KafkaProducer(producer_configs) - self.kafka_producer.produce(KAFKA_TOPICS['request'], key=collector_id, value= str(msg_value), callback=self.delivery_callback) - # producerObj.produce(KAFKA_TOPICS['request'], key=collector_id, value= str(msg_value), callback=self.delivery_callback) - LOGGER.info("Collector Request Generated: {:}, {:}, {:}, {:}".format(collector_id, kpi, duration, interval)) - # producerObj.produce(topic_request, key=collector_id, value= str(msg_value), callback=self.delivery_callback) + collector_id = collector_obj.collector_id.collector_id.uuid + collector_to_generate : Tuple [str, int, int] = ( + collector_obj.kpi_id.kpi_id.uuid, + collector_obj.duration_s, + collector_obj.interval_s + ) + self.kafka_producer.produce( + KafkaTopic.REQUEST.value, + key = collector_id, + value = str(collector_to_generate), + callback = self.delivery_callback + ) + LOGGER.info("Collector Request Generated: Collector Id: {:}, Value: {:}".format(collector_id, collector_to_generate)) ACTIVE_COLLECTORS.append(collector_id) self.kafka_producer.flush() - + def run_kafka_listener(self): # print ("--- STARTED: run_kafka_listener ---") threading.Thread(target=self.kafka_listener).start() @@ -201,4 +175,5 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): response.collector_list.append(collector_obj) return response except Exception as e: - LOGGER.info('Unable to process response {:}'.format(e)) \ No newline at end of file + LOGGER.info('Unable to process response {:}'.format(e)) + diff --git a/src/telemetry/frontend/tests/Messages.py b/src/telemetry/frontend/tests/Messages.py index 1205898d1..106c2a5a7 100644 --- a/src/telemetry/frontend/tests/Messages.py +++ b/src/telemetry/frontend/tests/Messages.py @@ -17,7 +17,6 @@ import random from common.proto import telemetry_frontend_pb2 from common.proto.kpi_sample_types_pb2 import KpiSampleType - # ----------------------- "2nd" Iteration -------------------------------- def create_collector_id(): _collector_id = telemetry_frontend_pb2.CollectorId() @@ -32,7 +31,7 @@ def create_collector_id(): def create_collector_request(): _create_collector_request = telemetry_frontend_pb2.Collector() _create_collector_request.collector_id.collector_id.uuid = str(uuid.uuid4()) - _create_collector_request.kpi_id.kpi_id.uuid = "165d20c5-a446-42fa-812f-e2b7ed283c6f" + _create_collector_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) # _create_collector_request.collector = "collector description" _create_collector_request.duration_s = float(random.randint(8, 16)) _create_collector_request.interval_s = float(random.randint(2, 4)) -- GitLab From 97655a60341f791b221bc869417ad9c5e5cfc683 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 6 Aug 2024 15:12:42 +0000 Subject: [PATCH 21/70] Telemetry frontend test file updated. - pytest logging is changed to DEBUG. - ManagemetDB test is removed. - Irrelevent imports and methods are removed from the test file. - --- .../run_tests_locally-telemetry-frontend.sh | 2 +- scripts/run_tests_locally-telemetry-mgtDB.sh | 26 --- src/telemetry/frontend/tests/test_frontend.py | 169 +++++------------- 3 files changed, 45 insertions(+), 152 deletions(-) delete mode 100755 scripts/run_tests_locally-telemetry-mgtDB.sh diff --git a/scripts/run_tests_locally-telemetry-frontend.sh b/scripts/run_tests_locally-telemetry-frontend.sh index 7652ccb58..a2a1de523 100755 --- a/scripts/run_tests_locally-telemetry-frontend.sh +++ b/scripts/run_tests_locally-telemetry-frontend.sh @@ -24,5 +24,5 @@ cd $PROJECTDIR/src # python3 kpi_manager/tests/test_unitary.py RCFILE=$PROJECTDIR/coverage/.coveragerc -python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \ +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ telemetry/frontend/tests/test_frontend.py diff --git a/scripts/run_tests_locally-telemetry-mgtDB.sh b/scripts/run_tests_locally-telemetry-mgtDB.sh deleted file mode 100755 index 8b68104ea..000000000 --- a/scripts/run_tests_locally-telemetry-mgtDB.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -PROJECTDIR=`pwd` - -cd $PROJECTDIR/src -# RCFILE=$PROJECTDIR/coverage/.coveragerc -# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ -# kpi_manager/tests/test_unitary.py - -RCFILE=$PROJECTDIR/coverage/.coveragerc -python3 -m pytest --log-cli-level=INFO --verbose \ - telemetry/database/tests/managementDBtests.py diff --git a/src/telemetry/frontend/tests/test_frontend.py b/src/telemetry/frontend/tests/test_frontend.py index 002cc4307..ca2d61370 100644 --- a/src/telemetry/frontend/tests/test_frontend.py +++ b/src/telemetry/frontend/tests/test_frontend.py @@ -13,129 +13,39 @@ # limitations under the License. import os -import time import pytest import logging -from typing import Union -from common.proto.context_pb2 import Empty +# from common.proto.context_pb2 import Empty from common.Constants import ServiceNameEnum from common.proto.telemetry_frontend_pb2 import CollectorId, CollectorList -from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server -from context.client.ContextClient import ContextClient -from common.tools.service.GenericGrpcService import GenericGrpcService -from common.tests.MockServicerImpl_Context import MockServicerImpl_Context + from common.Settings import ( get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendClient from telemetry.frontend.service.TelemetryFrontendService import TelemetryFrontendService -from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl -from telemetry.frontend.tests.Messages import ( create_collector_request, create_collector_filter) -from telemetry.database.managementDB import managementDB -from telemetry.database.TelemetryEngine import TelemetryEngine - -from device.client.DeviceClient import DeviceClient -from device.service.DeviceService import DeviceService -from device.service.driver_api.DriverFactory import DriverFactory -from device.service.driver_api.DriverInstanceCache import DriverInstanceCache - -from monitoring.service.NameMapping import NameMapping +from telemetry.frontend.tests.Messages import ( + create_collector_request, create_collector_filter) -os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE' -from device.service.drivers import DRIVERS ########################### # Tests Setup ########################### LOCAL_HOST = '127.0.0.1' -MOCKSERVICE_PORT = 10000 -TELEMETRY_FRONTEND_PORT = str(MOCKSERVICE_PORT) + str(get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND)) +TELEMETRY_FRONTEND_PORT = str(get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND)) os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(TELEMETRY_FRONTEND_PORT) LOGGER = logging.getLogger(__name__) -class MockContextService(GenericGrpcService): - # Mock Service implementing Context to simplify unitary tests of Monitoring - - def __init__(self, bind_port: Union[str, int]) -> None: - super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') - - # pylint: disable=attribute-defined-outside-init - def install_servicers(self): - self.context_servicer = MockServicerImpl_Context() - add_ContextServiceServicer_to_server(self.context_servicer, self.server) - -@pytest.fixture(scope='session') -def context_service(): - LOGGER.info('Initializing MockContextService...') - _service = MockContextService(MOCKSERVICE_PORT) - _service.start() - - LOGGER.info('Yielding MockContextService...') - yield _service - - LOGGER.info('Terminating MockContextService...') - _service.context_servicer.msg_broker.terminate() - _service.stop() - - LOGGER.info('Terminated MockContextService...') - -@pytest.fixture(scope='session') -def context_client(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument - LOGGER.info('Initializing ContextClient...') - _client = ContextClient() - - LOGGER.info('Yielding ContextClient...') - yield _client - - LOGGER.info('Closing ContextClient...') - _client.close() - - LOGGER.info('Closed ContextClient...') - -@pytest.fixture(scope='session') -def device_service(context_service : MockContextService): # pylint: disable=redefined-outer-name,unused-argument - LOGGER.info('Initializing DeviceService...') - driver_factory = DriverFactory(DRIVERS) - driver_instance_cache = DriverInstanceCache(driver_factory) - _service = DeviceService(driver_instance_cache) - _service.start() - - # yield the server, when test finishes, execution will resume to stop it - LOGGER.info('Yielding DeviceService...') - yield _service - - LOGGER.info('Terminating DeviceService...') - _service.stop() - - LOGGER.info('Terminated DeviceService...') - @pytest.fixture(scope='session') -def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument - LOGGER.info('Initializing DeviceClient...') - _client = DeviceClient() - - LOGGER.info('Yielding DeviceClient...') - yield _client - - LOGGER.info('Closing DeviceClient...') - _client.close() - - LOGGER.info('Closed DeviceClient...') - -@pytest.fixture(scope='session') -def telemetryFrontend_service( - context_service : MockContextService, - device_service : DeviceService - ): +def telemetryFrontend_service(): LOGGER.info('Initializing TelemetryFrontendService...') - name_mapping = NameMapping() - _service = TelemetryFrontendService(name_mapping) + _service = TelemetryFrontendService() _service.start() # yield the server, when test finishes, execution will resume to stop it @@ -168,37 +78,46 @@ def telemetryFrontend_client( # Tests Implementation of Telemetry Frontend ########################### -def test_verify_db_and_table(): - LOGGER.info(' >>> test_verify_database_and_tables START: <<< ') - _engine = TelemetryEngine.get_engine() - managementDB.create_database(_engine) - managementDB.create_tables(_engine) - +# ------- Re-structuring Test --------- def test_StartCollector(telemetryFrontend_client): LOGGER.info(' >>> test_StartCollector START: <<< ') response = telemetryFrontend_client.StartCollector(create_collector_request()) LOGGER.debug(str(response)) assert isinstance(response, CollectorId) -def test_run_kafka_listener(): - LOGGER.info(' >>> test_run_kafka_listener START: <<< ') - name_mapping = NameMapping() - TelemetryFrontendServiceObj = TelemetryFrontendServiceServicerImpl(name_mapping) - response = TelemetryFrontendServiceObj.run_kafka_listener() # Method "run_kafka_listener" is not define in frontend.proto - LOGGER.debug(str(response)) - assert isinstance(response, bool) - -def test_StopCollector(telemetryFrontend_client): - LOGGER.info(' >>> test_StopCollector START: <<< ') - _collector_id = telemetryFrontend_client.StartCollector(create_collector_request()) - time.sleep(3) # wait for small amount before call the stopCollecter() - response = telemetryFrontend_client.StopCollector(_collector_id) - LOGGER.debug(str(response)) - assert isinstance(response, Empty) - -def test_select_collectors(telemetryFrontend_client): - LOGGER.info(' >>> test_select_collector requesting <<< ') - response = telemetryFrontend_client.SelectCollectors(create_collector_filter()) - LOGGER.info('Received Rows after applying Filter: {:} '.format(response)) - LOGGER.debug(str(response)) - assert isinstance(response, CollectorList) \ No newline at end of file +# ------- previous test ---------------- + +# def test_verify_db_and_table(): +# LOGGER.info(' >>> test_verify_database_and_tables START: <<< ') +# _engine = TelemetryEngine.get_engine() +# managementDB.create_database(_engine) +# managementDB.create_tables(_engine) + +# def test_StartCollector(telemetryFrontend_client): +# LOGGER.info(' >>> test_StartCollector START: <<< ') +# response = telemetryFrontend_client.StartCollector(create_collector_request()) +# LOGGER.debug(str(response)) +# assert isinstance(response, CollectorId) + +# def test_run_kafka_listener(): +# LOGGER.info(' >>> test_run_kafka_listener START: <<< ') +# name_mapping = NameMapping() +# TelemetryFrontendServiceObj = TelemetryFrontendServiceServicerImpl(name_mapping) +# response = TelemetryFrontendServiceObj.run_kafka_listener() # Method "run_kafka_listener" is not define in frontend.proto +# LOGGER.debug(str(response)) +# assert isinstance(response, bool) + +# def test_StopCollector(telemetryFrontend_client): +# LOGGER.info(' >>> test_StopCollector START: <<< ') +# _collector_id = telemetryFrontend_client.StartCollector(create_collector_request()) +# time.sleep(3) # wait for small amount before call the stopCollecter() +# response = telemetryFrontend_client.StopCollector(_collector_id) +# LOGGER.debug(str(response)) +# assert isinstance(response, Empty) + +# def test_select_collectors(telemetryFrontend_client): +# LOGGER.info(' >>> test_select_collector requesting <<< ') +# response = telemetryFrontend_client.SelectCollectors(create_collector_filter()) +# LOGGER.info('Received Rows after applying Filter: {:} '.format(response)) +# LOGGER.debug(str(response)) +# assert isinstance(response, CollectorList) \ No newline at end of file -- GitLab From 9605c9a10a2c80682f146c897e7118b4cd3da499 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 8 Aug 2024 09:32:37 +0000 Subject: [PATCH 22/70] Telemetry Frontend gRPC NBI Re-structuring - Changed the column type of the start and end timestamps to Float. - Added ConvertRowToCollector() in TelemetryModel. - Renamed the class variable from "DBobj" to "tele_db_obj". - Renamed the local variable from "collector_id" to "collector_uuid". - Added PublishStopRequestOnKafka() to publish the stop collector request on Kafka. - Improved the test files. --- src/telemetry/database/TelemetryModel.py | 24 +++++-- src/telemetry/database/Telemetry_DB.py | 6 +- .../TelemetryFrontendServiceServicerImpl.py | 64 +++++++++++++------ src/telemetry/frontend/tests/Messages.py | 54 ++-------------- src/telemetry/frontend/tests/test_frontend.py | 15 ++++- 5 files changed, 86 insertions(+), 77 deletions(-) diff --git a/src/telemetry/database/TelemetryModel.py b/src/telemetry/database/TelemetryModel.py index 1faf16e1a..611ce7e70 100644 --- a/src/telemetry/database/TelemetryModel.py +++ b/src/telemetry/database/TelemetryModel.py @@ -16,6 +16,7 @@ import logging from sqlalchemy.dialects.postgresql import UUID from sqlalchemy import Column, String, Float from sqlalchemy.orm import registry +from common.proto import telemetry_frontend_pb2 logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) @@ -30,8 +31,8 @@ class Collector(Base): kpi_id = Column(UUID(as_uuid=False), nullable=False) sampling_duration_s = Column(Float , nullable=False) sampling_interval_s = Column(Float , nullable=False) - start_timestamp = Column(String , nullable=False) - end_timestamp = Column(String , nullable=False) + start_timestamp = Column(Float , nullable=False) + end_timestamp = Column(Float , nullable=False) # helps in logging the information def __repr__(self): @@ -42,7 +43,7 @@ class Collector(Base): @classmethod def ConvertCollectorToRow(cls, request): """ - Create an instance of collector rows from a request object. + Create an instance of Collector table rows from a request object. Args: request: The request object containing collector gRPC message. Returns: A row (an instance of Collector table) initialized with content of the request. """ @@ -55,5 +56,18 @@ class Collector(Base): end_timestamp = request.end_time.timestamp ) -# add method to convert gRPC requests to rows if necessary... - + @classmethod + def ConvertRowToCollector(cls, row): + """ + Create and return a dictionary representation of a Collector table instance. + Args: row: The Collector table instance (row) containing the data. + Returns: collector gRPC message initialized with the content of a row. + """ + response = telemetry_frontend_pb2.Collector() + response.collector_id.collector_id.uuid = row.collector_id + response.kpi_id.kpi_id.uuid = row.kpi_id + response.duration_s = row.sampling_duration_s + response.interval_s = row.sampling_interval_s + response.start_time.timestamp = row.start_timestamp + response.end_time.timestamp = row.end_timestamp + return response diff --git a/src/telemetry/database/Telemetry_DB.py b/src/telemetry/database/Telemetry_DB.py index ec7da9e40..32acfd73a 100644 --- a/src/telemetry/database/Telemetry_DB.py +++ b/src/telemetry/database/Telemetry_DB.py @@ -121,13 +121,13 @@ class TelemetryDB: query = session.query(CollectorModel) # Apply filters based on the filter_object if filter_object.kpi_id: - query = query.filter(CollectorModel.kpi_id.in_([k.kpi_id.uuid for k in filter_object.kpi_id])) + query = query.filter(CollectorModel.kpi_id.in_([k.kpi_id.uuid for k in filter_object.kpi_id])) result = query.all() - + # query should be added to return all rows if result: LOGGER.debug(f"Fetched filtered rows from {model.__name__} table with filters: {filter_object}") # - Results: {result} else: - LOGGER.debug(f"No matching row found in {model.__name__} table with filters: {filter_object}") + LOGGER.warning(f"No matching row found in {model.__name__} table with filters: {filter_object}") return result except Exception as e: LOGGER.error(f"Error fetching filtered rows from {model.__name__} table with filters {filter_object} ::: {e}") diff --git a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py index 49641aae1..29c192bdf 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -34,18 +34,20 @@ from telemetry.database.Telemetry_DB import TelemetryDB LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('TelemetryFrontend', 'NBIgRPC') -ACTIVE_COLLECTORS = [] +ACTIVE_COLLECTORS = [] # keep and can be populated from DB class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): def __init__(self): LOGGER.info('Init TelemetryFrontendService') - self.DBobj = TelemetryDB() + self.tele_db_obj = TelemetryDB() self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value}) self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, 'group.id' : 'frontend', 'auto.offset.reset' : 'latest'}) + # --->>> SECTION: StartCollector with all helper methods <<<--- + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StartCollector(self, request : Collector, grpc_context: grpc.ServicerContext # type: ignore @@ -54,7 +56,7 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): response = CollectorId() # TODO: Verify the presence of Kpi ID in KpiDB or assume that KPI ID already exists. - self.DBobj.add_row_to_db( + self.tele_db_obj.add_row_to_db( CollectorModel.ConvertCollectorToRow(request) ) self.PublishRequestOnKafka(request) @@ -66,7 +68,7 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): """ Method to generate collector request on Kafka. """ - collector_id = collector_obj.collector_id.collector_id.uuid + collector_uuid = collector_obj.collector_id.collector_id.uuid collector_to_generate : Tuple [str, int, int] = ( collector_obj.kpi_id.kpi_id.uuid, collector_obj.duration_s, @@ -74,12 +76,12 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): ) self.kafka_producer.produce( KafkaTopic.REQUEST.value, - key = collector_id, + key = collector_uuid, value = str(collector_to_generate), callback = self.delivery_callback ) - LOGGER.info("Collector Request Generated: Collector Id: {:}, Value: {:}".format(collector_id, collector_to_generate)) - ACTIVE_COLLECTORS.append(collector_id) + LOGGER.info("Collector Request Generated: Collector Id: {:}, Value: {:}".format(collector_uuid, collector_to_generate)) + ACTIVE_COLLECTORS.append(collector_uuid) self.kafka_producer.flush() def run_kafka_listener(self): @@ -141,39 +143,59 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): msg (Message): Kafka message object. """ if err: - print(f'Message delivery failed: {err}') + LOGGER.debug('Message delivery failed: {:}'.format(err)) + print('Message delivery failed: {:}'.format(err)) else: - print(f'Message delivered to topic {msg.topic()}') + LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) + print('Message delivered to topic {:}'.format(msg.topic())) + + # <<<--- SECTION: StopCollector with all helper methods --->>> @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StopCollector(self, request : CollectorId, grpc_context: grpc.ServicerContext # type: ignore ) -> Empty: # type: ignore LOGGER.info ("gRPC message: {:}".format(request)) - _collector_id = request.collector_id.uuid - self.publish_to_kafka_request_topic(_collector_id, "", -1, -1) + self.PublishStopRequestOnKafka(request) return Empty() + def PublishStopRequestOnKafka(self, collector_id): + """ + Method to generate stop collector request on Kafka. + """ + collector_uuid = collector_id.collector_id.uuid + collector_to_stop : Tuple [str, int, int] = ( + collector_uuid , -1, -1 + ) + self.kafka_producer.produce( + KafkaTopic.REQUEST.value, + key = collector_uuid, + value = str(collector_to_stop), + callback = self.delivery_callback + ) + LOGGER.info("Collector Stop Request Generated: Collector Id: {:}, Value: {:}".format(collector_uuid, collector_to_stop)) + try: + ACTIVE_COLLECTORS.remove(collector_uuid) + except ValueError: + LOGGER.warning('Collector ID {:} not found in active collector list'.format(collector_uuid)) + self.kafka_producer.flush() + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SelectCollectors(self, request : CollectorFilter, contextgrpc_context: grpc.ServicerContext # type: ignore ) -> CollectorList: # type: ignore LOGGER.info("gRPC message: {:}".format(request)) response = CollectorList() - filter_to_apply = dict() - filter_to_apply['kpi_id'] = request.kpi_id[0].kpi_id.uuid - # filter_to_apply['duration_s'] = request.duration_s[0] + try: - rows = self.managementDBobj.select_with_filter(CollectorModel, **filter_to_apply) + rows = self.tele_db_obj.select_with_filter(CollectorModel, request) except Exception as e: LOGGER.info('Unable to apply filter on kpi descriptor. {:}'.format(e)) try: - if len(rows) != 0: - for row in rows: - collector_obj = Collector() - collector_obj.collector_id.collector_id.uuid = row.collector_id - response.collector_list.append(collector_obj) + for row in rows: + collector_obj = CollectorModel.ConvertRowToCollector(row) + response.collector_list.append(collector_obj) return response except Exception as e: - LOGGER.info('Unable to process response {:}'.format(e)) + LOGGER.info('Unable to process filter response {:}'.format(e)) diff --git a/src/telemetry/frontend/tests/Messages.py b/src/telemetry/frontend/tests/Messages.py index 106c2a5a7..a0e93e8a1 100644 --- a/src/telemetry/frontend/tests/Messages.py +++ b/src/telemetry/frontend/tests/Messages.py @@ -16,67 +16,27 @@ import uuid import random from common.proto import telemetry_frontend_pb2 from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.proto.kpi_manager_pb2 import KpiId # ----------------------- "2nd" Iteration -------------------------------- def create_collector_id(): _collector_id = telemetry_frontend_pb2.CollectorId() - _collector_id.collector_id.uuid = uuid.uuid4() + # _collector_id.collector_id.uuid = str(uuid.uuid4()) + _collector_id.collector_id.uuid = "5d45f53f-d567-429f-9427-9196ac72ff0c" return _collector_id -# def create_collector_id_a(coll_id_str : str): -# _collector_id = telemetry_frontend_pb2.CollectorId() -# _collector_id.collector_id.uuid = str(coll_id_str) -# return _collector_id - def create_collector_request(): _create_collector_request = telemetry_frontend_pb2.Collector() _create_collector_request.collector_id.collector_id.uuid = str(uuid.uuid4()) _create_collector_request.kpi_id.kpi_id.uuid = str(uuid.uuid4()) - # _create_collector_request.collector = "collector description" _create_collector_request.duration_s = float(random.randint(8, 16)) _create_collector_request.interval_s = float(random.randint(2, 4)) return _create_collector_request def create_collector_filter(): _create_collector_filter = telemetry_frontend_pb2.CollectorFilter() - new_kpi_id = _create_collector_filter.kpi_id.add() - new_kpi_id.kpi_id.uuid = "165d20c5-a446-42fa-812f-e2b7ed283c6f" + kpi_id_obj = KpiId() + # kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + kpi_id_obj.kpi_id.uuid = "a7237fa3-caf4-479d-84b6-4d9f9738fb7f" + _create_collector_filter.kpi_id.append(kpi_id_obj) return _create_collector_filter - -# ----------------------- "First" Iteration -------------------------------- -# def create_collector_request_a(): -# _create_collector_request_a = telemetry_frontend_pb2.Collector() -# _create_collector_request_a.collector_id.collector_id.uuid = "-1" -# return _create_collector_request_a - -# def create_collector_request_b(str_kpi_id, coll_duration_s, coll_interval_s -# ) -> telemetry_frontend_pb2.Collector: -# _create_collector_request_b = telemetry_frontend_pb2.Collector() -# _create_collector_request_b.collector_id.collector_id.uuid = '1' -# _create_collector_request_b.kpi_id.kpi_id.uuid = str_kpi_id -# _create_collector_request_b.duration_s = coll_duration_s -# _create_collector_request_b.interval_s = coll_interval_s -# return _create_collector_request_b - -# def create_collector_filter(): -# _create_collector_filter = telemetry_frontend_pb2.CollectorFilter() -# new_collector_id = _create_collector_filter.collector_id.add() -# new_collector_id.collector_id.uuid = "COLL1" -# new_kpi_id = _create_collector_filter.kpi_id.add() -# new_kpi_id.kpi_id.uuid = "KPI1" -# new_device_id = _create_collector_filter.device_id.add() -# new_device_id.device_uuid.uuid = 'DEV1' -# new_service_id = _create_collector_filter.service_id.add() -# new_service_id.service_uuid.uuid = 'SERV1' -# new_slice_id = _create_collector_filter.slice_id.add() -# new_slice_id.slice_uuid.uuid = 'SLC1' -# new_endpoint_id = _create_collector_filter.endpoint_id.add() -# new_endpoint_id.endpoint_uuid.uuid = 'END1' -# new_connection_id = _create_collector_filter.connection_id.add() -# new_connection_id.connection_uuid.uuid = 'CON1' -# _create_collector_filter.kpi_sample_type.append(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED) -# return _create_collector_filter - -# def create_collector_list(): -# _create_collector_list = telemetry_frontend_pb2.CollectorList() -# return _create_collector_list \ No newline at end of file diff --git a/src/telemetry/frontend/tests/test_frontend.py b/src/telemetry/frontend/tests/test_frontend.py index ca2d61370..d967e306a 100644 --- a/src/telemetry/frontend/tests/test_frontend.py +++ b/src/telemetry/frontend/tests/test_frontend.py @@ -19,6 +19,7 @@ import logging # from common.proto.context_pb2 import Empty from common.Constants import ServiceNameEnum from common.proto.telemetry_frontend_pb2 import CollectorId, CollectorList +from common.proto.context_pb2 import Empty from common.Settings import ( get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) @@ -26,7 +27,7 @@ from common.Settings import ( from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendClient from telemetry.frontend.service.TelemetryFrontendService import TelemetryFrontendService from telemetry.frontend.tests.Messages import ( - create_collector_request, create_collector_filter) + create_collector_request, create_collector_id, create_collector_filter) ########################### @@ -85,6 +86,18 @@ def test_StartCollector(telemetryFrontend_client): LOGGER.debug(str(response)) assert isinstance(response, CollectorId) +def test_StopCollector(telemetryFrontend_client): + LOGGER.info(' >>> test_StopCollector START: <<< ') + response = telemetryFrontend_client.StopCollector(create_collector_id()) + LOGGER.debug(str(response)) + assert isinstance(response, Empty) + +def test_SelectCollectors(telemetryFrontend_client): + LOGGER.info(' >>> test_SelectCollectors START: <<< ') + response = telemetryFrontend_client.SelectCollectors(create_collector_filter()) + LOGGER.debug(str(response)) + assert isinstance(response, CollectorList) + # ------- previous test ---------------- # def test_verify_db_and_table(): -- GitLab From ce7fbb402c47698c74e8ffafcf9f97badb9e73c7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 8 Aug 2024 13:51:33 +0000 Subject: [PATCH 23/70] Telemetry Backend Re-structuring - BackendService is restructured according to the design in report. - ResponseListener is added in Frontend - Improvements in test and messages files --- .../service/TelemetryBackendService.py | 334 ++++++++---------- .../backend/tests/testTelemetryBackend.py | 30 +- .../TelemetryFrontendServiceServicerImpl.py | 168 +++++---- src/telemetry/frontend/tests/__init__.py | 14 - src/telemetry/frontend/tests/test_frontend.py | 8 + 5 files changed, 243 insertions(+), 311 deletions(-) delete mode 100644 src/telemetry/frontend/tests/__init__.py diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index d81be79db..937409d15 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -12,64 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ast +import json import time import random import logging -import requests import threading -from typing import Any, Tuple +from typing import Any, Dict from common.proto.context_pb2 import Empty from confluent_kafka import Producer as KafkaProducer from confluent_kafka import Consumer as KafkaConsumer -from confluent_kafka import KafkaException from confluent_kafka import KafkaError -from confluent_kafka.admin import AdminClient, NewTopic -from common.proto.telemetry_frontend_pb2 import Collector, CollectorId -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic +from common.method_wrappers.Decorator import MetricsPool + LOGGER = logging.getLogger(__name__) -METRICS_POOL = MetricsPool('Telemetry', 'TelemetryBackend') -KAFKA_SERVER_IP = '127.0.0.1:9092' -# KAFKA_SERVER_IP = '10.152.183.175:30092' -ADMIN_KAFKA_CLIENT = AdminClient({'bootstrap.servers': KAFKA_SERVER_IP}) -KAFKA_TOPICS = {'request' : 'topic_request', 'response': 'topic_response', - 'raw' : 'topic_raw' , 'labeled' : 'topic_labeled'} -EXPORTER_ENDPOINT = "http://10.152.183.2:9100/metrics" -PRODUCER_CONFIG = {'bootstrap.servers': KAFKA_SERVER_IP,} +METRICS_POOL = MetricsPool('TelemetryBackend', 'backendService') +# EXPORTER_ENDPOINT = "http://10.152.183.2:9100/metrics" class TelemetryBackendService: """ - Class to listens for request on Kafka topic, fetches metrics and produces measured values to another Kafka topic. + Class listens for request on Kafka topic, fetches requested metrics from device. + Produces metrics on both RESPONSE and VALUE kafka topics. """ def __init__(self): LOGGER.info('Init TelemetryBackendService') + self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value}) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, + 'group.id' : 'backend', + 'auto.offset.reset' : 'latest'}) self.running_threads = {} - - def run_kafka_listener(self)->bool: - threading.Thread(target=self.kafka_listener).start() - return True - - def kafka_listener(self): + + def RunRequestListener(self)->bool: + threading.Thread(target=self.RequestListener).start() + return True + + def RequestListener(self): """ listener for requests on Kafka topic. """ - conusmer_configs = { - 'bootstrap.servers' : KAFKA_SERVER_IP, - 'group.id' : 'backend', - 'auto.offset.reset' : 'latest' - } - # topic_request = "topic_request" - consumerObj = KafkaConsumer(conusmer_configs) - # consumerObj.subscribe([topic_request]) - consumerObj.subscribe([KAFKA_TOPICS['request']]) - + consumer = self.kafka_consumer + consumer.subscribe([KafkaTopic.REQUEST.value]) while True: - receive_msg = consumerObj.poll(2.0) + receive_msg = consumer.poll(2.0) if receive_msg is None: - # print (time.time(), " - Telemetry backend is listening on Kafka Topic: ", KAFKA_TOPICS['request']) # added for debugging purposes continue elif receive_msg.error(): if receive_msg.error().code() == KafkaError._PARTITION_EOF: @@ -77,177 +65,159 @@ class TelemetryBackendService: else: print("Consumer error: {}".format(receive_msg.error())) break - (kpi_id, duration, interval) = ast.literal_eval(receive_msg.value().decode('utf-8')) + + collector = json.loads(receive_msg.value().decode('utf-8')) collector_id = receive_msg.key().decode('utf-8') - if duration == -1 and interval == -1: - self.terminate_collector_backend(collector_id) - # threading.Thread(target=self.terminate_collector_backend, args=(collector_id)) + LOGGER.debug('Recevied Collector: {:} - {:}'.format(collector_id, collector)) + print('Recevied Collector: {:} - {:}'.format(collector_id, collector)) + + if collector['duration'] == -1 and collector['interval'] == -1: + self.TerminateCollectorBackend(collector_id) else: - self.run_initiate_collector_backend(collector_id, kpi_id, duration, interval) + self.RunInitiateCollectorBackend(collector_id, collector) + def TerminateCollectorBackend(self, collector_id): + if collector_id in self.running_threads: + thread, stop_event = self.running_threads[collector_id] + stop_event.set() + thread.join() + print ("Terminating backend (by StopCollector): Collector Id: ", collector_id) + del self.running_threads[collector_id] + self.GenerateCollectorResponse(collector_id, "-1", -1) # Termination confirmation to frontend. + else: + print ('Backend collector {:} not found'.format(collector_id)) - def run_initiate_collector_backend(self, collector_id: str, kpi_id: str, duration: int, interval: int): + def RunInitiateCollectorBackend(self, collector_id: str, collector: str): stop_event = threading.Event() - thread = threading.Thread(target=self.initiate_collector_backend, - args=(collector_id, kpi_id, duration, interval, stop_event)) + thread = threading.Thread(target=self.InitiateCollectorBackend, + args=(collector_id, collector, stop_event)) self.running_threads[collector_id] = (thread, stop_event) thread.start() - def initiate_collector_backend(self, collector_id, kpi_id, duration, interval, stop_event - ): # type: ignore + def InitiateCollectorBackend(self, collector_id, collector, stop_event): """ - Method to receive collector request attribues and initiates collecter backend. + Method receives collector request and initiates collecter backend. """ print("Initiating backend for collector: ", collector_id) start_time = time.time() while not stop_event.is_set(): - if time.time() - start_time >= duration: # condition to terminate backend + if time.time() - start_time >= collector['duration']: # condition to terminate backend print("Execuation duration completed: Terminating backend: Collector Id: ", collector_id, " - ", time.time() - start_time) - self.generate_kafka_response(collector_id, "-1", -1) - # write to Kafka to send the termination confirmation. + self.GenerateCollectorResponse(collector_id, "-1", -1) # Termination confirmation to frontend. break - # print ("Received KPI: ", kpi_id, ", Duration: ", duration, ", Fetch Interval: ", interval) - self.extract_kpi_value(collector_id, kpi_id) - # print ("Telemetry Backend running for KPI: ", kpi_id, "after FETCH INTERVAL: ", interval) - time.sleep(interval) + self.ExtractKpiValue(collector_id, collector['kpi_id']) + time.sleep(collector['interval']) - def extract_kpi_value(self, collector_id: str, kpi_id: str): + def ExtractKpiValue(self, collector_id: str, kpi_id: str): """ Method to extract kpi value. """ - measured_kpi_value = random.randint(1,100) # Should be extracted from exporter/stream - # measured_kpi_value = self.fetch_node_exporter_metrics() # exporter extracted metric value against default KPI - self.generate_kafka_response(collector_id, kpi_id , measured_kpi_value) + measured_kpi_value = random.randint(1,100) # TODO: To be extracted from a device + print ("Measured Kpi value: {:}".format(measured_kpi_value)) + # measured_kpi_value = self.fetch_node_exporter_metrics() # exporter extracted metric value against default KPI + self.GenerateCollectorResponse(collector_id, kpi_id , measured_kpi_value) - def generate_kafka_response(self, collector_id: str, kpi_id: str, kpi_value: Any): + def GenerateCollectorResponse(self, collector_id: str, kpi_id: str, measured_kpi_value: Any): """ Method to write response on Kafka topic """ - # topic_response = "topic_response" - msg_value : Tuple [str, Any] = (kpi_id, kpi_value) - msg_key = collector_id - producerObj = KafkaProducer(PRODUCER_CONFIG) - # producerObj.produce(topic_response, key=msg_key, value= str(msg_value), callback=self.delivery_callback) - producerObj.produce(KAFKA_TOPICS['response'], key=msg_key, value= str(msg_value), callback=TelemetryBackendService.delivery_callback) - producerObj.flush() - - def terminate_collector_backend(self, collector_id): - if collector_id in self.running_threads: - thread, stop_event = self.running_threads[collector_id] - stop_event.set() - thread.join() - print ("Terminating backend (by StopCollector): Collector Id: ", collector_id) - del self.running_threads[collector_id] - self.generate_kafka_response(collector_id, "-1", -1) - - def create_topic_if_not_exists(self, new_topics: list) -> bool: - """ - Method to create Kafka topic if it does not exist. - Args: - admin_client (AdminClient): Kafka admin client. - """ - for topic in new_topics: - try: - topic_metadata = ADMIN_KAFKA_CLIENT.list_topics(timeout=5) - if topic not in topic_metadata.topics: - # If the topic does not exist, create a new topic - print(f"Topic '{topic}' does not exist. Creating...") - LOGGER.warning("Topic {:} does not exist. Creating...".format(topic)) - new_topic = NewTopic(topic, num_partitions=1, replication_factor=1) - ADMIN_KAFKA_CLIENT.create_topics([new_topic]) - except KafkaException as e: - print(f"Failed to create topic: {e}") - return False - return True + producer = self.kafka_producer + kpi_value : Dict = { + "kpi_id" : kpi_id, + "kpi_value" : measured_kpi_value + } + producer.produce( + KafkaTopic.RESPONSE.value, + key = collector_id, + value = json.dumps(kpi_value), + callback = self.delivery_callback + ) + producer.flush() - @staticmethod - def delivery_callback( err, msg): + def delivery_callback(self, err, msg): """ Callback function to handle message delivery status. - Args: - err (KafkaError): Kafka error object. - msg (Message): Kafka message object. - """ - if err: - print(f'Message delivery failed: {err}') - else: - print(f'Message delivered to topic {msg.topic()}') - -# ----------- BELOW: Actual Implementation of Kafka Producer with Node Exporter ----------- - @staticmethod - def fetch_single_node_exporter_metric(): - """ - Method to fetch metrics from Node Exporter. - Returns: - str: Metrics fetched from Node Exporter. - """ - KPI = "node_network_receive_packets_total" - try: - response = requests.get(EXPORTER_ENDPOINT) # type: ignore - LOGGER.info("Request status {:}".format(response)) - if response.status_code == 200: - # print(f"Metrics fetched sucessfully...") - metrics = response.text - # Check if the desired metric is available in the response - if KPI in metrics: - KPI_VALUE = TelemetryBackendService.extract_metric_value(metrics, KPI) - # Extract the metric value - if KPI_VALUE is not None: - LOGGER.info("Extracted value of {:} is {:}".format(KPI, KPI_VALUE)) - print(f"Extracted value of {KPI} is: {KPI_VALUE}") - return KPI_VALUE - else: - LOGGER.info("Failed to fetch metrics. Status code: {:}".format(response.status_code)) - # print(f"Failed to fetch metrics. Status code: {response.status_code}") - return None - except Exception as e: - LOGGER.info("Failed to fetch metrics. Status code: {:}".format(e)) - # print(f"Failed to fetch metrics: {str(e)}") - return None - - @staticmethod - def extract_metric_value(metrics, metric_name): - """ - Method to extract the value of a metric from the metrics string. - Args: - metrics (str): Metrics string fetched from Exporter. - metric_name (str): Name of the metric to extract. - Returns: - float: Value of the extracted metric, or None if not found. - """ - try: - # Find the metric line containing the desired metric name - metric_line = next(line for line in metrics.split('\n') if line.startswith(metric_name)) - # Split the line to extract the metric value - metric_value = float(metric_line.split()[1]) - return metric_value - except StopIteration: - print(f"Metric '{metric_name}' not found in the metrics.") - return None - - @staticmethod - def stream_node_export_metrics_to_raw_topic(): - try: - while True: - response = requests.get(EXPORTER_ENDPOINT) - # print("Response Status {:} ".format(response)) - # LOGGER.info("Response Status {:} ".format(response)) - try: - if response.status_code == 200: - producerObj = KafkaProducer(PRODUCER_CONFIG) - producerObj.produce(KAFKA_TOPICS['raw'], key="raw", value= str(response.text), callback=TelemetryBackendService.delivery_callback) - producerObj.flush() - LOGGER.info("Produce to topic") - else: - LOGGER.info("Didn't received expected response. Status code: {:}".format(response.status_code)) - print(f"Didn't received expected response. Status code: {response.status_code}") - return None - time.sleep(15) - except Exception as e: - LOGGER.info("Failed to process response. Status code: {:}".format(e)) - return None - except Exception as e: - LOGGER.info("Failed to fetch metrics. Status code: {:}".format(e)) - print(f"Failed to fetch metrics: {str(e)}") - return None -# ----------- ABOVE: Actual Implementation of Kafka Producer with Node Exporter ----------- \ No newline at end of file + Args: err (KafkaError): Kafka error object. + msg (Message): Kafka message object. + """ + if err: print(f'Message delivery failed: {err}') + # else: print(f'Message delivered to topic {msg.topic()}') + +# # ----------- BELOW: Actual Implementation of Kafka Producer with Node Exporter ----------- +# @staticmethod +# def fetch_single_node_exporter_metric(): +# """ +# Method to fetch metrics from Node Exporter. +# Returns: +# str: Metrics fetched from Node Exporter. +# """ +# KPI = "node_network_receive_packets_total" +# try: +# response = requests.get(EXPORTER_ENDPOINT) # type: ignore +# LOGGER.info("Request status {:}".format(response)) +# if response.status_code == 200: +# # print(f"Metrics fetched sucessfully...") +# metrics = response.text +# # Check if the desired metric is available in the response +# if KPI in metrics: +# KPI_VALUE = TelemetryBackendService.extract_metric_value(metrics, KPI) +# # Extract the metric value +# if KPI_VALUE is not None: +# LOGGER.info("Extracted value of {:} is {:}".format(KPI, KPI_VALUE)) +# print(f"Extracted value of {KPI} is: {KPI_VALUE}") +# return KPI_VALUE +# else: +# LOGGER.info("Failed to fetch metrics. Status code: {:}".format(response.status_code)) +# # print(f"Failed to fetch metrics. Status code: {response.status_code}") +# return None +# except Exception as e: +# LOGGER.info("Failed to fetch metrics. Status code: {:}".format(e)) +# # print(f"Failed to fetch metrics: {str(e)}") +# return None + +# @staticmethod +# def extract_metric_value(metrics, metric_name): +# """ +# Method to extract the value of a metric from the metrics string. +# Args: +# metrics (str): Metrics string fetched from Exporter. +# metric_name (str): Name of the metric to extract. +# Returns: +# float: Value of the extracted metric, or None if not found. +# """ +# try: +# # Find the metric line containing the desired metric name +# metric_line = next(line for line in metrics.split('\n') if line.startswith(metric_name)) +# # Split the line to extract the metric value +# metric_value = float(metric_line.split()[1]) +# return metric_value +# except StopIteration: +# print(f"Metric '{metric_name}' not found in the metrics.") +# return None + +# @staticmethod +# def stream_node_export_metrics_to_raw_topic(): +# try: +# while True: +# response = requests.get(EXPORTER_ENDPOINT) +# # print("Response Status {:} ".format(response)) +# # LOGGER.info("Response Status {:} ".format(response)) +# try: +# if response.status_code == 200: +# producerObj = KafkaProducer(PRODUCER_CONFIG) +# producerObj.produce(KAFKA_TOPICS['raw'], key="raw", value= str(response.text), callback=TelemetryBackendService.delivery_callback) +# producerObj.flush() +# LOGGER.info("Produce to topic") +# else: +# LOGGER.info("Didn't received expected response. Status code: {:}".format(response.status_code)) +# print(f"Didn't received expected response. Status code: {response.status_code}") +# return None +# time.sleep(15) +# except Exception as e: +# LOGGER.info("Failed to process response. Status code: {:}".format(e)) +# return None +# except Exception as e: +# LOGGER.info("Failed to fetch metrics. Status code: {:}".format(e)) +# print(f"Failed to fetch metrics: {str(e)}") +# return None +# # ----------- ABOVE: Actual Implementation of Kafka Producer with Node Exporter ----------- \ No newline at end of file diff --git a/src/telemetry/backend/tests/testTelemetryBackend.py b/src/telemetry/backend/tests/testTelemetryBackend.py index d832e54e7..3d7ec82ac 100644 --- a/src/telemetry/backend/tests/testTelemetryBackend.py +++ b/src/telemetry/backend/tests/testTelemetryBackend.py @@ -12,15 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -print (sys.path) -sys.path.append('/home/tfs/tfs-ctrl') -import threading import logging -from typing import Tuple -# from common.proto.context_pb2 import Empty from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService + LOGGER = logging.getLogger(__name__) @@ -28,26 +23,9 @@ LOGGER = logging.getLogger(__name__) # Tests Implementation of Telemetry Backend ########################### -def test_verify_kafka_topics(): - LOGGER.info('test_verify_kafka_topics requesting') +def test_RunRequestListener(): + LOGGER.info('test_RunRequestListener') TelemetryBackendServiceObj = TelemetryBackendService() - KafkaTopics = ['topic_request', 'topic_response', 'topic_raw', 'topic_labled'] - response = TelemetryBackendServiceObj.create_topic_if_not_exists(KafkaTopics) + response = TelemetryBackendServiceObj.RunRequestListener() LOGGER.debug(str(response)) assert isinstance(response, bool) - -# def test_run_kafka_listener(): -# LOGGER.info('test_receive_kafka_request requesting') -# TelemetryBackendServiceObj = TelemetryBackendService() -# response = TelemetryBackendServiceObj.run_kafka_listener() -# LOGGER.debug(str(response)) -# assert isinstance(response, bool) - -# def test_fetch_node_exporter_metrics(): -# LOGGER.info(' >>> test_fetch_node_exporter_metrics START <<< ') -# TelemetryBackendService.fetch_single_node_exporter_metric() - -def test_stream_node_export_metrics_to_raw_topic(): - LOGGER.info(' >>> test_stream_node_export_metrics_to_raw_topic START <<< ') - threading.Thread(target=TelemetryBackendService.stream_node_export_metrics_to_raw_topic, args=()).start() - diff --git a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py index 29c192bdf..e6a6d0cd5 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -12,25 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ast +import json import threading -from typing import Tuple, Any +from typing import Any, Dict import grpc import logging from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.tools.kafka.Variables import KafkaConfig, KafkaTopic -from confluent_kafka import Consumer as KafkaConsumer -from confluent_kafka import Producer as KafkaProducer -from confluent_kafka import KafkaError - from common.proto.context_pb2 import Empty - from common.proto.telemetry_frontend_pb2 import CollectorId, Collector, CollectorFilter, CollectorList from common.proto.telemetry_frontend_pb2_grpc import TelemetryFrontendServiceServicer + from telemetry.database.TelemetryModel import Collector as CollectorModel from telemetry.database.Telemetry_DB import TelemetryDB +from confluent_kafka import Consumer as KafkaConsumer +from confluent_kafka import Producer as KafkaProducer +from confluent_kafka import KafkaError + LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('TelemetryFrontend', 'NBIgRPC') @@ -46,8 +46,7 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): 'group.id' : 'frontend', 'auto.offset.reset' : 'latest'}) - # --->>> SECTION: StartCollector with all helper methods <<<--- - + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StartCollector(self, request : Collector, grpc_context: grpc.ServicerContext # type: ignore @@ -55,101 +54,35 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): LOGGER.info ("gRPC message: {:}".format(request)) response = CollectorId() - # TODO: Verify the presence of Kpi ID in KpiDB or assume that KPI ID already exists. + # TODO: Verify the presence of Kpi ID in KpiDB or assume that KPI ID already exists? self.tele_db_obj.add_row_to_db( CollectorModel.ConvertCollectorToRow(request) ) - self.PublishRequestOnKafka(request) + self.PublishStartRequestOnKafka(request) - response.collector_id.uuid = request.collector_id.collector_id.uuid # type: ignore + response.collector_id.uuid = request.collector_id.collector_id.uuid return response - def PublishRequestOnKafka(self, collector_obj): + def PublishStartRequestOnKafka(self, collector_obj): """ Method to generate collector request on Kafka. """ collector_uuid = collector_obj.collector_id.collector_id.uuid - collector_to_generate : Tuple [str, int, int] = ( - collector_obj.kpi_id.kpi_id.uuid, - collector_obj.duration_s, - collector_obj.interval_s - ) + collector_to_generate : Dict = { + "kpi_id" : collector_obj.kpi_id.kpi_id.uuid, + "duration": collector_obj.duration_s, + "interval": collector_obj.interval_s + } self.kafka_producer.produce( KafkaTopic.REQUEST.value, key = collector_uuid, - value = str(collector_to_generate), + value = json.dumps(collector_to_generate), callback = self.delivery_callback ) LOGGER.info("Collector Request Generated: Collector Id: {:}, Value: {:}".format(collector_uuid, collector_to_generate)) ACTIVE_COLLECTORS.append(collector_uuid) self.kafka_producer.flush() - - def run_kafka_listener(self): - # print ("--- STARTED: run_kafka_listener ---") - threading.Thread(target=self.kafka_listener).start() - return True - - def kafka_listener(self): - """ - listener for response on Kafka topic. - """ - # # print ("--- STARTED: kafka_listener ---") - # conusmer_configs = { - # 'bootstrap.servers' : KAFKA_SERVER_IP, - # 'group.id' : 'frontend', - # 'auto.offset.reset' : 'latest' - # } - # # topic_response = "topic_response" - - # consumerObj = KafkaConsumer(conusmer_configs) - self.kafka_consumer.subscribe([KAFKA_TOPICS['response']]) - # print (time.time()) - while True: - receive_msg = self.kafka_consumer.poll(2.0) - if receive_msg is None: - # print (" - Telemetry frontend listening on Kafka Topic: ", KAFKA_TOPICS['response']) # added for debugging purposes - continue - elif receive_msg.error(): - if receive_msg.error().code() == KafkaError._PARTITION_EOF: - continue - else: - print("Consumer error: {}".format(receive_msg.error())) - break - try: - collector_id = receive_msg.key().decode('utf-8') - if collector_id in ACTIVE_COLLECTORS: - (kpi_id, kpi_value) = ast.literal_eval(receive_msg.value().decode('utf-8')) - self.process_response(collector_id, kpi_id, kpi_value) - else: - print(f"collector id does not match.\nRespone ID: '{collector_id}' --- Active IDs: '{ACTIVE_COLLECTORS}' ") - except Exception as e: - print(f"No message key found: {str(e)}") - continue - # return None - def process_response(self, collector_id: str, kpi_id: str, kpi_value: Any): - if kpi_id == "-1" and kpi_value == -1: - # LOGGER.info("Sucessfully terminated Collector: {:}".format(collector_id)) - print ("Sucessfully terminated Collector: ", collector_id) - else: - print ("Frontend-Received values Collector Id:", collector_id, "-KPI:", kpi_id, "-VALUE:", kpi_value) - - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def delivery_callback(self, err, msg): - """ - Callback function to handle message delivery status. - Args: - err (KafkaError): Kafka error object. - msg (Message): Kafka message object. - """ - if err: - LOGGER.debug('Message delivery failed: {:}'.format(err)) - print('Message delivery failed: {:}'.format(err)) - else: - LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) - print('Message delivered to topic {:}'.format(msg.topic())) - - # <<<--- SECTION: StopCollector with all helper methods --->>> @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StopCollector(self, @@ -164,13 +97,15 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): Method to generate stop collector request on Kafka. """ collector_uuid = collector_id.collector_id.uuid - collector_to_stop : Tuple [str, int, int] = ( - collector_uuid , -1, -1 - ) + collector_to_stop : Dict = { + "kpi_id" : collector_uuid, + "duration": -1, + "interval": -1 + } self.kafka_producer.produce( KafkaTopic.REQUEST.value, key = collector_uuid, - value = str(collector_to_stop), + value = json.dumps(collector_to_stop), callback = self.delivery_callback ) LOGGER.info("Collector Stop Request Generated: Collector Id: {:}, Value: {:}".format(collector_uuid, collector_to_stop)) @@ -180,6 +115,7 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): LOGGER.warning('Collector ID {:} not found in active collector list'.format(collector_uuid)) self.kafka_producer.flush() + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SelectCollectors(self, request : CollectorFilter, contextgrpc_context: grpc.ServicerContext # type: ignore @@ -199,3 +135,57 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): except Exception as e: LOGGER.info('Unable to process filter response {:}'.format(e)) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def delivery_callback(self, err, msg): + """ + Callback function to handle message delivery status. + Args: + err (KafkaError): Kafka error object. + msg (Message): Kafka message object. + """ + if err: + LOGGER.debug('Message delivery failed: {:}'.format(err)) + print('Message delivery failed: {:}'.format(err)) + # else: + # LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) + # print('Message delivered to topic {:}'.format(msg.topic())) + + # ---------- Independent Method --------------- + # Listener method is independent of any method (same lifetime as service) + # continously listens for responses + def RunResponseListener(self): + threading.Thread(target=self.ResponseListener).start() + return True + + def ResponseListener(self): + """ + listener for response on Kafka topic. + """ + self.kafka_consumer.subscribe([KafkaTopic.RESPONSE.value]) + while True: + receive_msg = self.kafka_consumer.poll(2.0) + if receive_msg is None: + continue + elif receive_msg.error(): + if receive_msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + print("Consumer error: {}".format(receive_msg.error())) + break + try: + collector_id = receive_msg.key().decode('utf-8') + if collector_id in ACTIVE_COLLECTORS: + kpi_value = json.loads(receive_msg.value().decode('utf-8')) + self.process_response(collector_id, kpi_value['kpi_id'], kpi_value['kpi_value']) + else: + print(f"collector id does not match.\nRespone ID: '{collector_id}' --- Active IDs: '{ACTIVE_COLLECTORS}' ") + except Exception as e: + print(f"Error extarcting msg key or value: {str(e)}") + continue + + def process_response(self, collector_id: str, kpi_id: str, kpi_value: Any): + if kpi_id == "-1" and kpi_value == -1: + print ("Backend termination confirmation for collector id: ", collector_id) + else: + print ("KPI Value: Collector Id:", collector_id, ", Kpi Id:", kpi_id, ", Value:", kpi_value) diff --git a/src/telemetry/frontend/tests/__init__.py b/src/telemetry/frontend/tests/__init__.py deleted file mode 100644 index 3ee6f7071..000000000 --- a/src/telemetry/frontend/tests/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/src/telemetry/frontend/tests/test_frontend.py b/src/telemetry/frontend/tests/test_frontend.py index d967e306a..3f8f3ebc8 100644 --- a/src/telemetry/frontend/tests/test_frontend.py +++ b/src/telemetry/frontend/tests/test_frontend.py @@ -28,6 +28,7 @@ from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendC from telemetry.frontend.service.TelemetryFrontendService import TelemetryFrontendService from telemetry.frontend.tests.Messages import ( create_collector_request, create_collector_id, create_collector_filter) +from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import TelemetryFrontendServiceServicerImpl ########################### @@ -98,6 +99,13 @@ def test_SelectCollectors(telemetryFrontend_client): LOGGER.debug(str(response)) assert isinstance(response, CollectorList) +def test_RunResponseListener(): + LOGGER.info(' >>> test_RunResponseListener START: <<< ') + TelemetryFrontendServiceObj = TelemetryFrontendServiceServicerImpl() + response = TelemetryFrontendServiceObj.RunResponseListener() # becasue Method "run_kafka_listener" is not define in frontend.proto + LOGGER.debug(str(response)) + assert isinstance(response, bool) + # ------- previous test ---------------- # def test_verify_db_and_table(): -- GitLab From df455e9164011098b8c664bd86dd63f421d41c1d Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 8 Aug 2024 14:23:29 +0000 Subject: [PATCH 24/70] Telemetry Backend Service - GenerateRawMetrics() add --- .../backend/service/TelemetryBackendService.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index 937409d15..048474d93 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -119,7 +119,7 @@ class TelemetryBackendService: def GenerateCollectorResponse(self, collector_id: str, kpi_id: str, measured_kpi_value: Any): """ - Method to write response on Kafka topic + Method to write kpi value on RESPONSE Kafka topic """ producer = self.kafka_producer kpi_value : Dict = { @@ -134,6 +134,22 @@ class TelemetryBackendService: ) producer.flush() + def GenerateRawMetric(self, metrics: Any): + """ + Method writes raw metrics on VALUE Kafka topic + """ + producer = self.kafka_producer + some_metric : Dict = { + "some_id" : metrics + } + producer.produce( + KafkaTopic.VALUE.value, + key = 'raw', + value = json.dumps(some_metric), + callback = self.delivery_callback + ) + producer.flush() + def delivery_callback(self, err, msg): """ Callback function to handle message delivery status. -- GitLab From 706d284c9287700000ecc24852d35f60595f8196 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 8 Aug 2024 15:20:06 +0000 Subject: [PATCH 25/70] Changes in Telemetry backend service. - __main__ is added. - DockerFile added. - .gitlab-ci.yml file added. --- .gitlab-ci.yml | 3 +- src/telemetry/.gitlab-ci.yml | 142 +++++++++++++++++++++ src/telemetry/backend/Dockerfile | 69 ++++++++++ src/telemetry/backend/requirements.in | 15 +++ src/telemetry/backend/service/__main__.py | 51 ++++++++ src/telemetry/frontend/Dockerfile | 69 ++++++++++ src/telemetry/frontend/requirements.in | 15 +++ src/telemetry/frontend/service/__main__.py | 39 ++---- src/telemetry/telemetry_virenv.txt | 49 ------- 9 files changed, 372 insertions(+), 80 deletions(-) create mode 100644 src/telemetry/.gitlab-ci.yml create mode 100644 src/telemetry/backend/Dockerfile create mode 100644 src/telemetry/backend/requirements.in create mode 100644 src/telemetry/backend/service/__main__.py create mode 100644 src/telemetry/frontend/Dockerfile create mode 100644 src/telemetry/frontend/requirements.in delete mode 100644 src/telemetry/telemetry_virenv.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c5ff9325..42292dc37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,7 @@ include: - local: '/src/kpi_manager/.gitlab-ci.yml' - local: '/src/kpi_value_api/.gitlab-ci.yml' - local: '/src/kpi_value_writer/.gitlab-ci.yml' - + # - local: '/src/telemetry/frontend/.gitlab-ci.yml' + # - local: '/src/telemetry/backend/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml new file mode 100644 index 000000000..d2e7e8cf3 --- /dev/null +++ b/src/telemetry/.gitlab-ci.yml @@ -0,0 +1,142 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build kpi-manager: + variables: + IMAGE_NAME: 'telemetry' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "${IMAGE_NAME}-frontend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/frontend/Dockerfile . + - docker buildx build -t "${IMAGE_NAME}-backend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/backend/Dockerfile . + - docker tag "${IMAGE_NAME}-frontend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker tag "${IMAGE_NAME}-backend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/frontend/**/*.{py,in,yml} + - src/$IMAGE_NAME/frontend/Dockerfile + - src/$IMAGE_NAME/frontend/tests/*.py + - src/$IMAGE_NAME/backend/Dockerfile + - src/$IMAGE_NAME/backend/**/*.{py,in,yml} + - src/$IMAGE_NAME/backend/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test telemetry: + variables: + IMAGE_NAME: 'telemetry' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build telemetry + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - docker logs crdb + - docker ps -a + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - > + docker run --name $IMAGE_NAME -d -p 30010:30010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs $IMAGE_NAME + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/frontend/Dockerfile + - src/$IMAGE_NAME/frontend/tests/*.py + - src/$IMAGE_NAME/frontend/tests/Dockerfile + - src/$IMAGE_NAME/backend/Dockerfile + - src/$IMAGE_NAME/backend/tests/*.py + - src/$IMAGE_NAME/backend/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + # artifacts: + # when: always + # reports: + # junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml + +## Deployment of the service in Kubernetes Cluster +#deploy context: +# variables: +# IMAGE_NAME: 'context' # name of the microservice +# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) +# stage: deploy +# needs: +# - unit test context +# # - integ_test execute +# script: +# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' +# - kubectl version +# - kubectl get all +# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" +# - kubectl get all +# # environment: +# # name: test +# # url: https://example.com +# # kubernetes: +# # namespace: test +# rules: +# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' +# when: manual +# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' +# when: manual diff --git a/src/telemetry/backend/Dockerfile b/src/telemetry/backend/Dockerfile new file mode 100644 index 000000000..eebfe24ab --- /dev/null +++ b/src/telemetry/backend/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/telemetry/backend +WORKDIR /var/teraflow/telemetry/backend +COPY src/telemetry/backend/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/telemetry/__init__.py telemetry/__init__.py +COPY src/telemetry/backend/. telemetry/backend/ + +# Start the service +ENTRYPOINT ["python", "-m", "telemetry.backend.service"] diff --git a/src/telemetry/backend/requirements.in b/src/telemetry/backend/requirements.in new file mode 100644 index 000000000..e6a559be7 --- /dev/null +++ b/src/telemetry/backend/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +confluent-kafka==2.3.* diff --git a/src/telemetry/backend/service/__main__.py b/src/telemetry/backend/service/__main__.py new file mode 100644 index 000000000..4ad867331 --- /dev/null +++ b/src/telemetry/backend/service/__main__.py @@ -0,0 +1,51 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Settings import get_log_level +from .TelemetryBackendService import TelemetryBackendService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = TelemetryBackendService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/telemetry/frontend/Dockerfile b/src/telemetry/frontend/Dockerfile new file mode 100644 index 000000000..0c3e1a66a --- /dev/null +++ b/src/telemetry/frontend/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/telemetry/frontend +WORKDIR /var/teraflow/telemetry/frontend +COPY src/telemetry/frontend/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/telemetry/__init__.py telemetry/__init__.py +COPY src/telemetry/frontend/. telemetry/frontend/ + +# Start the service +ENTRYPOINT ["python", "-m", "telemetry.frontend.service"] diff --git a/src/telemetry/frontend/requirements.in b/src/telemetry/frontend/requirements.in new file mode 100644 index 000000000..e6a559be7 --- /dev/null +++ b/src/telemetry/frontend/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +confluent-kafka==2.3.* diff --git a/src/telemetry/frontend/service/__main__.py b/src/telemetry/frontend/service/__main__.py index 3b0263706..238619f2e 100644 --- a/src/telemetry/frontend/service/__main__.py +++ b/src/telemetry/frontend/service/__main__.py @@ -12,16 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import signal -import sys -import logging, threading -from prometheus_client import start_http_server -from monitoring.service.NameMapping import NameMapping +import logging, signal, sys, threading +from common.Settings import get_log_level from .TelemetryFrontendService import TelemetryFrontendService -from monitoring.service.EventTools import EventsDeviceCollector -from common.Settings import ( - get_log_level, wait_for_environment_variables, get_env_var_name, - get_metrics_port ) terminate = threading.Event() LOGGER = None @@ -31,42 +24,28 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name terminate.set() def main(): - global LOGGER + global LOGGER # pylint: disable=global-statement log_level = get_log_level() - logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") + logging.basicConfig(level=log_level) LOGGER = logging.getLogger(__name__) -# ------- will be added later -------------- - # wait_for_environment_variables([ - # get_env_var_name - - - # ]) -# ------- will be added later -------------- - signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - LOGGER.info('Starting...') - - # Start metrics server - metrics_port = get_metrics_port() - start_http_server(metrics_port) - - name_mapping = NameMapping() + LOGGER.debug('Starting...') - grpc_service = TelemetryFrontendService(name_mapping) + grpc_service = TelemetryFrontendService() grpc_service.start() # Wait for Ctrl+C or termination signal while not terminate.wait(timeout=1.0): pass - LOGGER.info('Terminating...') + LOGGER.debug('Terminating...') grpc_service.stop() - LOGGER.info('Bye') + LOGGER.debug('Bye') return 0 if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/src/telemetry/telemetry_virenv.txt b/src/telemetry/telemetry_virenv.txt deleted file mode 100644 index e39f80b65..000000000 --- a/src/telemetry/telemetry_virenv.txt +++ /dev/null @@ -1,49 +0,0 @@ -anytree==2.8.0 -APScheduler==3.10.1 -attrs==23.2.0 -certifi==2024.2.2 -charset-normalizer==2.0.12 -colorama==0.4.6 -confluent-kafka==2.3.0 -coverage==6.3 -future-fstrings==1.2.0 -greenlet==3.0.3 -grpcio==1.47.5 -grpcio-health-checking==1.47.5 -grpcio-tools==1.47.5 -grpclib==0.4.4 -h2==4.1.0 -hpack==4.0.0 -hyperframe==6.0.1 -idna==3.7 -influx-line-protocol==0.1.4 -iniconfig==2.0.0 -kafka-python==2.0.2 -multidict==6.0.5 -networkx==3.3 -packaging==24.0 -pluggy==1.5.0 -prettytable==3.5.0 -prometheus-client==0.13.0 -protobuf==3.20.3 -psycopg2-binary==2.9.3 -py==1.11.0 -py-cpuinfo==9.0.0 -pytest==6.2.5 -pytest-benchmark==3.4.1 -pytest-depends==1.0.1 -python-dateutil==2.8.2 -python-json-logger==2.0.2 -pytz==2024.1 -questdb==1.0.1 -requests==2.27.1 -six==1.16.0 -SQLAlchemy==1.4.52 -sqlalchemy-cockroachdb==1.4.4 -SQLAlchemy-Utils==0.38.3 -toml==0.10.2 -typing_extensions==4.12.0 -tzlocal==5.2 -urllib3==1.26.18 -wcwidth==0.2.13 -xmltodict==0.12.0 -- GitLab From 466fd377d23f5931be7a033bb56e2d3943e77a11 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 07:37:32 +0000 Subject: [PATCH 26/70] Kafka deployment script in gitlab-ci.file - In Kafka.variables files: get_kafka_address() and get_admin_client() is added. - In KpiValueApiServerImpl Kafka Admin Client call is updated. - Kafka deployment script is added. --- scripts/run_tests_locally-kpi-value-API.sh | 3 ++- src/common/tools/kafka/Variables.py | 27 +++++++++++++------ src/kpi_value_api/.gitlab-ci.yml | 22 ++++++++++++++- .../service/KpiValueApiServiceServicerImpl.py | 2 +- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/scripts/run_tests_locally-kpi-value-API.sh b/scripts/run_tests_locally-kpi-value-API.sh index 8dfbfb162..3953d2a89 100755 --- a/scripts/run_tests_locally-kpi-value-API.sh +++ b/scripts/run_tests_locally-kpi-value-API.sh @@ -19,7 +19,8 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src RCFILE=$PROJECTDIR/coverage/.coveragerc - +KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") +KFK_SERVER_ADDRESS=${KAFKA_IP}:9092 # helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG -o log_cli=true --verbose \ kpi_value_api/tests/test_kpi_value_api.py diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 168957a26..1abbe7d7e 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -20,16 +20,27 @@ from common.Settings import get_setting LOGGER = logging.getLogger(__name__) +KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' class KafkaConfig(Enum): - KFK_SERVER_ADDRESS_TEMPLATE = 'kafka-service.{:s}.svc.cluster.local:{:s}' - KFK_NAMESPACE = 'kafka' - # KFK_NAMESPACE = get_setting('KFK_NAMESPACE') - KFK_PORT = '9092' - # KFK_PORT = get_setting('KFK_SERVER_PORT') - # SERVER_ADDRESS = "127.0.0.1:9092" - SERVER_ADDRESS = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) - ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) + + @staticmethod + def get_kafka_address() -> str: + kafka_server_address = get_setting('KFK_SERVER_ADDRESS', default=None) + if kafka_server_address is None: + KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = get_setting('KFK_SERVER_PORT') + SERVER_ADDRESS = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) + return SERVER_ADDRESS + + @staticmethod + def get_admin_client(): + SERVER_ADDRESS = KafkaConfig.get_kafka_address() + LOGGER.debug("KAFKA_SERVER_ADDRESS {:}".format(SERVER_ADDRESS)) + # SERVER_ADDRESS = "127.0.0.1:9092" + ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) + return ADMIN_CLIENT + class KafkaTopic(Enum): REQUEST = 'topic_request' diff --git a/src/kpi_value_api/.gitlab-ci.yml b/src/kpi_value_api/.gitlab-ci.yml index 166e9d3cb..1919f0361 100644 --- a/src/kpi_value_api/.gitlab-ci.yml +++ b/src/kpi_value_api/.gitlab-ci.yml @@ -50,10 +50,30 @@ unit_test kpi-value-api: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi + - if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 30020:30020 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker pull "bitnami/zookeeper:latest" + - docker pull "bitnami/kafka:latest" + - > + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + bitnami/zookeeper:latest + - sleep 10 # Wait for Zookeeper to start + - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + --env ALLOW_PLAINTEXT_LISTENER=yes + bitnami/kafka:latest + - sleep 20 # Wait for Kafka to start + - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $KAFKA_IP + - > + docker run --name $IMAGE_NAME -d -p 30020:30020 + --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs $IMAGE_NAME diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py index 5e7c3d139..05ab63fdf 100644 --- a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -43,7 +43,7 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): ) -> Empty: LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) producer_obj = KafkaProducer({ - 'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value + 'bootstrap.servers' : KafkaConfig.get_admin_client() }) for kpi_value in request.kpi_value_list: kpi_value_to_produce : Tuple [str, Any, Any] = ( -- GitLab From b53a76a56b4d9e662ce14149ca22706cdb092566 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 08:06:23 +0000 Subject: [PATCH 27/70] Kafka deployment script in gitlab-ci.file (2) - Improvements in Kafka.variables files. - In KpiValueApiServiceImpl corrected the call from "admin_client()" to "kafka_address()" --- src/common/tools/kafka/Variables.py | 16 +++++++--------- .../service/KpiValueApiServiceServicerImpl.py | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 1abbe7d7e..9d42f1550 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -28,16 +28,14 @@ class KafkaConfig(Enum): def get_kafka_address() -> str: kafka_server_address = get_setting('KFK_SERVER_ADDRESS', default=None) if kafka_server_address is None: - KFK_NAMESPACE = get_setting('KFK_NAMESPACE') - KFK_PORT = get_setting('KFK_SERVER_PORT') - SERVER_ADDRESS = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) - return SERVER_ADDRESS + KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = get_setting('KFK_SERVER_PORT') + kafka_server_address = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) + return kafka_server_address @staticmethod def get_admin_client(): SERVER_ADDRESS = KafkaConfig.get_kafka_address() - LOGGER.debug("KAFKA_SERVER_ADDRESS {:}".format(SERVER_ADDRESS)) - # SERVER_ADDRESS = "127.0.0.1:9092" ADMIN_CLIENT = AdminClient({'bootstrap.servers': SERVER_ADDRESS }) return ADMIN_CLIENT @@ -55,7 +53,7 @@ class KafkaTopic(Enum): Method to create Kafka topics defined as class members """ all_topics = [member.value for member in KafkaTopic] - LOGGER.debug("Kafka server address is: {:} ".format(KafkaConfig.SERVER_ADDRESS.value)) + LOGGER.debug("Kafka server address is: {:} ".format(KafkaConfig.get_kafka_address())) if( KafkaTopic.create_new_topic_if_not_exists( all_topics )): LOGGER.debug("All topics are created sucsessfully") return True @@ -73,14 +71,14 @@ class KafkaTopic(Enum): LOGGER.debug("Topics names to be verified and created: {:}".format(new_topics)) for topic in new_topics: try: - topic_metadata = KafkaConfig.ADMIN_CLIENT.value.list_topics(timeout=5) + topic_metadata = KafkaConfig.get_admin_client().list_topics(timeout=5) # LOGGER.debug("Existing topic list: {:}".format(topic_metadata.topics)) if topic not in topic_metadata.topics: # If the topic does not exist, create a new topic print("Topic {:} does not exist. Creating...".format(topic)) LOGGER.debug("Topic {:} does not exist. Creating...".format(topic)) new_topic = NewTopic(topic, num_partitions=1, replication_factor=1) - KafkaConfig.ADMIN_CLIENT.value.create_topics([new_topic]) + KafkaConfig.get_admin_client().create_topics([new_topic]) else: print("Topic name already exists: {:}".format(topic)) LOGGER.debug("Topic name already exists: {:}".format(topic)) diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py index 05ab63fdf..3df8dd5b6 100644 --- a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -43,7 +43,7 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): ) -> Empty: LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) producer_obj = KafkaProducer({ - 'bootstrap.servers' : KafkaConfig.get_admin_client() + 'bootstrap.servers' : KafkaConfig.get_kafka_address() }) for kpi_value in request.kpi_value_list: kpi_value_to_produce : Tuple [str, Any, Any] = ( -- GitLab From 1f495c78b88d8e688ba33bca506dc942e30261a5 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 09:08:34 +0000 Subject: [PATCH 28/70] Changes in KpiValueWriter - Kafka deployment script is added in .gitlab-ci file - call is changed to "get_kafka_address()" in KpiValueWriter.py --- src/kpi_value_writer/.gitlab-ci.yml | 24 ++++++++++++++++++- .../service/KpiValueWriter.py | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml index 25619ce7f..2c300db0e 100644 --- a/src/kpi_value_writer/.gitlab-ci.yml +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -50,10 +50,30 @@ unit_test kpi-value-writer: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi + - if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - docker container prune -f script: - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - - docker run --name $IMAGE_NAME -d -p 30030:30030 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker pull "bitnami/zookeeper:latest" + - docker pull "bitnami/kafka:latest" + - > + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + bitnami/zookeeper:latest + - sleep 10 # Wait for Zookeeper to start + - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + --env ALLOW_PLAINTEXT_LISTENER=yes + bitnami/kafka:latest + - sleep 20 # Wait for Kafka to start + - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $KAFKA_IP + - > + docker run --name $IMAGE_NAME -d -p 30030:30030 + --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG - sleep 5 - docker ps -a - docker logs $IMAGE_NAME @@ -64,6 +84,8 @@ unit_test kpi-value-writer: coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME + - docker rm zookeeper + - docker rm kafka - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py index 5e2b6babe..5d1a98808 100644 --- a/src/kpi_value_writer/service/KpiValueWriter.py +++ b/src/kpi_value_writer/service/KpiValueWriter.py @@ -51,7 +51,7 @@ class KpiValueWriter(GenericGrpcService): metric_writer = MetricWriterToPrometheus() kafka_consumer = KafkaConsumer( - { 'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, + { 'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : __class__, 'auto.offset.reset' : 'latest'} ) -- GitLab From dbb3690f68fbccc488edc0c9fe7ac455e891cdf7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 11:02:46 +0000 Subject: [PATCH 29/70] Changes to telemetry service - TelemetryBackendService name and port is added in constants - get_kafka_address() call added in TelemetryBackendService - kafk topics test added in telemetrybackend test file - get_kafka_address() call added in TelemetryBackendServiceImpl - kafk topics test added in telemetryfrontend test file - improvements in Telemetry gitlab-ci file + unit_test for telemetry backend and frontend added. + many other small changes --- src/telemetry/.gitlab-ci.yml | 148 ++++++++++++------ .../service/TelemetryBackendService.py | 4 +- .../backend/tests/testTelemetryBackend.py | 7 + .../TelemetryFrontendServiceServicerImpl.py | 4 +- src/telemetry/frontend/tests/test_frontend.py | 11 +- 5 files changed, 123 insertions(+), 51 deletions(-) diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index d2e7e8cf3..1d63654d9 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -21,6 +21,8 @@ build kpi-manager: before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: + # This first build tags the builder resulting image to prevent being removed by dangling image removal command + - docker buildx build -t "${IMAGE_NAME}-backend:${IMAGE_TAG}-builder" --target builder -f ./src/$IMAGE_NAME/backend/Dockerfile . - docker buildx build -t "${IMAGE_NAME}-frontend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/frontend/Dockerfile . - docker buildx build -t "${IMAGE_NAME}-backend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/backend/Dockerfile . - docker tag "${IMAGE_NAME}-frontend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" @@ -35,6 +37,7 @@ build kpi-manager: - changes: - src/common/**/*.py - proto/*.proto + - src/$IMAGE_NAME/.gitlab-ci.yml - src/$IMAGE_NAME/frontend/**/*.{py,in,yml} - src/$IMAGE_NAME/frontend/Dockerfile - src/$IMAGE_NAME/frontend/tests/*.py @@ -45,7 +48,75 @@ build kpi-manager: - .gitlab-ci.yml # Apply unit test to the component -unit_test telemetry: +unit_test telemetry-backend: + variables: + IMAGE_NAME: 'telemetry' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build telemetry + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi + - if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi + # - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi + - if docker container ls | grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" + - docker pull "bitnami/zookeeper:latest" + - docker pull "bitnami/kafka:latest" + - > + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + bitnami/zookeeper:latest + - sleep 10 # Wait for Zookeeper to start + - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + --env ALLOW_PLAINTEXT_LISTENER=yes + bitnami/kafka:latest + - sleep 20 # Wait for Kafka to start + - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $KAFKA_IP + - > + docker run --name $IMAGE_NAME -d -p 30060:30060 + --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" + --volume "$PWD/src/$IMAGE_NAME/backend/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs ${IMAGE_NAME}-backend + - > + docker exec -i ${IMAGE_NAME}-backend bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}-backend_report.xml $IMAGE_NAME/backend/tests/test_*.py" + - docker exec -i ${IMAGE_NAME}-backend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + - docker rm -f ${IMAGE_NAME}-backend + - docker rm -f zookeeper + - docker rm -f kafka + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/backend/**/*.{py,in,yml} + - src/$IMAGE_NAME/backend/Dockerfile + - src/$IMAGE_NAME/backend/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/backend/tests/${IMAGE_NAME}-backend_report.xml + +# Apply unit test to the component +unit_test telemetry-frontend: variables: IMAGE_NAME: 'telemetry' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) @@ -57,12 +128,14 @@ unit_test telemetry: - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep kafka; then docker rm -f kafka; else echo "Kafka container is not in the system"; fi + - if docker container ls | grep zookeeper; then docker rm -f zookeeper; else echo "Zookeeper container is not in the system"; fi - if docker container ls | grep ${IMAGE_NAME}-frontend; then docker rm -f ${IMAGE_NAME}-frontend; else echo "${IMAGE_NAME}-frontend container is not in the system"; fi - - if docker container ls | grep ${IMAGE_NAME}-backend; then docker rm -f ${IMAGE_NAME}-backend; else echo "${IMAGE_NAME}-backend container is not in the system"; fi - docker container prune -f script: - - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-backend:$IMAGE_TAG" - docker pull "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" + - docker pull "bitnami/zookeeper:latest" + - docker pull "bitnami/kafka:latest" - docker pull "cockroachdb/cockroach:latest-v22.2" - docker volume create crdb - > @@ -77,66 +150,51 @@ unit_test telemetry: - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - > - docker run --name $IMAGE_NAME -d -p 30010:30010 + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + bitnami/zookeeper:latest + - sleep 10 # Wait for Zookeeper to start + - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 + --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + --env ALLOW_PLAINTEXT_LISTENER=yes + bitnami/kafka:latest + - sleep 20 # Wait for Kafka to start + - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $KAFKA_IP + - > + docker run --name $IMAGE_NAME -d -p 30050:30050 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" - --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" + --volume "$PWD/src/$IMAGE_NAME/frontend/tests:/opt/results" --network=teraflowbridge - $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + $CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG - docker ps -a - sleep 5 - - docker logs $IMAGE_NAME + - docker logs ${IMAGE_NAME}-frontend - > - docker exec -i $IMAGE_NAME bash -c - "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" - - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + docker exec -i ${IMAGE_NAME}-frontend bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}-frontend_report.xml $IMAGE_NAME/frontend/tests/test_*.py" + - docker exec -i ${IMAGE_NAME}-frontend bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker volume rm -f crdb - docker network rm teraflowbridge - docker volume prune --force - docker image prune --force + - docker rm -f ${IMAGE_NAME}-frontend + - docker rm -f zookeeper + - docker rm -f kafka rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: - src/common/**/*.py - proto/*.proto - - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/frontend/**/*.{py,in,yml} - src/$IMAGE_NAME/frontend/Dockerfile - src/$IMAGE_NAME/frontend/tests/*.py - - src/$IMAGE_NAME/frontend/tests/Dockerfile - - src/$IMAGE_NAME/backend/Dockerfile - - src/$IMAGE_NAME/backend/tests/*.py - - src/$IMAGE_NAME/backend/tests/Dockerfile - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml - # artifacts: - # when: always - # reports: - # junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml - -## Deployment of the service in Kubernetes Cluster -#deploy context: -# variables: -# IMAGE_NAME: 'context' # name of the microservice -# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) -# stage: deploy -# needs: -# - unit test context -# # - integ_test execute -# script: -# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml' -# - kubectl version -# - kubectl get all -# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml" -# - kubectl get all -# # environment: -# # name: test -# # url: https://example.com -# # kubernetes: -# # namespace: test -# rules: -# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -# when: manual -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' -# when: manual + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/frontend/tests/${IMAGE_NAME}-frontend_report.xml \ No newline at end of file diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index 048474d93..6b9a6a8da 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -39,8 +39,8 @@ class TelemetryBackendService: def __init__(self): LOGGER.info('Init TelemetryBackendService') - self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value}) - self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, + self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'backend', 'auto.offset.reset' : 'latest'}) self.running_threads = {} diff --git a/src/telemetry/backend/tests/testTelemetryBackend.py b/src/telemetry/backend/tests/testTelemetryBackend.py index 3d7ec82ac..95710ff88 100644 --- a/src/telemetry/backend/tests/testTelemetryBackend.py +++ b/src/telemetry/backend/tests/testTelemetryBackend.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +from common.tools.kafka.Variables import KafkaTopic from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService @@ -23,6 +24,12 @@ LOGGER = logging.getLogger(__name__) # Tests Implementation of Telemetry Backend ########################### +# --- "test_validate_kafka_topics" should be run before the functionality tests --- +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) + def test_RunRequestListener(): LOGGER.info('test_RunRequestListener') TelemetryBackendServiceObj = TelemetryBackendService() diff --git a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py index e6a6d0cd5..2b872dba3 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -41,8 +41,8 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): def __init__(self): LOGGER.info('Init TelemetryFrontendService') self.tele_db_obj = TelemetryDB() - self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value}) - self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.SERVER_ADDRESS.value, + self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'frontend', 'auto.offset.reset' : 'latest'}) diff --git a/src/telemetry/frontend/tests/test_frontend.py b/src/telemetry/frontend/tests/test_frontend.py index 3f8f3ebc8..9c3f9d3a8 100644 --- a/src/telemetry/frontend/tests/test_frontend.py +++ b/src/telemetry/frontend/tests/test_frontend.py @@ -16,11 +16,10 @@ import os import pytest import logging -# from common.proto.context_pb2 import Empty from common.Constants import ServiceNameEnum from common.proto.telemetry_frontend_pb2 import CollectorId, CollectorList from common.proto.context_pb2 import Empty - +from common.tools.kafka.Variables import KafkaTopic from common.Settings import ( get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC) @@ -81,6 +80,13 @@ def telemetryFrontend_client( ########################### # ------- Re-structuring Test --------- +# --- "test_validate_kafka_topics" should be run before the functionality tests --- +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) + +# ----- core funtionality test ----- def test_StartCollector(telemetryFrontend_client): LOGGER.info(' >>> test_StartCollector START: <<< ') response = telemetryFrontend_client.StartCollector(create_collector_request()) @@ -99,6 +105,7 @@ def test_SelectCollectors(telemetryFrontend_client): LOGGER.debug(str(response)) assert isinstance(response, CollectorList) +# ----- Non-gRPC method tests ----- def test_RunResponseListener(): LOGGER.info(' >>> test_RunResponseListener START: <<< ') TelemetryFrontendServiceObj = TelemetryFrontendServiceServicerImpl() -- GitLab From fd6f40d88c60413cc411286818754195b9c1a5ca Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 11:06:32 +0000 Subject: [PATCH 30/70] Telemetry fronend and backend ci tests activated --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42292dc37..115b33676 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,7 +48,6 @@ include: - local: '/src/kpi_manager/.gitlab-ci.yml' - local: '/src/kpi_value_api/.gitlab-ci.yml' - local: '/src/kpi_value_writer/.gitlab-ci.yml' - # - local: '/src/telemetry/frontend/.gitlab-ci.yml' - # - local: '/src/telemetry/backend/.gitlab-ci.yml' + - local: '/src/telemetry/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' -- GitLab From 2ef3ba3e3181fd23797fe9ed707b1dbbbae4a85a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 11:09:30 +0000 Subject: [PATCH 31/70] Telemetry build added --- src/telemetry/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 1d63654d9..2a6b416be 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -13,7 +13,7 @@ # limitations under the License. # Build, tag, and push the Docker image to the GitLab Docker registry -build kpi-manager: +build telemetry: variables: IMAGE_NAME: 'telemetry' # name of the microservice IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) -- GitLab From 88bfacfb7ac904ff2eed954087f5f6d072ea05f8 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 9 Aug 2024 16:31:29 +0000 Subject: [PATCH 32/70] telemetry gitlab-ci file update. - docker build ... -target builder is removed --- src/telemetry/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 2a6b416be..78301fe07 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -22,7 +22,7 @@ build telemetry: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: # This first build tags the builder resulting image to prevent being removed by dangling image removal command - - docker buildx build -t "${IMAGE_NAME}-backend:${IMAGE_TAG}-builder" --target builder -f ./src/$IMAGE_NAME/backend/Dockerfile . + # - docker buildx build -t "${IMAGE_NAME}-backend:${IMAGE_TAG}-builder" --target builder -f ./src/$IMAGE_NAME/backend/Dockerfile . - docker buildx build -t "${IMAGE_NAME}-frontend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/frontend/Dockerfile . - docker buildx build -t "${IMAGE_NAME}-backend:$IMAGE_TAG" -f ./src/$IMAGE_NAME/backend/Dockerfile . - docker tag "${IMAGE_NAME}-frontend:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/${IMAGE_NAME}-frontend:$IMAGE_TAG" -- GitLab From 7c90ff4f6d2abdda7deb38c236230f111ad14afa Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 10 Aug 2024 05:37:00 +0000 Subject: [PATCH 33/70] telemetry .gitlab.ci file - docker run --name ... cmd missing (backend/frontend) after image name. --- src/telemetry/.gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index 78301fe07..c7950eb91 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -79,7 +79,7 @@ unit_test telemetry-backend: - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $KAFKA_IP - > - docker run --name $IMAGE_NAME -d -p 30060:30060 + docker run --name $IMAGE_NAME-backend -d -p 30060:30060 --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" --volume "$PWD/src/$IMAGE_NAME/backend/tests:/opt/results" --network=teraflowbridge @@ -159,9 +159,11 @@ unit_test telemetry-frontend: bitnami/kafka:latest - sleep 20 # Wait for Kafka to start - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - - echo $KAFKA_IP + - echo $KAFKA_IP + - docker logs zookeeper + - docker logs kafka - > - docker run --name $IMAGE_NAME -d -p 30050:30050 + docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" --env "KFK_SERVER_ADDRESS=${KAFKA_IP}:9092" --volume "$PWD/src/$IMAGE_NAME/frontend/tests:/opt/results" -- GitLab From 14cc7a763f087f3323932d7095825db464992f05 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 10 Aug 2024 05:40:41 +0000 Subject: [PATCH 34/70] changes in kpi Value API/writer gitlab-ci files. - src/$IMAGE_NAME/tests/Dockerfile is removed from API. - -f is added in "docker rm -f zookeeper" and "docker rm -f kafka" in Writer file. --- src/kpi_value_api/.gitlab-ci.yml | 2 +- src/kpi_value_writer/.gitlab-ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kpi_value_api/.gitlab-ci.yml b/src/kpi_value_api/.gitlab-ci.yml index 1919f0361..1a6f821ba 100644 --- a/src/kpi_value_api/.gitlab-ci.yml +++ b/src/kpi_value_api/.gitlab-ci.yml @@ -94,7 +94,7 @@ unit_test kpi-value-api: - src/$IMAGE_NAME/**/*.{py,in,yml} - src/$IMAGE_NAME/Dockerfile - src/$IMAGE_NAME/tests/*.py - - src/$IMAGE_NAME/tests/Dockerfile + # - src/$IMAGE_NAME/tests/Dockerfile # mayne not needed - manifests/${IMAGE_NAME}service.yaml - .gitlab-ci.yml artifacts: diff --git a/src/kpi_value_writer/.gitlab-ci.yml b/src/kpi_value_writer/.gitlab-ci.yml index 2c300db0e..9a2f9fd47 100644 --- a/src/kpi_value_writer/.gitlab-ci.yml +++ b/src/kpi_value_writer/.gitlab-ci.yml @@ -84,8 +84,8 @@ unit_test kpi-value-writer: coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' after_script: - docker rm -f $IMAGE_NAME - - docker rm zookeeper - - docker rm kafka + - docker rm -f zookeeper + - docker rm -f kafka - docker network rm teraflowbridge rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' -- GitLab From 89e7755e7bcf5dc3a98aebc91b2e13dac686b3ea Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 10 Aug 2024 06:07:15 +0000 Subject: [PATCH 35/70] changes in Metric Writer to Prometheus. - start_http_server() call is move to main - CollectorRegistory variable is removed --- .../service/MetricWriterToPrometheus.py | 12 ++---------- src/kpi_value_writer/service/__main__.py | 3 +++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py index f1d079783..4e6106255 100644 --- a/src/kpi_value_writer/service/MetricWriterToPrometheus.py +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -18,7 +18,7 @@ import ast import time import threading import logging -from prometheus_client import start_http_server, Gauge, CollectorRegistry +from prometheus_client import Gauge, CollectorRegistry from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.kpi_value_api_pb2 import KpiValue @@ -26,7 +26,6 @@ from common.proto.kpi_manager_pb2 import KpiDescriptor LOGGER = logging.getLogger(__name__) PROM_METRICS = {} -PROM_REGISTERY = CollectorRegistry() class MetricWriterToPrometheus: ''' @@ -34,13 +33,7 @@ class MetricWriterToPrometheus: cooked KPI value = KpiDescriptor (gRPC message) + KpiValue (gRPC message) ''' def __init__(self): - # prometheus server address and configs - self.start_prometheus_client() pass - - def start_prometheus_client(self): - start_http_server(10808, registry=PROM_REGISTERY) - LOGGER.debug("Prometheus client is started on port 10808") def merge_kpi_descriptor_and_kpi_value(self, kpi_descriptor, kpi_value): # Creating a dictionary from the kpi_descriptor's attributes @@ -71,8 +64,7 @@ class MetricWriterToPrometheus: PROM_METRICS[metric_name] = Gauge ( metric_name, cooked_kpi['kpi_description'], - metric_tags, - registry=PROM_REGISTERY + metric_tags ) LOGGER.debug("Metric is created with labels: {:}".format(metric_tags)) PROM_METRICS[metric_name].labels( diff --git a/src/kpi_value_writer/service/__main__.py b/src/kpi_value_writer/service/__main__.py index aa67540fb..be9f8f29b 100644 --- a/src/kpi_value_writer/service/__main__.py +++ b/src/kpi_value_writer/service/__main__.py @@ -13,6 +13,7 @@ # limitations under the License. import logging, signal, sys, threading +from prometheus_client import start_http_server from kpi_value_writer.service.KpiValueWriter import KpiValueWriter from common.Settings import get_log_level @@ -38,6 +39,8 @@ def main(): grpc_service = KpiValueWriter() grpc_service.start() + start_http_server(10808) + LOGGER.debug("Prometheus client is started on port 10808") # Wait for Ctrl+C or termination signal while not terminate.wait(timeout=1.0): pass -- GitLab From d1c99c6b51c30d40d08ad97ff145651e67601683 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 10 Aug 2024 06:32:53 +0000 Subject: [PATCH 36/70] Changes in Telemetry. - GenericGrpcService inheritance is added in Telemetry backend class. - Database folder were missing in working directory in DockerFile of Telemetry Frontend. --- src/telemetry/backend/service/TelemetryBackendService.py | 5 +++-- src/telemetry/frontend/Dockerfile | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index 6b9a6a8da..991298d37 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -24,6 +24,8 @@ from confluent_kafka import Consumer as KafkaConsumer from confluent_kafka import KafkaError from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from common.method_wrappers.Decorator import MetricsPool +from common.tools.service.GenericGrpcService import GenericGrpcService + LOGGER = logging.getLogger(__name__) @@ -31,12 +33,11 @@ METRICS_POOL = MetricsPool('TelemetryBackend', 'backendService') # EXPORTER_ENDPOINT = "http://10.152.183.2:9100/metrics" -class TelemetryBackendService: +class TelemetryBackendService(GenericGrpcService): """ Class listens for request on Kafka topic, fetches requested metrics from device. Produces metrics on both RESPONSE and VALUE kafka topics. """ - def __init__(self): LOGGER.info('Init TelemetryBackendService') self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) diff --git a/src/telemetry/frontend/Dockerfile b/src/telemetry/frontend/Dockerfile index 0c3e1a66a..7125d31fe 100644 --- a/src/telemetry/frontend/Dockerfile +++ b/src/telemetry/frontend/Dockerfile @@ -64,6 +64,7 @@ RUN python3 -m pip install -r requirements.txt WORKDIR /var/teraflow COPY src/telemetry/__init__.py telemetry/__init__.py COPY src/telemetry/frontend/. telemetry/frontend/ +COPY src/telemetry/database/. telemetry/database/ # Start the service ENTRYPOINT ["python", "-m", "telemetry.frontend.service"] -- GitLab From 25f71f10e3ea26cb6a2fb11c32dcb86507255092 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 10 Aug 2024 07:11:35 +0000 Subject: [PATCH 37/70] Changes on KPI Value Writer and Telemetry Backend - Renamed the method to "KafkaKpiConsumer" to avoid conflict with the "KafkaConsumer" import in KpiApiWriter. - Removed unnecessary imports in KpiWriterToProm. - Added `get_service_port_grpc` call and imports in the Telemetry backend service. - Added new libraries to `requirements.in` for Telemetry. --- src/kpi_value_writer/service/KpiValueWriter.py | 6 +++--- src/kpi_value_writer/service/MetricWriterToPrometheus.py | 8 ++++---- src/telemetry/backend/requirements.in | 4 ++++ src/telemetry/backend/service/TelemetryBackendService.py | 8 ++++++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py index 5d1a98808..eba651674 100644 --- a/src/kpi_value_writer/service/KpiValueWriter.py +++ b/src/kpi_value_writer/service/KpiValueWriter.py @@ -41,18 +41,18 @@ class KpiValueWriter(GenericGrpcService): @staticmethod def RunKafkaConsumer(): - thread = threading.Thread(target=KpiValueWriter.KafkaConsumer, args=()) + thread = threading.Thread(target=KpiValueWriter.KafkaKpiConsumer, args=()) ACTIVE_CONSUMERS.append(thread) thread.start() @staticmethod - def KafkaConsumer(): + def KafkaKpiConsumer(): kpi_manager_client = KpiManagerClient() metric_writer = MetricWriterToPrometheus() kafka_consumer = KafkaConsumer( { 'bootstrap.servers' : KafkaConfig.get_kafka_address(), - 'group.id' : __class__, + 'group.id' : 'KpiValueWriter', 'auto.offset.reset' : 'latest'} ) kafka_consumer.subscribe([KafkaTopic.VALUE.value]) diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py index 4e6106255..a86b8f34e 100644 --- a/src/kpi_value_writer/service/MetricWriterToPrometheus.py +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -14,11 +14,11 @@ # read Kafka stream from Kafka topic -import ast -import time -import threading +# import ast +# import time +# import threading import logging -from prometheus_client import Gauge, CollectorRegistry +from prometheus_client import Gauge from common.proto.kpi_sample_types_pb2 import KpiSampleType from common.proto.kpi_value_api_pb2 import KpiValue diff --git a/src/telemetry/backend/requirements.in b/src/telemetry/backend/requirements.in index e6a559be7..1d22df11b 100644 --- a/src/telemetry/backend/requirements.in +++ b/src/telemetry/backend/requirements.in @@ -13,3 +13,7 @@ # limitations under the License. confluent-kafka==2.3.* +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* \ No newline at end of file diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index 991298d37..bb9f0a314 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -18,10 +18,12 @@ import random import logging import threading from typing import Any, Dict -from common.proto.context_pb2 import Empty +# from common.proto.context_pb2 import Empty from confluent_kafka import Producer as KafkaProducer from confluent_kafka import Consumer as KafkaConsumer from confluent_kafka import KafkaError +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from common.method_wrappers.Decorator import MetricsPool from common.tools.service.GenericGrpcService import GenericGrpcService @@ -38,8 +40,10 @@ class TelemetryBackendService(GenericGrpcService): Class listens for request on Kafka topic, fetches requested metrics from device. Produces metrics on both RESPONSE and VALUE kafka topics. """ - def __init__(self): + def __init__(self, cls_name : str = __name__) -> None: LOGGER.info('Init TelemetryBackendService') + port = get_service_port_grpc(ServiceNameEnum.TELEMETRYBACKEND) + super().__init__(port, cls_name=cls_name) self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'backend', -- GitLab From 333b6d376e99dd129829073214b9cf36ee61acdd Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 10 Aug 2024 07:12:29 +0000 Subject: [PATCH 38/70] Added new libraries to requirements.in for Telemetry. --- src/telemetry/backend/requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry/backend/requirements.in b/src/telemetry/backend/requirements.in index 1d22df11b..231dc04e8 100644 --- a/src/telemetry/backend/requirements.in +++ b/src/telemetry/backend/requirements.in @@ -16,4 +16,4 @@ confluent-kafka==2.3.* psycopg2-binary==2.9.* SQLAlchemy==1.4.* sqlalchemy-cockroachdb==1.4.* -SQLAlchemy-Utils==0.38.* \ No newline at end of file +SQLAlchemy-Utils==0.38.* -- GitLab From 622fe067f9943cfbccc7967511628d1ad0fb1cb6 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sun, 11 Aug 2024 14:01:51 +0000 Subject: [PATCH 39/70] Changes in Telemetry. - tests file name is corrected. - Telmetry frontend and backend requriements.in is updated --- src/telemetry/backend/requirements.in | 4 -- .../backend/tests/test_TelemetryBackend.py | 38 +++++++++++++++++++ src/telemetry/frontend/requirements.in | 4 ++ 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 src/telemetry/backend/tests/test_TelemetryBackend.py diff --git a/src/telemetry/backend/requirements.in b/src/telemetry/backend/requirements.in index 231dc04e8..e6a559be7 100644 --- a/src/telemetry/backend/requirements.in +++ b/src/telemetry/backend/requirements.in @@ -13,7 +13,3 @@ # limitations under the License. confluent-kafka==2.3.* -psycopg2-binary==2.9.* -SQLAlchemy==1.4.* -sqlalchemy-cockroachdb==1.4.* -SQLAlchemy-Utils==0.38.* diff --git a/src/telemetry/backend/tests/test_TelemetryBackend.py b/src/telemetry/backend/tests/test_TelemetryBackend.py new file mode 100644 index 000000000..95710ff88 --- /dev/null +++ b/src/telemetry/backend/tests/test_TelemetryBackend.py @@ -0,0 +1,38 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from common.tools.kafka.Variables import KafkaTopic +from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService + + +LOGGER = logging.getLogger(__name__) + + +########################### +# Tests Implementation of Telemetry Backend +########################### + +# --- "test_validate_kafka_topics" should be run before the functionality tests --- +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) + +def test_RunRequestListener(): + LOGGER.info('test_RunRequestListener') + TelemetryBackendServiceObj = TelemetryBackendService() + response = TelemetryBackendServiceObj.RunRequestListener() + LOGGER.debug(str(response)) + assert isinstance(response, bool) diff --git a/src/telemetry/frontend/requirements.in b/src/telemetry/frontend/requirements.in index e6a559be7..231dc04e8 100644 --- a/src/telemetry/frontend/requirements.in +++ b/src/telemetry/frontend/requirements.in @@ -13,3 +13,7 @@ # limitations under the License. confluent-kafka==2.3.* +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* -- GitLab From 3de3bf9865aabccc9d5a2712fb59ccb4250349ca Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sun, 11 Aug 2024 15:18:06 +0000 Subject: [PATCH 40/70] Changes in KPI Manager KPI value API and value Writer. - updated cmd in test file of KPI manager - move kafka producer object to __init__ function. - write JSON object to Kafka - Read JSON object from Kafka - Slight to manage the affect of JSON object. - Static methods are removed. --- scripts/run_tests_locally-kpi-manager.sh | 2 +- src/common/tools/kafka/Variables.py | 2 +- .../service/KpiValueApiServiceServicerImpl.py | 42 ++++++++++++------- .../service/KpiValueWriter.py | 33 +++++++-------- .../service/MetricWriterToPrometheus.py | 12 +++--- .../tests/test_kpi_value_writer.py | 3 +- 6 files changed, 49 insertions(+), 45 deletions(-) diff --git a/scripts/run_tests_locally-kpi-manager.sh b/scripts/run_tests_locally-kpi-manager.sh index a6a24f90d..8a4ce8d95 100755 --- a/scripts/run_tests_locally-kpi-manager.sh +++ b/scripts/run_tests_locally-kpi-manager.sh @@ -24,7 +24,7 @@ cd $PROJECTDIR/src # python3 kpi_manager/tests/test_unitary.py RCFILE=$PROJECTDIR/coverage/.coveragerc -CRDB_SQL_ADDRESS=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.clusterIP}') +CRDB_SQL_ADDRESS=$(kubectl get service cockroachdb-public --namespace ${CRDB_NAMESPACE} -o 'jsonpath={.spec.clusterIP}') export CRDB_URI="cockroachdb://tfs:tfs123@${CRDB_SQL_ADDRESS}:26257/tfs_kpi_mgmt?sslmode=require" python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ kpi_manager/tests/test_kpi_manager.py diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 9d42f1550..5ada88a1e 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -55,7 +55,7 @@ class KafkaTopic(Enum): all_topics = [member.value for member in KafkaTopic] LOGGER.debug("Kafka server address is: {:} ".format(KafkaConfig.get_kafka_address())) if( KafkaTopic.create_new_topic_if_not_exists( all_topics )): - LOGGER.debug("All topics are created sucsessfully") + LOGGER.debug("All topics are created sucsessfully or Already Exists") return True else: LOGGER.debug("Error creating all topics") diff --git a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py index 3df8dd5b6..4ea978faf 100644 --- a/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py +++ b/src/kpi_value_api/service/KpiValueApiServiceServicerImpl.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, grpc -from typing import Tuple, Any +import logging, grpc, json +from typing import Dict from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.tools.kafka.Variables import KafkaConfig, KafkaTopic @@ -37,32 +37,42 @@ PROM_URL = "http://prometheus-k8s.monitoring.svc.cluster.local:9090" # TO class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): def __init__(self): LOGGER.debug('Init KpiValueApiService') - + self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StoreKpiValues(self, request: KpiValueList, grpc_context: grpc.ServicerContext ) -> Empty: LOGGER.debug('StoreKpiValues: Received gRPC message object: {:}'.format(request)) - producer_obj = KafkaProducer({ - 'bootstrap.servers' : KafkaConfig.get_kafka_address() - }) + + producer = self.kafka_producer for kpi_value in request.kpi_value_list: - kpi_value_to_produce : Tuple [str, Any, Any] = ( - kpi_value.kpi_id.kpi_id, - kpi_value.timestamp, - kpi_value.kpi_value_type # kpi_value.kpi_value_type.(many options) how? - ) + kpi_value_to_produce : Dict = { + "kpi_uuid" : kpi_value.kpi_id.kpi_id.uuid, + "timestamp" : kpi_value.timestamp.timestamp, + "kpi_value_type" : self.ExtractKpiValueByType(kpi_value.kpi_value_type) + } LOGGER.debug('KPI to produce is {:}'.format(kpi_value_to_produce)) msg_key = "gRPC-kpivalueapi" # str(__class__.__name__) can be used - producer_obj.produce( + producer.produce( KafkaTopic.VALUE.value, key = msg_key, - value = kpi_value.SerializeToString(), # value = json.dumps(kpi_value_to_produce), + value = json.dumps(kpi_value_to_produce), callback = self.delivery_callback ) - producer_obj.flush() + producer.flush() return Empty() + def ExtractKpiValueByType(self, value): + attributes = [ 'floatVal' , 'int32Val' , 'uint32Val','int64Val', + 'uint64Val', 'stringVal', 'boolVal'] + for attr in attributes: + try: + return getattr(value, attr) + except (ValueError, TypeError, AttributeError): + continue + return None + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SelectKpiValues(self, request: KpiValueFilter, grpc_context: grpc.ServicerContext ) -> KpiValueList: @@ -130,13 +140,13 @@ class KpiValueApiServiceServicerImpl(KpiValueAPIServiceServicer): try: int_value = int(value) return KpiValueType(int64Val=int_value) - except ValueError: + except (ValueError, TypeError): pass # Check if the value is a float try: float_value = float(value) return KpiValueType(floatVal=float_value) - except ValueError: + except (ValueError, TypeError): pass # Check if the value is a boolean if value.lower() in ['true', 'false']: diff --git a/src/kpi_value_writer/service/KpiValueWriter.py b/src/kpi_value_writer/service/KpiValueWriter.py index eba651674..8b258a142 100644 --- a/src/kpi_value_writer/service/KpiValueWriter.py +++ b/src/kpi_value_writer/service/KpiValueWriter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import threading from common.tools.kafka.Variables import KafkaConfig, KafkaTopic @@ -38,28 +39,25 @@ class KpiValueWriter(GenericGrpcService): def __init__(self, cls_name : str = __name__) -> None: port = get_service_port_grpc(ServiceNameEnum.KPIVALUEWRITER) super().__init__(port, cls_name=cls_name) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), + 'group.id' : 'KpiValueWriter', + 'auto.offset.reset' : 'latest'}) - @staticmethod - def RunKafkaConsumer(): - thread = threading.Thread(target=KpiValueWriter.KafkaKpiConsumer, args=()) + def RunKafkaConsumer(self): + thread = threading.Thread(target=self.KafkaKpiConsumer, args=()) ACTIVE_CONSUMERS.append(thread) thread.start() - @staticmethod - def KafkaKpiConsumer(): + def KafkaKpiConsumer(self): kpi_manager_client = KpiManagerClient() metric_writer = MetricWriterToPrometheus() - kafka_consumer = KafkaConsumer( - { 'bootstrap.servers' : KafkaConfig.get_kafka_address(), - 'group.id' : 'KpiValueWriter', - 'auto.offset.reset' : 'latest'} - ) - kafka_consumer.subscribe([KafkaTopic.VALUE.value]) + consumer = self.kafka_consumer + consumer.subscribe([KafkaTopic.VALUE.value]) LOGGER.debug("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) print("Kafka Consumer start listenng on topic: {:}".format(KafkaTopic.VALUE.value)) while True: - raw_kpi = kafka_consumer.poll(1.0) + raw_kpi = consumer.poll(1.0) if raw_kpi is None: continue elif raw_kpi.error(): @@ -69,24 +67,21 @@ class KpiValueWriter(GenericGrpcService): print("Consumer error: {}".format(raw_kpi.error())) continue try: - kpi_value = KpiValue() - kpi_value.ParseFromString(raw_kpi.value()) + kpi_value = json.loads(raw_kpi.value().decode('utf-8')) LOGGER.info("Received KPI : {:}".format(kpi_value)) print("Received KPI : {:}".format(kpi_value)) - KpiValueWriter.get_kpi_descriptor(kpi_value, kpi_manager_client, metric_writer) + self.get_kpi_descriptor(kpi_value, kpi_manager_client, metric_writer) except Exception as e: print("Error detail: {:}".format(e)) continue - @staticmethod - def get_kpi_descriptor(kpi_value: str, kpi_manager_client, metric_writer): + def get_kpi_descriptor(self, kpi_value: str, kpi_manager_client, metric_writer): print("--- START -----") kpi_id = KpiId() - kpi_id.kpi_id.uuid = kpi_value.kpi_id.kpi_id.uuid + kpi_id.kpi_id.uuid = kpi_value['kpi_uuid'] print("KpiId generated: {:}".format(kpi_id)) # print("Kpi manger client created: {:}".format(kpi_manager_client)) - try: kpi_descriptor_object = KpiDescriptor() kpi_descriptor_object = kpi_manager_client.GetKpiDescriptor(kpi_id) diff --git a/src/kpi_value_writer/service/MetricWriterToPrometheus.py b/src/kpi_value_writer/service/MetricWriterToPrometheus.py index a86b8f34e..85e618a4b 100644 --- a/src/kpi_value_writer/service/MetricWriterToPrometheus.py +++ b/src/kpi_value_writer/service/MetricWriterToPrometheus.py @@ -14,10 +14,8 @@ # read Kafka stream from Kafka topic -# import ast -# import time -# import threading import logging +from typing import Dict from prometheus_client import Gauge from common.proto.kpi_sample_types_pb2 import KpiSampleType @@ -47,13 +45,13 @@ class MetricWriterToPrometheus: 'slice_id' : kpi_descriptor.slice_id.slice_uuid.uuid, 'connection_id' : kpi_descriptor.connection_id.connection_uuid.uuid, 'link_id' : kpi_descriptor.link_id.link_uuid.uuid, - 'time_stamp' : kpi_value.timestamp.timestamp, - 'kpi_value' : kpi_value.kpi_value_type.floatVal + 'time_stamp' : kpi_value['timestamp'], + 'kpi_value' : kpi_value['kpi_value_type'] } # LOGGER.debug("Cooked Kpi: {:}".format(cooked_kpi)) return cooked_kpi - def create_and_expose_cooked_kpi(self, kpi_descriptor: KpiDescriptor, kpi_value: KpiValue): + def create_and_expose_cooked_kpi(self, kpi_descriptor: KpiDescriptor, kpi_value: Dict): # merge both gRPC messages into single varible. cooked_kpi = self.merge_kpi_descriptor_and_kpi_value(kpi_descriptor, kpi_value) tags_to_exclude = {'kpi_description', 'kpi_sample_type', 'kpi_value'} @@ -76,7 +74,7 @@ class MetricWriterToPrometheus: connection_id = cooked_kpi['connection_id'], link_id = cooked_kpi['link_id'], time_stamp = cooked_kpi['time_stamp'], - ).set(float(cooked_kpi['kpi_value'])) + ).set(cooked_kpi['kpi_value']) LOGGER.debug("Metric pushed to the endpoints: {:}".format(PROM_METRICS[metric_name])) except ValueError as e: diff --git a/src/kpi_value_writer/tests/test_kpi_value_writer.py b/src/kpi_value_writer/tests/test_kpi_value_writer.py index fce043d7f..b784fae5d 100755 --- a/src/kpi_value_writer/tests/test_kpi_value_writer.py +++ b/src/kpi_value_writer/tests/test_kpi_value_writer.py @@ -29,4 +29,5 @@ def test_validate_kafka_topics(): def test_KafkaConsumer(): LOGGER.debug(" --->>> test_kafka_consumer: START <<<--- ") - KpiValueWriter.RunKafkaConsumer() + kpi_value_writer = KpiValueWriter() + kpi_value_writer.RunKafkaConsumer() -- GitLab From 162e29e19c09a91caf57f7a3be408517caefe1cb Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sun, 11 Aug 2024 15:22:23 +0000 Subject: [PATCH 41/70] Minor Changes in Telemetry test file. - src folder refernce is removed from header --- .../backend/tests/testTelemetryBackend.py | 38 ------------------- .../backend/tests/test_TelemetryBackend.py | 2 +- 2 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 src/telemetry/backend/tests/testTelemetryBackend.py diff --git a/src/telemetry/backend/tests/testTelemetryBackend.py b/src/telemetry/backend/tests/testTelemetryBackend.py deleted file mode 100644 index 95710ff88..000000000 --- a/src/telemetry/backend/tests/testTelemetryBackend.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from common.tools.kafka.Variables import KafkaTopic -from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService - - -LOGGER = logging.getLogger(__name__) - - -########################### -# Tests Implementation of Telemetry Backend -########################### - -# --- "test_validate_kafka_topics" should be run before the functionality tests --- -def test_validate_kafka_topics(): - LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") - response = KafkaTopic.create_all_topics() - assert isinstance(response, bool) - -def test_RunRequestListener(): - LOGGER.info('test_RunRequestListener') - TelemetryBackendServiceObj = TelemetryBackendService() - response = TelemetryBackendServiceObj.RunRequestListener() - LOGGER.debug(str(response)) - assert isinstance(response, bool) diff --git a/src/telemetry/backend/tests/test_TelemetryBackend.py b/src/telemetry/backend/tests/test_TelemetryBackend.py index 95710ff88..a2bbee540 100644 --- a/src/telemetry/backend/tests/test_TelemetryBackend.py +++ b/src/telemetry/backend/tests/test_TelemetryBackend.py @@ -14,7 +14,7 @@ import logging from common.tools.kafka.Variables import KafkaTopic -from src.telemetry.backend.service.TelemetryBackendService import TelemetryBackendService +from telemetry.backend.service.TelemetryBackendService import TelemetryBackendService LOGGER = logging.getLogger(__name__) -- GitLab From bcc8c4901dd3173b2f941a52f310400c6fbffc8b Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 26 Aug 2024 10:48:48 +0000 Subject: [PATCH 42/70] Changes in Telemetry - Created the CRDB secret (`crdb-telemetry`). - Updated the IF conditions in the `$COMPONENT` deployment loop within the `tfs.sh` file according to telemetry service requirements. - Added the `Telemetryservice.yaml` file to the manifest folder. - Updated the CRDB URL in `TelemetryEngine.py`. - Made a minor formatting change in `TelemetryModel.py`. --- deploy/tfs.sh | 28 +++-- manifests/telemetryservice.yaml | 128 ++++++++++++++++++++++ src/telemetry/database/TelemetryEngine.py | 8 +- src/telemetry/database/TelemetryModel.py | 2 +- 4 files changed, 153 insertions(+), 13 deletions(-) create mode 100644 manifests/telemetryservice.yaml diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 4ecfaae99..e72014418 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -170,7 +170,19 @@ kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --t --from-literal=CRDB_SSLMODE=require printf "\n" -echo "Create secret with Apache Kafka kfk-kpi-data for KPI and Telemetry microservices" +echo "Create secret with CockroachDB data for Telemetry microservices" +CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') +CRDB_DATABASE_TELEMETRY="tfs_telemetry" # TODO: change by specific configurable environment variable +kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ + --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ + --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \ + --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ + --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ + --from-literal=CRDB_SSLMODE=require +printf "\n" + +echo "Create secret with Apache Kafka data for KPI and Telemetry microservices" KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}') kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \ @@ -252,15 +264,17 @@ for COMPONENT in $TFS_COMPONENTS; do if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" - elif [ "$COMPONENT" == "pathcomp" ]; then + elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" - # next command is redundant, but helpful to keep cache updated between rebuilds - IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" - $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + if [ "$COMPONENT" == "pathcomp" ]; then + # next command is redundant, but helpful to keep cache updated between rebuilds + IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" + $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + fi elif [ "$COMPONENT" == "dlt" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" @@ -273,7 +287,7 @@ for COMPONENT in $TFS_COMPONENTS; do echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..." - if [ "$COMPONENT" == "pathcomp" ]; then + if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" @@ -324,7 +338,7 @@ for COMPONENT in $TFS_COMPONENTS; do cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST" fi - if [ "$COMPONENT" == "pathcomp" ]; then + if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" diff --git a/manifests/telemetryservice.yaml b/manifests/telemetryservice.yaml new file mode 100644 index 000000000..2f9917499 --- /dev/null +++ b/manifests/telemetryservice.yaml @@ -0,0 +1,128 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: telemetryservice +spec: + selector: + matchLabels: + app: telemetryservice + #replicas: 1 + template: + metadata: + labels: + app: telemetryservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: frontend + image: labs.etsi.org:5050/tfs/controller/telemetry-frontend:latest + imagePullPolicy: Always + ports: + - containerPort: 30050 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + envFrom: + - secretRef: + name: crdb-telemetry + - secretRef: + name: kfk-kpi-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30050"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30050"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi + - name: backend + image: labs.etsi.org:5050/tfs/controller/telemetry-backend:latest + imagePullPolicy: Always + ports: + - containerPort: 30060 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + envFrom: + - secretRef: + name: kfk-kpi-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30060"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30060"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: telemetryservice + labels: + app: telemetryservice +spec: + type: ClusterIP + selector: + app: telemetryservice + ports: + - name: frontend-grpc + protocol: TCP + port: 30050 + targetPort: 30050 + - name: backend-grpc + protocol: TCP + port: 30060 + targetPort: 30060 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: telemetryservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: telemetryservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/src/telemetry/database/TelemetryEngine.py b/src/telemetry/database/TelemetryEngine.py index 965b7c38d..18ec2ddbc 100644 --- a/src/telemetry/database/TelemetryEngine.py +++ b/src/telemetry/database/TelemetryEngine.py @@ -17,8 +17,8 @@ from common.Settings import get_setting LOGGER = logging.getLogger(__name__) -CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' -# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' +# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' +CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' class TelemetryEngine: @staticmethod @@ -32,7 +32,7 @@ class TelemetryEngine: CRDB_PASSWORD = "tfs123" CRDB_SSLMODE = "require" crdb_uri = CRDB_URI_TEMPLATE.format( - CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: engine = sqlalchemy.create_engine(crdb_uri, echo=False) LOGGER.info(' TelemetryDB initalized with DB URL: {:}'.format(crdb_uri)) @@ -40,5 +40,3 @@ class TelemetryEngine: LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) return None # type: ignore return engine # type: ignore - - diff --git a/src/telemetry/database/TelemetryModel.py b/src/telemetry/database/TelemetryModel.py index 611ce7e70..4e71ce813 100644 --- a/src/telemetry/database/TelemetryModel.py +++ b/src/telemetry/database/TelemetryModel.py @@ -63,7 +63,7 @@ class Collector(Base): Args: row: The Collector table instance (row) containing the data. Returns: collector gRPC message initialized with the content of a row. """ - response = telemetry_frontend_pb2.Collector() + response = telemetry_frontend_pb2.Collector() response.collector_id.collector_id.uuid = row.collector_id response.kpi_id.kpi_id.uuid = row.kpi_id response.duration_s = row.sampling_duration_s -- GitLab From 7e8ae67a9732ea8c519bac43f58f886df06642ba Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 26 Aug 2024 14:56:03 +0000 Subject: [PATCH 43/70] Minor chanages in Telemetry. - Unnecessary echo statements are removed from .gitlab-ci.yml file. - "-e ALLOW_ANONYMOUS_LOGIN=yes" flag is added to allow unauthorized connection with the zookeeper. --- src/telemetry/.gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/telemetry/.gitlab-ci.yml b/src/telemetry/.gitlab-ci.yml index c7950eb91..110a6490d 100644 --- a/src/telemetry/.gitlab-ci.yml +++ b/src/telemetry/.gitlab-ci.yml @@ -145,12 +145,13 @@ unit_test telemetry-frontend: cockroachdb/cockroach:latest-v22.2 start-single-node - echo "Waiting for initialization..." - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done - - docker logs crdb - - docker ps -a + # - docker logs crdb + # - docker ps -a - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $CRDB_ADDRESS - > - docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 + docker run --name zookeeper -d --network=teraflowbridge -p 2181:2181 \ + -e ALLOW_ANONYMOUS_LOGIN=yes \ bitnami/zookeeper:latest - sleep 10 # Wait for Zookeeper to start - docker run --name kafka -d --network=teraflowbridge -p 9092:9092 @@ -160,8 +161,8 @@ unit_test telemetry-frontend: - sleep 20 # Wait for Kafka to start - KAFKA_IP=$(docker inspect kafka --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") - echo $KAFKA_IP - - docker logs zookeeper - - docker logs kafka + # - docker logs zookeeper + # - docker logs kafka - > docker run --name $IMAGE_NAME-frontend -d -p 30050:30050 --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" -- GitLab From 93fe4ea718d7bffa7ca73d996c023a5a4a0e7a5a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 27 Aug 2024 15:31:23 +0000 Subject: [PATCH 44/70] change in Analytics proto file. - AnalyzerId is added in "Analyzer" message type. --- proto/analytics_frontend.proto | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto index 096c1ee03..801fcdbc4 100644 --- a/proto/analytics_frontend.proto +++ b/proto/analytics_frontend.proto @@ -35,10 +35,11 @@ enum AnalyzerOperationMode { } message Analyzer { - string algorithm_name = 1; // The algorithm to be executed - repeated kpi_manager.KpiId input_kpi_ids = 2; // The KPI Ids to be processed by the analyzer - repeated kpi_manager.KpiId output_kpi_ids = 3; // The KPI Ids produced by the analyzer - AnalyzerOperationMode operation_mode = 4; // Operation mode of the analyzer + AnalyzerId analyzer_id = 1; + string algorithm_name = 2; // The algorithm to be executed + repeated kpi_manager.KpiId input_kpi_ids = 3; // The KPI Ids to be processed by the analyzer + repeated kpi_manager.KpiId output_kpi_ids = 4; // The KPI Ids produced by the analyzer + AnalyzerOperationMode operation_mode = 5; // Operation mode of the analyzer // In batch mode... float batch_min_duration_s = 5; // ..., min duration to collect before executing batch -- GitLab From 8e9f2eaf39b9a35a788130063c79da53e15296e7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 27 Aug 2024 15:35:53 +0000 Subject: [PATCH 45/70] changes in Analyzer proto file. - Assignement number was incorrect, 5 was assigned twice. --- proto/analytics_frontend.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto index 801fcdbc4..45e910a70 100644 --- a/proto/analytics_frontend.proto +++ b/proto/analytics_frontend.proto @@ -42,10 +42,10 @@ message Analyzer { AnalyzerOperationMode operation_mode = 5; // Operation mode of the analyzer // In batch mode... - float batch_min_duration_s = 5; // ..., min duration to collect before executing batch - float batch_max_duration_s = 6; // ..., max duration collected to execute the batch - uint64 batch_min_size = 7; // ..., min number of samples to collect before executing batch - uint64 batch_max_size = 8; // ..., max number of samples collected to execute the batch + float batch_min_duration_s = 6; // ..., min duration to collect before executing batch + float batch_max_duration_s = 7; // ..., max duration collected to execute the batch + uint64 batch_min_size = 8; // ..., min number of samples to collect before executing batch + uint64 batch_max_size = 9; // ..., max number of samples collected to execute the batch } message AnalyzerFilter { -- GitLab From e0a77d5f43a2a539beffe8681835bfa7f80f827e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 28 Aug 2024 09:58:22 +0000 Subject: [PATCH 46/70] Changes in Analytics Frontend. - Actul logic is added in StartAnalyzer method. - Actual logic is added in StopAnalyzer method. - Improvements in message formats. --- .../AnalyticsFrontendServiceServicerImpl.py | 80 +++++++++++++++++-- src/analytics/frontend/tests/messages.py | 16 +++- 2 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index b981c038b..071890105 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -13,22 +13,31 @@ # limitations under the License. -import logging, grpc +import logging, grpc, json +from typing import Dict +from confluent_kafka import Consumer as KafkaConsumer +from confluent_kafka import Producer as KafkaProducer +from confluent_kafka import KafkaError + +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from common.proto.context_pb2 import Empty from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method - from common.proto.analytics_frontend_pb2 import Analyzer, AnalyzerId, AnalyzerFilter, AnalyzerList from common.proto.analytics_frontend_pb2_grpc import AnalyticsFrontendServiceServicer -LOGGER = logging.getLogger(__name__) -METRICS_POOL = MetricsPool('AnalyticsFrontend', 'NBIgRPC') - +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('AnalyticsFrontend', 'NBIgRPC') +ACTIVE_ANALYZERS = [] # In case of sevice restarts, the list can be populated from the DB. class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): def __init__(self): LOGGER.info('Init AnalyticsFrontendService') + self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), + 'group.id' : 'analytics-frontend', + 'auto.offset.reset' : 'latest'}) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) @@ -37,17 +46,64 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): ) -> AnalyzerId: # type: ignore LOGGER.info ("At Service gRPC message: {:}".format(request)) response = AnalyzerId() - + self.PublishStartRequestOnKafka(request) return response - + + def PublishStartRequestOnKafka(self, analyzer_obj): + """ + Method to generate analyzer request on Kafka. + """ + analyzer_uuid = analyzer_obj.analyzer_id.analyzer_id.uuid + analyzer_to_generate : Dict = { + "algo_name" : analyzer_obj.algorithm_name, + "input_kpis" : [k.kpi_id.uuid for k in analyzer_obj.input_kpi_ids], + "output_kpis" : [k.kpi_id.uuid for k in analyzer_obj.output_kpi_ids], + "oper_mode" : analyzer_obj.operation_mode + } + self.kafka_producer.produce( + KafkaTopic.VALUE.value, + key = analyzer_uuid, + value = json.dumps(analyzer_to_generate), + callback = self.delivery_callback + ) + LOGGER.info("Analyzer Start Request Generated: Analyzer Id: {:}, Value: {:}".format(analyzer_uuid, analyzer_to_generate)) + ACTIVE_ANALYZERS.append(analyzer_uuid) + self.kafka_producer.flush() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StopAnalyzer(self, request : AnalyzerId, grpc_context: grpc.ServicerContext # type: ignore ) -> Empty: # type: ignore LOGGER.info ("At Service gRPC message: {:}".format(request)) - + self.PublishStopRequestOnKafka(request) return Empty() + def PublishStopRequestOnKafka(self, analyzer_id): + """ + Method to generate stop analyzer request on Kafka. + """ + analyzer_uuid = analyzer_id.analyzer_id.uuid + analyzer_to_stop : Dict = { + "algo_name" : -1, + "input_kpis" : [], + "output_kpis" : [], + "oper_mode" : -1 + } + self.kafka_producer.produce( + KafkaTopic.VALUE.value, + key = analyzer_uuid, + value = json.dumps(analyzer_to_stop), + callback = self.delivery_callback + ) + LOGGER.info("Analyzer Stop Request Generated: Analyzer Id: {:}".format(analyzer_uuid)) + self.kafka_producer.flush() + try: + ACTIVE_ANALYZERS.remove(analyzer_uuid) + except ValueError: + LOGGER.warning('Analyzer ID {:} not found in active analyzers'.format(analyzer_uuid)) + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SelectAnalyzers(self, request : AnalyzerFilter, contextgrpc_context: grpc.ServicerContext # type: ignore @@ -56,3 +112,11 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): response = AnalyzerList() return response + + def delivery_callback(self, err, msg): + if err: + LOGGER.debug('Message delivery failed: {:}'.format(err)) + print('Message delivery failed: {:}'.format(err)) + # else: + # LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) + # print('Message delivered to topic {:}'.format(msg.topic())) diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py index 1aaf8dd47..04653857d 100644 --- a/src/analytics/frontend/tests/messages.py +++ b/src/analytics/frontend/tests/messages.py @@ -28,9 +28,15 @@ def create_analyzer(): _create_analyzer.algorithm_name = "some_algo_name" _kpi_id = KpiId() - _kpi_id.kpi_id.uuid = str(uuid.uuid4()) # input IDs to analyze + # input IDs to analyze + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer.input_kpi_ids.append(_kpi_id) - _kpi_id.kpi_id.uuid = str(uuid.uuid4()) # output IDs after analysis + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_analyzer.input_kpi_ids.append(_kpi_id) + # output IDs after analysis + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _create_analyzer.output_kpi_ids.append(_kpi_id) + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer.output_kpi_ids.append(_kpi_id) _create_analyzer.operation_mode = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING @@ -49,10 +55,16 @@ def create_analyzer_filter(): _input_kpi_id_obj = KpiId() _input_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer_filter.input_kpi_ids.append(_input_kpi_id_obj) + # another input kpi Id + # _input_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + # _create_analyzer_filter.input_kpi_ids.append(_input_kpi_id_obj) _output_kpi_id_obj = KpiId() _output_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer_filter.output_kpi_ids.append(_output_kpi_id_obj) + # another output kpi Id + # _output_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + # _create_analyzer_filter.input_kpi_ids.append(_output_kpi_id_obj) return _create_analyzer_filter -- GitLab From 28bf80d3f22f3f38490ec96387dbaa414bd23557 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 28 Aug 2024 15:28:18 +0000 Subject: [PATCH 47/70] Changes in Analytic DB and Frontend - Created the Analytic Engine, Model, and DB files. - Added the DB connection. - Added `add_row_to_db` in `StartCollector`. - Added `delete_db_row_by_id` in `StopCollector`. - Improved message formatting. - Added a DB test file. --- scripts/run_tests_locally-analytics-DB.sh | 24 +++ .../run_tests_locally-analytics-frontend.sh | 3 +- src/analytics/database/AnalyzerEngine.py | 40 +++++ src/analytics/database/AnalyzerModel.py | 101 +++++++++++++ src/analytics/database/Analyzer_DB.py | 137 ++++++++++++++++++ src/analytics/database/__init__.py | 14 ++ .../AnalyticsFrontendServiceServicerImpl.py | 18 ++- src/analytics/frontend/tests/messages.py | 12 +- src/analytics/requirements.in | 21 +++ src/analytics/tests/test_analytics_db.py | 28 ++++ 10 files changed, 390 insertions(+), 8 deletions(-) create mode 100755 scripts/run_tests_locally-analytics-DB.sh create mode 100644 src/analytics/database/AnalyzerEngine.py create mode 100644 src/analytics/database/AnalyzerModel.py create mode 100644 src/analytics/database/Analyzer_DB.py create mode 100644 src/analytics/database/__init__.py create mode 100644 src/analytics/requirements.in create mode 100644 src/analytics/tests/test_analytics_db.py diff --git a/scripts/run_tests_locally-analytics-DB.sh b/scripts/run_tests_locally-analytics-DB.sh new file mode 100755 index 000000000..9df5068d6 --- /dev/null +++ b/scripts/run_tests_locally-analytics-DB.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc +CRDB_SQL_ADDRESS=$(kubectl get service cockroachdb-public --namespace crdb -o jsonpath='{.spec.clusterIP}') +export CRDB_URI="cockroachdb://tfs:tfs123@${CRDB_SQL_ADDRESS}:26257/tfs_kpi_mgmt?sslmode=require" +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + analytics/tests/test_analytics_db.py diff --git a/scripts/run_tests_locally-analytics-frontend.sh b/scripts/run_tests_locally-analytics-frontend.sh index 58fd062f2..e30d30da6 100755 --- a/scripts/run_tests_locally-analytics-frontend.sh +++ b/scripts/run_tests_locally-analytics-frontend.sh @@ -17,7 +17,8 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src - RCFILE=$PROJECTDIR/coverage/.coveragerc +CRDB_SQL_ADDRESS=$(kubectl get service cockroachdb-public --namespace crdb -o jsonpath='{.spec.clusterIP}') +export CRDB_URI="cockroachdb://tfs:tfs123@${CRDB_SQL_ADDRESS}:26257/tfs_kpi_mgmt?sslmode=require" python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ analytics/frontend/tests/test_frontend.py diff --git a/src/analytics/database/AnalyzerEngine.py b/src/analytics/database/AnalyzerEngine.py new file mode 100644 index 000000000..4bed9f93a --- /dev/null +++ b/src/analytics/database/AnalyzerEngine.py @@ -0,0 +1,40 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, sqlalchemy +from common.Settings import get_setting + +LOGGER = logging.getLogger(__name__) +CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' + +class AnalyzerEngine: + @staticmethod + def get_engine() -> sqlalchemy.engine.Engine: + crdb_uri = get_setting('CRDB_URI', default=None) + if crdb_uri is None: + CRDB_NAMESPACE = "crdb" + CRDB_SQL_PORT = "26257" + CRDB_DATABASE = "tfs-analyzer" + CRDB_USERNAME = "tfs" + CRDB_PASSWORD = "tfs123" + CRDB_SSLMODE = "require" + crdb_uri = CRDB_URI_TEMPLATE.format( + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + try: + engine = sqlalchemy.create_engine(crdb_uri, echo=False) + LOGGER.info(' AnalyzerDB initalized with DB URL: {:}'.format(crdb_uri)) + except: + LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) + return None + return engine diff --git a/src/analytics/database/AnalyzerModel.py b/src/analytics/database/AnalyzerModel.py new file mode 100644 index 000000000..0b66e04d0 --- /dev/null +++ b/src/analytics/database/AnalyzerModel.py @@ -0,0 +1,101 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import enum + +from sqlalchemy import Column, String, Float, Enum +from sqlalchemy.orm import registry +from common.proto import analytics_frontend_pb2 +from common.proto import kpi_manager_pb2 + +from sqlalchemy.dialects.postgresql import UUID, ARRAY + + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + +# Create a base class for declarative models +Base = registry().generate_base() + +class AnalyzerOperationMode (enum.Enum): + BATCH = analytics_frontend_pb2.AnalyzerOperationMode.ANALYZEROPERATIONMODE_BATCH + STREAMING = analytics_frontend_pb2.AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING + +class Analyzer(Base): + __tablename__ = 'analyzer' + + analyzer_id = Column(UUID(as_uuid=False) , primary_key=True) + algorithm_name = Column(String , nullable=False) + input_kpi_ids = Column(ARRAY(UUID(as_uuid=False)) , nullable=False) + output_kpi_ids = Column(ARRAY(UUID(as_uuid=False)) , nullable=False) + operation_mode = Column(Enum(AnalyzerOperationMode), nullable=False) + batch_min_duration_s = Column(Float , nullable=False) + batch_max_duration_s = Column(Float , nullable=False) + bacth_min_size = Column(Float , nullable=False) + bacth_max_size = Column(Float , nullable=False) + + # helps in logging the information + def __repr__(self): + return (f"") + + @classmethod + def ConvertAnalyzerToRow(cls, request): + """ + Create an instance of Analyzer table rows from a request object. + Args: request: The request object containing analyzer gRPC message. + Returns: A row (an instance of Analyzer table) initialized with content of the request. + """ + return cls( + analyzer_id = request.analyzer_id.analyzer_id.uuid, + algorithm_name = request.algorithm_name, + input_kpi_ids = [k.kpi_id.uuid for k in request.input_kpi_ids], + output_kpi_ids = [k.kpi_id.uuid for k in request.output_kpi_ids], + operation_mode = AnalyzerOperationMode(request.operation_mode), # converts integer to coresponding Enum class member + batch_min_duration_s = request.batch_min_duration_s, + batch_max_duration_s = request.batch_max_duration_s, + bacth_min_size = request.batch_min_size, + bacth_max_size = request.batch_max_size + ) + + @classmethod + def ConvertRowToAnalyzer(cls, row): + """ + Create and return an Analyzer gRPC message initialized with the content of a row. + Args: row: The Analyzer table instance (row) containing the data. + Returns: An Analyzer gRPC message initialized with the content of the row. + """ + # Create an instance of the Analyzer message + response = analytics_frontend_pb2.Analyzer() + response.analyzer_id.analyzer_id.uuid = row.analyzer_id + response.algorithm_name = row.algorithm_name + response.operation_mode = row.operation_mode + + _kpi_id = kpi_manager_pb2.KpiId() + for input_kpi_id in row.input_kpi_ids: + _kpi_id.kpi_id.uuid = input_kpi_id + response.input_kpi_ids.append(_kpi_id) + for output_kpi_id in row.output_kpi_ids: + _kpi_id.kpi_id.uuid = output_kpi_id + response.output_kpi_ids.append(_kpi_id) + + response.batch_min_duration_s = row.batch_min_duration_s + response.batch_max_duration_s = row.batch_max_duration_s + response.batch_min_size = row.bacth_min_size + response.batch_max_size = row.bacth_max_size + return response diff --git a/src/analytics/database/Analyzer_DB.py b/src/analytics/database/Analyzer_DB.py new file mode 100644 index 000000000..896ba1100 --- /dev/null +++ b/src/analytics/database/Analyzer_DB.py @@ -0,0 +1,137 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sqlalchemy_utils + +from sqlalchemy import inspect +from sqlalchemy.orm import sessionmaker + +from analytics.database.AnalyzerModel import Analyzer as AnalyzerModel +from analytics.database.AnalyzerEngine import AnalyzerEngine +from common.method_wrappers.ServiceExceptions import (OperationFailedException, AlreadyExistsException) + +LOGGER = logging.getLogger(__name__) +DB_NAME = "tfs_analyzer" # TODO: export name from enviornment variable + +class AnalyzerDB: + def __init__(self): + self.db_engine = AnalyzerEngine.get_engine() + if self.db_engine is None: + LOGGER.error('Unable to get SQLAlchemy DB Engine...') + return False + self.db_name = DB_NAME + self.Session = sessionmaker(bind=self.db_engine) + + def create_database(self): + if not sqlalchemy_utils.database_exists(self.db_engine.url): + LOGGER.debug("Database created. {:}".format(self.db_engine.url)) + sqlalchemy_utils.create_database(self.db_engine.url) + + def drop_database(self) -> None: + if sqlalchemy_utils.database_exists(self.db_engine.url): + sqlalchemy_utils.drop_database(self.db_engine.url) + + def create_tables(self): + try: + AnalyzerModel.metadata.create_all(self.db_engine) # type: ignore + LOGGER.debug("Tables created in the database: {:}".format(self.db_name)) + except Exception as e: + LOGGER.debug("Tables cannot be created in the database. {:s}".format(str(e))) + raise OperationFailedException ("Tables can't be created", extra_details=["unable to create table {:}".format(e)]) + + def verify_tables(self): + try: + inspect_object = inspect(self.db_engine) + if(inspect_object.has_table('analyzer', None)): + LOGGER.info("Table exists in DB: {:}".format(self.db_name)) + except Exception as e: + LOGGER.info("Unable to fetch Table names. {:s}".format(str(e))) + +# ----------------- CURD OPERATIONS --------------------- + + def add_row_to_db(self, row): + session = self.Session() + try: + session.add(row) + session.commit() + LOGGER.debug(f"Row inserted into {row.__class__.__name__} table.") + return True + except Exception as e: + session.rollback() + if "psycopg2.errors.UniqueViolation" in str(e): + LOGGER.error(f"Unique key voilation: {row.__class__.__name__} table. {str(e)}") + raise AlreadyExistsException(row.__class__.__name__, row, + extra_details=["Unique key voilation: {:}".format(e)] ) + else: + LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}") + raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) + finally: + session.close() + + def search_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + entity = session.query(model).filter_by(**{col_name: id_to_search}).first() + if entity: + # LOGGER.debug(f"{model.__name__} ID found: {str(entity)}") + return entity + else: + LOGGER.debug(f"{model.__name__} ID not found, No matching row: {str(id_to_search)}") + print("{:} ID not found, No matching row: {:}".format(model.__name__, id_to_search)) + return None + except Exception as e: + session.rollback() + LOGGER.debug(f"Failed to retrieve {model.__name__} ID. {str(e)}") + raise OperationFailedException ("search by column id", extra_details=["unable to search row {:}".format(e)]) + finally: + session.close() + + def delete_db_row_by_id(self, model, col_name, id_to_search): + session = self.Session() + try: + record = session.query(model).filter_by(**{col_name: id_to_search}).first() + if record: + session.delete(record) + session.commit() + LOGGER.debug("Deleted %s with %s: %s", model.__name__, col_name, id_to_search) + else: + LOGGER.debug("%s with %s %s not found", model.__name__, col_name, id_to_search) + return None + except Exception as e: + session.rollback() + LOGGER.error("Error deleting %s with %s %s: %s", model.__name__, col_name, id_to_search, e) + raise OperationFailedException ("Deletion by column id", extra_details=["unable to delete row {:}".format(e)]) + finally: + session.close() + + def select_with_filter(self, model, filter_object): + session = self.Session() + try: + query = session.query(AnalyzerModel) + # Apply filters based on the filter_object + if filter_object.kpi_id: + query = query.filter(AnalyzerModel.kpi_id.in_([k.kpi_id.uuid for k in filter_object.kpi_id])) # Need to be updated + result = query.all() + # query should be added to return all rows + if result: + LOGGER.debug(f"Fetched filtered rows from {model.__name__} table with filters: {filter_object}") # - Results: {result} + else: + LOGGER.warning(f"No matching row found in {model.__name__} table with filters: {filter_object}") + return result + except Exception as e: + LOGGER.error(f"Error fetching filtered rows from {model.__name__} table with filters {filter_object} ::: {e}") + raise OperationFailedException ("Select by filter", extra_details=["unable to apply the filter {:}".format(e)]) + finally: + session.close() diff --git a/src/analytics/database/__init__.py b/src/analytics/database/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/analytics/database/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index 071890105..ccbef3599 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -25,6 +25,8 @@ from common.proto.context_pb2 import Empty from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.analytics_frontend_pb2 import Analyzer, AnalyzerId, AnalyzerFilter, AnalyzerList from common.proto.analytics_frontend_pb2_grpc import AnalyticsFrontendServiceServicer +from analytics.database.Analyzer_DB import AnalyzerDB +from analytics.database.AnalyzerModel import Analyzer as AnalyzerModel LOGGER = logging.getLogger(__name__) @@ -34,6 +36,7 @@ ACTIVE_ANALYZERS = [] # In case of sevice restarts, the list can be populated class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): def __init__(self): LOGGER.info('Init AnalyticsFrontendService') + self.db_obj = AnalyzerDB() self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'analytics-frontend', @@ -46,7 +49,13 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): ) -> AnalyzerId: # type: ignore LOGGER.info ("At Service gRPC message: {:}".format(request)) response = AnalyzerId() + + self.db_obj.add_row_to_db( + AnalyzerModel.ConvertAnalyzerToRow(request) + ) self.PublishStartRequestOnKafka(request) + + response.analyzer_id.uuid = request.analyzer_id.analyzer_id.uuid return response def PublishStartRequestOnKafka(self, analyzer_obj): @@ -76,9 +85,16 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): request : AnalyzerId, grpc_context: grpc.ServicerContext # type: ignore ) -> Empty: # type: ignore LOGGER.info ("At Service gRPC message: {:}".format(request)) + try: + analyzer_id_to_delete = request.analyzer_id.uuid + self.db_obj.delete_db_row_by_id( + AnalyzerModel, "analyzer_id", analyzer_id_to_delete + ) + except Exception as e: + LOGGER.warning('Unable to delete analyzer. Error: {:}'.format(e)) self.PublishStopRequestOnKafka(request) return Empty() - + def PublishStopRequestOnKafka(self, analyzer_id): """ Method to generate stop analyzer request on Kafka. diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py index 04653857d..4c826e5c3 100644 --- a/src/analytics/frontend/tests/messages.py +++ b/src/analytics/frontend/tests/messages.py @@ -19,13 +19,15 @@ from common.proto.analytics_frontend_pb2 import ( AnalyzerOperationMode, Analyze def create_analyzer_id(): _create_analyzer_id = AnalyzerId() - _create_analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) + # _create_analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) + _create_analyzer_id.analyzer_id.uuid = "9baa306d-d91c-48c2-a92f-76a21bab35b6" return _create_analyzer_id def create_analyzer(): - _create_analyzer = Analyzer() - - _create_analyzer.algorithm_name = "some_algo_name" + _create_analyzer = Analyzer() + _create_analyzer.analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) + _create_analyzer.algorithm_name = "some_algo_name" + _create_analyzer.operation_mode = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING _kpi_id = KpiId() # input IDs to analyze @@ -39,8 +41,6 @@ def create_analyzer(): _kpi_id.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer.output_kpi_ids.append(_kpi_id) - _create_analyzer.operation_mode = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING - return _create_analyzer def create_analyzer_filter(): diff --git a/src/analytics/requirements.in b/src/analytics/requirements.in new file mode 100644 index 000000000..98cf96710 --- /dev/null +++ b/src/analytics/requirements.in @@ -0,0 +1,21 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +java==11.0.* +pyspark==3.5.2 +confluent-kafka==2.3.* +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* \ No newline at end of file diff --git a/src/analytics/tests/test_analytics_db.py b/src/analytics/tests/test_analytics_db.py new file mode 100644 index 000000000..58e7d0167 --- /dev/null +++ b/src/analytics/tests/test_analytics_db.py @@ -0,0 +1,28 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from analytics.database.Analyzer_DB import AnalyzerDB + +LOGGER = logging.getLogger(__name__) + +def test_verify_databases_and_tables(): + LOGGER.info('>>> test_verify_databases_and_tables : START <<< ') + AnalyzerDBobj = AnalyzerDB() + # AnalyzerDBobj.drop_database() + # AnalyzerDBobj.verify_tables() + AnalyzerDBobj.create_database() + AnalyzerDBobj.create_tables() + AnalyzerDBobj.verify_tables() -- GitLab From 0c5d285636e88abaacdc5f44d70897cb63817763 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 3 Sep 2024 09:40:01 +0000 Subject: [PATCH 48/70] Analytics Backend Initial Version - Added CRDB secret. - Added test script. - Added `main` and `__init__.py` files. - Added Backend Service file. - Added Spark Streamer file (successfully consuming streams from Kafka Topics). - Added Tests folder and files. - Added requirements file. - Added ANALYTICS Kafka topics in `common.tools.kafka.variables` file. --- deploy/tfs.sh | 12 +++ .../run_tests_locally-analytics-backend.sh | 22 +++++ src/analytics/backend/__init__.py | 13 +++ src/analytics/backend/requirements.in | 17 ++++ .../service/AnalyticsBackendService.py | 29 +++++++ .../backend/service/SparkStreaming.py | 84 +++++++++++++++++++ src/analytics/backend/service/__init__.py | 13 +++ src/analytics/backend/service/__main__.py | 51 +++++++++++ src/analytics/backend/tests/__init__.py | 13 +++ src/analytics/backend/tests/messages.py | 15 ++++ src/analytics/backend/tests/test_backend.py | 37 ++++++++ src/common/tools/kafka/Variables.py | 13 +-- 12 files changed, 314 insertions(+), 5 deletions(-) create mode 100755 scripts/run_tests_locally-analytics-backend.sh create mode 100644 src/analytics/backend/__init__.py create mode 100644 src/analytics/backend/requirements.in create mode 100755 src/analytics/backend/service/AnalyticsBackendService.py create mode 100644 src/analytics/backend/service/SparkStreaming.py create mode 100644 src/analytics/backend/service/__init__.py create mode 100644 src/analytics/backend/service/__main__.py create mode 100644 src/analytics/backend/tests/__init__.py create mode 100644 src/analytics/backend/tests/messages.py create mode 100644 src/analytics/backend/tests/test_backend.py diff --git a/deploy/tfs.sh b/deploy/tfs.sh index e72014418..1aa32684b 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -179,6 +179,18 @@ kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} -- --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \ --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ + --from-literSLMODE=require +printf "\n" + +echo "Create secret with CockroachDB data for Analytics microservices" +CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}') +CRDB_DATABASE_ANALYTICS="tfs_analytics" # TODO: change by specific configurable environment variable +kubectl create secret generic crdb-analytics --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ + --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \ + --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \ + --from-literal=CRDB_DATABASE=${CRDB_DATABASE_ANALYTICS} \ + --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ + --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ --from-literal=CRDB_SSLMODE=require printf "\n" diff --git a/scripts/run_tests_locally-analytics-backend.sh b/scripts/run_tests_locally-analytics-backend.sh new file mode 100755 index 000000000..24afd456d --- /dev/null +++ b/scripts/run_tests_locally-analytics-backend.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \ + analytics/backend/tests/test_backend.py diff --git a/src/analytics/backend/__init__.py b/src/analytics/backend/__init__.py new file mode 100644 index 000000000..bbfc943b6 --- /dev/null +++ b/src/analytics/backend/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/analytics/backend/requirements.in b/src/analytics/backend/requirements.in new file mode 100644 index 000000000..5c2280c5d --- /dev/null +++ b/src/analytics/backend/requirements.in @@ -0,0 +1,17 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +java==11.0.* +pyspark==3.5.2 +confluent-kafka==2.3.* \ No newline at end of file diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py new file mode 100755 index 000000000..84f1887c6 --- /dev/null +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -0,0 +1,29 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +from common.tools.service.GenericGrpcService import GenericGrpcService +from analytics.backend.service.SparkStreaming import SparkStreamer + + +class AnalyticsBackendService(GenericGrpcService): + """ + Class listens for ... + """ + def __init__(self, cls_name : str = __name__) -> None: + pass + + def RunSparkStreamer(self): + threading.Thread(target=SparkStreamer).start() diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py new file mode 100644 index 000000000..92cc9a842 --- /dev/null +++ b/src/analytics/backend/service/SparkStreaming.py @@ -0,0 +1,84 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from pyspark.sql import SparkSession +from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType +from pyspark.sql.functions import from_json, col +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic + +LOGGER = logging.getLogger(__name__) + +def DefiningSparkSession(): + # Create a Spark session with specific spark verions (3.5.0) + return SparkSession.builder \ + .appName("Analytics") \ + .config("spark.jars.packages", "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0") \ + .getOrCreate() + +def SettingKafkaParameters(): # TODO: create get_kafka_consumer() in common with inputs (bootstrap server, subscribe, startingOffset and failOnDataLoss with default values) + return { + # "kafka.bootstrap.servers": '127.0.0.1:9092', + "kafka.bootstrap.servers": KafkaConfig.get_kafka_address(), + "subscribe" : KafkaTopic.ANALYTICS_REQUEST.value, + "startingOffsets" : 'latest', + "failOnDataLoss" : 'false' # Optional: Set to "true" to fail the query on data loss + } + +def DefiningRequestSchema(): + return StructType([ + StructField("algo_name", StringType() , True), + StructField("input_kpis", ArrayType(StringType()), True), + StructField("output_kpis", ArrayType(StringType()), True), + StructField("oper_mode", IntegerType() , True) + ]) + +def SparkStreamer(): + """ + Method to perform Spark operation Kafka stream. + NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session. + """ + kafka_params = SettingKafkaParameters() # Define the Kafka parameters + schema = DefiningRequestSchema() # Define the schema for the incoming JSON data + spark = DefiningSparkSession() # Define the spark session with app name and spark version + + try: + # Read data from Kafka + raw_stream_data = spark \ + .readStream \ + .format("kafka") \ + .options(**kafka_params) \ + .load() + + # Convert the value column from Kafka to a string + stream_data = raw_stream_data.selectExpr("CAST(value AS STRING)") + # Parse the JSON string into a DataFrame with the defined schema + parsed_stream_data = stream_data.withColumn("parsed_value", from_json(col("value"), schema)) + # Select the parsed fields + final_stream_data = parsed_stream_data.select("parsed_value.*") + + # Start the Spark streaming query + query = final_stream_data \ + .writeStream \ + .outputMode("append") \ + .format("console") # You can change this to other output modes or sinks + + # Start the query execution + query.start().awaitTermination() + except Exception as e: + print("Error in Spark streaming process: {:}".format(e)) + LOGGER.debug("Error in Spark streaming process: {:}".format(e)) + finally: + spark.stop() diff --git a/src/analytics/backend/service/__init__.py b/src/analytics/backend/service/__init__.py new file mode 100644 index 000000000..bbfc943b6 --- /dev/null +++ b/src/analytics/backend/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/analytics/backend/service/__main__.py b/src/analytics/backend/service/__main__.py new file mode 100644 index 000000000..371b5a7ca --- /dev/null +++ b/src/analytics/backend/service/__main__.py @@ -0,0 +1,51 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Settings import get_log_level +from analytics.backend.service.AnalyticsBackendService import AnalyticsBackendService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = AnalyticsBackendService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/analytics/backend/tests/__init__.py b/src/analytics/backend/tests/__init__.py new file mode 100644 index 000000000..bbfc943b6 --- /dev/null +++ b/src/analytics/backend/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/analytics/backend/tests/messages.py b/src/analytics/backend/tests/messages.py new file mode 100644 index 000000000..5cf553eaa --- /dev/null +++ b/src/analytics/backend/tests/messages.py @@ -0,0 +1,15 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/analytics/backend/tests/test_backend.py b/src/analytics/backend/tests/test_backend.py new file mode 100644 index 000000000..7a6175ecf --- /dev/null +++ b/src/analytics/backend/tests/test_backend.py @@ -0,0 +1,37 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from common.tools.kafka.Variables import KafkaTopic +from analytics.backend.service.AnalyticsBackendService import AnalyticsBackendService + + +LOGGER = logging.getLogger(__name__) + + +########################### +# Tests Implementation of Telemetry Backend +########################### + +# --- "test_validate_kafka_topics" should be run before the functionality tests --- +# def test_validate_kafka_topics(): +# LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") +# response = KafkaTopic.create_all_topics() +# assert isinstance(response, bool) + +def test_SparkListener(): + LOGGER.info('test_RunRequestListener') + AnalyticsBackendServiceObj = AnalyticsBackendService() + response = AnalyticsBackendServiceObj.RunSparkStreamer() + LOGGER.debug(str(response)) diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 5ada88a1e..215913c0e 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -41,11 +41,14 @@ class KafkaConfig(Enum): class KafkaTopic(Enum): - REQUEST = 'topic_request' - RESPONSE = 'topic_response' - RAW = 'topic_raw' - LABELED = 'topic_labeled' - VALUE = 'topic_value' + # TODO: Later to be populated from ENV variable. + REQUEST = 'topic_request' + RESPONSE = 'topic_response' + RAW = 'topic_raw' + LABELED = 'topic_labeled' + VALUE = 'topic_value' + ANALYTICS_REQUEST = 'topic_request_analytics' + ANALYTICS_RESPONSE = 'topic_response_analytics' @staticmethod def create_all_topics() -> bool: -- GitLab From c8ad3a5cc6aea20d6cf87d8ebf0c8f54eeb28b31 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 3 Sep 2024 09:45:12 +0000 Subject: [PATCH 49/70] **Changes in Frontend** - Added the missing `map` in the `analytics_frontend.proto` file. - Removed `pyspark` and `java` from the `requirements.in` file, as they are not required for the Frontend. --- proto/analytics_frontend.proto | 20 ++++++++++---------- src/analytics/frontend/requirements.in | 2 -- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto index 45e910a70..bd28a21bf 100644 --- a/proto/analytics_frontend.proto +++ b/proto/analytics_frontend.proto @@ -36,16 +36,16 @@ enum AnalyzerOperationMode { message Analyzer { AnalyzerId analyzer_id = 1; - string algorithm_name = 2; // The algorithm to be executed - repeated kpi_manager.KpiId input_kpi_ids = 3; // The KPI Ids to be processed by the analyzer - repeated kpi_manager.KpiId output_kpi_ids = 4; // The KPI Ids produced by the analyzer - AnalyzerOperationMode operation_mode = 5; // Operation mode of the analyzer - - // In batch mode... - float batch_min_duration_s = 6; // ..., min duration to collect before executing batch - float batch_max_duration_s = 7; // ..., max duration collected to execute the batch - uint64 batch_min_size = 8; // ..., min number of samples to collect before executing batch - uint64 batch_max_size = 9; // ..., max number of samples collected to execute the batch + string algorithm_name = 2; // The algorithm to be executed + repeated kpi_manager.KpiId input_kpi_ids = 3; // The KPI Ids to be processed by the analyzer + repeated kpi_manager.KpiId output_kpi_ids = 4; // The KPI Ids produced by the analyzer + AnalyzerOperationMode operation_mode = 5; // Operation mode of the analyzer + map parameters = 6; + // In batch mode... + float batch_min_duration_s = 7; // ..., min duration to collect before executing batch + float batch_max_duration_s = 8; // ..., max duration collected to execute the batch + uint64 batch_min_size = 9; // ..., min number of samples to collect before executing batch + uint64 batch_max_size = 10; // ..., max number of samples collected to execute the batch } message AnalyzerFilter { diff --git a/src/analytics/frontend/requirements.in b/src/analytics/frontend/requirements.in index 98cf96710..1d22df11b 100644 --- a/src/analytics/frontend/requirements.in +++ b/src/analytics/frontend/requirements.in @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -java==11.0.* -pyspark==3.5.2 confluent-kafka==2.3.* psycopg2-binary==2.9.* SQLAlchemy==1.4.* -- GitLab From 067ace160dcef4b3d07923bcece19bfef3f50a10 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 3 Sep 2024 13:54:24 +0000 Subject: [PATCH 50/70] Changes in Telemetry - Added Kafka Request Listener to listen for requests from the Analytics Frontend. - Updated `SparkStreamer.py` to consume and process streams from VALUE topics, and further filter rows based on KPIs in the `input_kpis` list. - Updated the frontend TOPIC from `VALUE` to `ANALYTICS_REQUEST`. - Updated messages to keep the KPI UUID consistent with the one generated by the Telemetry Backend service. --- .../service/AnalyticsBackendService.py | 42 +++++++++++++++++-- .../backend/service/SparkStreaming.py | 19 +++++---- src/analytics/backend/tests/test_backend.py | 11 ++++- .../AnalyticsFrontendServiceServicerImpl.py | 4 +- src/analytics/frontend/tests/messages.py | 4 ++ .../backend/tests/test_TelemetryBackend.py | 8 ++-- 6 files changed, 67 insertions(+), 21 deletions(-) diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 84f1887c6..5331d027d 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -12,18 +12,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -import threading +import json +import logging +import threading from common.tools.service.GenericGrpcService import GenericGrpcService from analytics.backend.service.SparkStreaming import SparkStreamer +from common.tools.kafka.Variables import KafkaConfig, KafkaTopic +from confluent_kafka import Consumer as KafkaConsumer +from confluent_kafka import KafkaError +LOGGER = logging.getLogger(__name__) class AnalyticsBackendService(GenericGrpcService): """ Class listens for ... """ def __init__(self, cls_name : str = __name__) -> None: - pass + self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), + 'group.id' : 'analytics-frontend', + 'auto.offset.reset' : 'latest'}) + + def RunSparkStreamer(self, kpi_list): + threading.Thread(target=SparkStreamer, args=(kpi_list,)).start() + + def RunRequestListener(self)->bool: + threading.Thread(target=self.RequestListener).start() + return True - def RunSparkStreamer(self): - threading.Thread(target=SparkStreamer).start() + def RequestListener(self): + """ + listener for requests on Kafka topic. + """ + consumer = self.kafka_consumer + consumer.subscribe([KafkaTopic.ANALYTICS_REQUEST.value]) + while True: + receive_msg = consumer.poll(2.0) + if receive_msg is None: + continue + elif receive_msg.error(): + if receive_msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + print("Consumer error: {}".format(receive_msg.error())) + break + analyzer = json.loads(receive_msg.value().decode('utf-8')) + analyzer_id = receive_msg.key().decode('utf-8') + LOGGER.debug('Recevied Collector: {:} - {:}'.format(analyzer_id, analyzer)) + print('Recevied Collector: {:} - {:} - {:}'.format(analyzer_id, analyzer, analyzer['input_kpis'])) + self.RunSparkStreamer(analyzer['input_kpis']) # TODO: Add active analyzer to list diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 92cc9a842..f42618a1c 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -15,7 +15,7 @@ import logging from pyspark.sql import SparkSession -from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType +from pyspark.sql.types import StructType, StructField, StringType, DoubleType from pyspark.sql.functions import from_json, col from common.tools.kafka.Variables import KafkaConfig, KafkaTopic @@ -32,20 +32,19 @@ def SettingKafkaParameters(): # TODO: create get_kafka_consumer() in common w return { # "kafka.bootstrap.servers": '127.0.0.1:9092', "kafka.bootstrap.servers": KafkaConfig.get_kafka_address(), - "subscribe" : KafkaTopic.ANALYTICS_REQUEST.value, - "startingOffsets" : 'latest', + "subscribe" : KafkaTopic.VALUE.value, + "startingOffsets" : 'latest', "failOnDataLoss" : 'false' # Optional: Set to "true" to fail the query on data loss } def DefiningRequestSchema(): return StructType([ - StructField("algo_name", StringType() , True), - StructField("input_kpis", ArrayType(StringType()), True), - StructField("output_kpis", ArrayType(StringType()), True), - StructField("oper_mode", IntegerType() , True) + StructField("time_stamp" , StringType() , True), + StructField("kpi_id" , StringType() , True), + StructField("kpi_value" , DoubleType() , True) ]) -def SparkStreamer(): +def SparkStreamer(kpi_list): """ Method to perform Spark operation Kafka stream. NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session. @@ -68,9 +67,11 @@ def SparkStreamer(): parsed_stream_data = stream_data.withColumn("parsed_value", from_json(col("value"), schema)) # Select the parsed fields final_stream_data = parsed_stream_data.select("parsed_value.*") + # Filter the stream to only include rows where the kpi_id is in the kpi_list + filtered_stream_data = final_stream_data.filter(col("kpi_id").isin(kpi_list)) # Start the Spark streaming query - query = final_stream_data \ + query = filtered_stream_data \ .writeStream \ .outputMode("append") \ .format("console") # You can change this to other output modes or sinks diff --git a/src/analytics/backend/tests/test_backend.py b/src/analytics/backend/tests/test_backend.py index 7a6175ecf..426c89e54 100644 --- a/src/analytics/backend/tests/test_backend.py +++ b/src/analytics/backend/tests/test_backend.py @@ -30,8 +30,15 @@ LOGGER = logging.getLogger(__name__) # response = KafkaTopic.create_all_topics() # assert isinstance(response, bool) -def test_SparkListener(): +def test_RunRequestListener(): LOGGER.info('test_RunRequestListener') AnalyticsBackendServiceObj = AnalyticsBackendService() - response = AnalyticsBackendServiceObj.RunSparkStreamer() + response = AnalyticsBackendServiceObj.RunRequestListener() LOGGER.debug(str(response)) + + +# def test_SparkListener(): +# LOGGER.info('test_RunRequestListener') +# AnalyticsBackendServiceObj = AnalyticsBackendService() +# response = AnalyticsBackendServiceObj.RunSparkStreamer() +# LOGGER.debug(str(response)) diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index ccbef3599..0f9f4e146 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -70,7 +70,7 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): "oper_mode" : analyzer_obj.operation_mode } self.kafka_producer.produce( - KafkaTopic.VALUE.value, + KafkaTopic.ANALYTICS_REQUEST.value, key = analyzer_uuid, value = json.dumps(analyzer_to_generate), callback = self.delivery_callback @@ -107,7 +107,7 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): "oper_mode" : -1 } self.kafka_producer.produce( - KafkaTopic.VALUE.value, + KafkaTopic.ANALYTICS_REQUEST.value, key = analyzer_uuid, value = json.dumps(analyzer_to_stop), callback = self.delivery_callback diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py index 4c826e5c3..0a8300436 100644 --- a/src/analytics/frontend/tests/messages.py +++ b/src/analytics/frontend/tests/messages.py @@ -32,6 +32,10 @@ def create_analyzer(): _kpi_id = KpiId() # input IDs to analyze _kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _kpi_id.kpi_id.uuid = "1e22f180-ba28-4641-b190-2287bf446666" + _create_analyzer.input_kpi_ids.append(_kpi_id) + _kpi_id.kpi_id.uuid = str(uuid.uuid4()) + _kpi_id.kpi_id.uuid = "6e22f180-ba28-4641-b190-2287bf448888" _create_analyzer.input_kpi_ids.append(_kpi_id) _kpi_id.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer.input_kpi_ids.append(_kpi_id) diff --git a/src/telemetry/backend/tests/test_TelemetryBackend.py b/src/telemetry/backend/tests/test_TelemetryBackend.py index a2bbee540..24a1b35cc 100644 --- a/src/telemetry/backend/tests/test_TelemetryBackend.py +++ b/src/telemetry/backend/tests/test_TelemetryBackend.py @@ -25,10 +25,10 @@ LOGGER = logging.getLogger(__name__) ########################### # --- "test_validate_kafka_topics" should be run before the functionality tests --- -def test_validate_kafka_topics(): - LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") - response = KafkaTopic.create_all_topics() - assert isinstance(response, bool) +# def test_validate_kafka_topics(): +# LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") +# response = KafkaTopic.create_all_topics() +# assert isinstance(response, bool) def test_RunRequestListener(): LOGGER.info('test_RunRequestListener') -- GitLab From e7e36f7d71e974056b8f79c74cdd33ae276245b7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Tue, 3 Sep 2024 15:48:37 +0000 Subject: [PATCH 51/70] Changes in Analytics Backend - SparkStreamer write Stream to Kafka topic ANALYTICS_RESPONSE. --- .../backend/service/SparkStreaming.py | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index f42618a1c..245a77d80 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -14,9 +14,9 @@ import logging -from pyspark.sql import SparkSession -from pyspark.sql.types import StructType, StructField, StringType, DoubleType -from pyspark.sql.functions import from_json, col +from pyspark.sql import SparkSession +from pyspark.sql.types import StructType, StructField, StringType, DoubleType +from pyspark.sql.functions import from_json, col from common.tools.kafka.Variables import KafkaConfig, KafkaTopic LOGGER = logging.getLogger(__name__) @@ -28,7 +28,7 @@ def DefiningSparkSession(): .config("spark.jars.packages", "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0") \ .getOrCreate() -def SettingKafkaParameters(): # TODO: create get_kafka_consumer() in common with inputs (bootstrap server, subscribe, startingOffset and failOnDataLoss with default values) +def SettingKafkaConsumerParams(): # TODO: create get_kafka_consumer() in common with inputs (bootstrap server, subscribe, startingOffset and failOnDataLoss with default values) return { # "kafka.bootstrap.servers": '127.0.0.1:9092', "kafka.bootstrap.servers": KafkaConfig.get_kafka_address(), @@ -44,37 +44,63 @@ def DefiningRequestSchema(): StructField("kpi_value" , DoubleType() , True) ]) +def SettingKafkaProducerParams(): + return { + "kafka.bootstrap.servers" : KafkaConfig.get_kafka_address(), + "topic" : KafkaTopic.ANALYTICS_RESPONSE.value + } + def SparkStreamer(kpi_list): """ Method to perform Spark operation Kafka stream. NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session. """ - kafka_params = SettingKafkaParameters() # Define the Kafka parameters - schema = DefiningRequestSchema() # Define the schema for the incoming JSON data - spark = DefiningSparkSession() # Define the spark session with app name and spark version + kafka_producer_params = SettingKafkaConsumerParams() # Define the Kafka producer parameters + kafka_consumer_params = SettingKafkaConsumerParams() # Define the Kafka consumer parameters + schema = DefiningRequestSchema() # Define the schema for the incoming JSON data + spark = DefiningSparkSession() # Define the spark session with app name and spark version try: # Read data from Kafka raw_stream_data = spark \ .readStream \ .format("kafka") \ - .options(**kafka_params) \ + .options(**kafka_consumer_params) \ .load() # Convert the value column from Kafka to a string - stream_data = raw_stream_data.selectExpr("CAST(value AS STRING)") + stream_data = raw_stream_data.selectExpr("CAST(value AS STRING)") # Parse the JSON string into a DataFrame with the defined schema - parsed_stream_data = stream_data.withColumn("parsed_value", from_json(col("value"), schema)) + parsed_stream_data = stream_data.withColumn("parsed_value", from_json(col("value"), schema)) # Select the parsed fields - final_stream_data = parsed_stream_data.select("parsed_value.*") + final_stream_data = parsed_stream_data.select("parsed_value.*") # Filter the stream to only include rows where the kpi_id is in the kpi_list filtered_stream_data = final_stream_data.filter(col("kpi_id").isin(kpi_list)) - # Start the Spark streaming query query = filtered_stream_data \ + .selectExpr("CAST(kpi_id AS STRING) AS key", "to_json(struct(*)) AS value") \ .writeStream \ - .outputMode("append") \ - .format("console") # You can change this to other output modes or sinks + .format("kafka") \ + .option("kafka.bootstrap.servers", KafkaConfig.get_kafka_address()) \ + .option("topic", KafkaTopic.ANALYTICS_RESPONSE.value) \ + .option("checkpointLocation", "/home/tfs/sparkcheckpoint") \ + .outputMode("append") + + # Start the Spark streaming query and write the output to the Kafka topic + # query = filtered_stream_data \ + # .selectExpr("CAST(kpi_id AS STRING) AS key", "to_json(struct(*)) AS value") \ + # .writeStream \ + # .format("kafka") \ + # .option(**kafka_producer_params) \ + # .option("checkpointLocation", "sparkcheckpoint") \ + # .outputMode("append") \ + # .start() + + # Start the Spark streaming query + # query = filtered_stream_data \ + # .writeStream \ + # .outputMode("append") \ + # .format("console") # You can change this to other output modes or sinks # Start the query execution query.start().awaitTermination() -- GitLab From 2183218975578c96f380cf69d6b5aea017dba41c Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 5 Sep 2024 10:29:44 +0000 Subject: [PATCH 52/70] Changes in Analytics Backend. - In the BackendService.py: + Updated SparkStreamer call with new parameters. - In SparkStreaming.py: + Added 'GetAggregerations' and 'ApplyThresholds' methods + Added 'window_size', 'win_slide_duration', 'time_stamp_col' and 'thresholds' parameters. - Added new messages. - Updated the 'RunSparkStreamer' call wth new parameters. --- .../service/AnalyticsBackendService.py | 9 +- .../backend/service/SparkStreaming.py | 90 +++++++++++++------ src/analytics/backend/tests/messages.py | 19 ++++ src/analytics/backend/tests/test_backend.py | 23 ++--- 4 files changed, 101 insertions(+), 40 deletions(-) diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 5331d027d..2842e2374 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -33,8 +33,13 @@ class AnalyticsBackendService(GenericGrpcService): 'group.id' : 'analytics-frontend', 'auto.offset.reset' : 'latest'}) - def RunSparkStreamer(self, kpi_list): - threading.Thread(target=SparkStreamer, args=(kpi_list,)).start() + def RunSparkStreamer(self, kpi_list, oper_list, thresholds_dict): + print ("Received parameters: {:} - {:} - {:}".format(kpi_list, oper_list, thresholds_dict)) + LOGGER.debug ("Received parameters: {:} - {:} - {:}".format(kpi_list, oper_list, thresholds_dict)) + threading.Thread(target=SparkStreamer, + args=(kpi_list, oper_list, None, None, thresholds_dict, None) + ).start() + return True def RunRequestListener(self)->bool: threading.Thread(target=self.RequestListener).start() diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 245a77d80..26d3c26d8 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -15,8 +15,8 @@ import logging from pyspark.sql import SparkSession -from pyspark.sql.types import StructType, StructField, StringType, DoubleType -from pyspark.sql.functions import from_json, col +from pyspark.sql.types import StructType, StructField, StringType, DoubleType, TimestampType +from pyspark.sql.functions import from_json, col, window, avg, min, max, first, last, stddev, when, round from common.tools.kafka.Variables import KafkaConfig, KafkaTopic LOGGER = logging.getLogger(__name__) @@ -44,22 +44,50 @@ def DefiningRequestSchema(): StructField("kpi_value" , DoubleType() , True) ]) -def SettingKafkaProducerParams(): - return { - "kafka.bootstrap.servers" : KafkaConfig.get_kafka_address(), - "topic" : KafkaTopic.ANALYTICS_RESPONSE.value +def get_aggregations(oper_list): + # Define the possible aggregation functions + agg_functions = { + 'avg' : round(avg ("kpi_value"), 3) .alias("avg_value"), + 'min' : round(min ("kpi_value"), 3) .alias("min_value"), + 'max' : round(max ("kpi_value"), 3) .alias("max_value"), + 'first': round(first ("kpi_value"), 3) .alias("first_value"), + 'last' : round(last ("kpi_value"), 3) .alias("last_value"), + 'stdev': round(stddev ("kpi_value"), 3) .alias("stdev_value") } + return [agg_functions[op] for op in oper_list if op in agg_functions] # Filter and return only the selected aggregations + +def apply_thresholds(aggregated_df, thresholds): + # Apply thresholds (TH-Fail and TH-RAISE) based on the thresholds dictionary on the aggregated DataFrame. + + # Loop through each column name and its associated thresholds + for col_name, (fail_th, raise_th) in thresholds.items(): + # Apply TH-Fail condition (if column value is less than the fail threshold) + aggregated_df = aggregated_df.withColumn( + f"{col_name}_THRESHOLD_FAIL", + when(col(col_name) < fail_th, True).otherwise(False) + ) + # Apply TH-RAISE condition (if column value is greater than the raise threshold) + aggregated_df = aggregated_df.withColumn( + f"{col_name}_THRESHOLD_RAISE", + when(col(col_name) > raise_th, True).otherwise(False) + ) + return aggregated_df -def SparkStreamer(kpi_list): +def SparkStreamer(kpi_list, oper_list, window_size=None, win_slide_duration=None, thresholds=None, time_stamp_col=None): """ Method to perform Spark operation Kafka stream. NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session. """ - kafka_producer_params = SettingKafkaConsumerParams() # Define the Kafka producer parameters kafka_consumer_params = SettingKafkaConsumerParams() # Define the Kafka consumer parameters schema = DefiningRequestSchema() # Define the schema for the incoming JSON data spark = DefiningSparkSession() # Define the spark session with app name and spark version - + + # extra options default assignment + if window_size is None: window_size = "60 seconds" # default + if win_slide_duration is None: win_slide_duration = "30 seconds" # default + if time_stamp_col is None: time_stamp_col = "time_stamp" # default + if thresholds is None: thresholds = {} # No threshold will be applied + try: # Read data from Kafka raw_stream_data = spark \ @@ -74,36 +102,42 @@ def SparkStreamer(kpi_list): parsed_stream_data = stream_data.withColumn("parsed_value", from_json(col("value"), schema)) # Select the parsed fields final_stream_data = parsed_stream_data.select("parsed_value.*") + # Convert the time_stamp to proper timestamp (assuming it's in ISO format) + final_stream_data = final_stream_data.withColumn(time_stamp_col, col(time_stamp_col).cast(TimestampType())) # Filter the stream to only include rows where the kpi_id is in the kpi_list filtered_stream_data = final_stream_data.filter(col("kpi_id").isin(kpi_list)) + # Define a window for aggregation + windowed_stream_data = filtered_stream_data \ + .groupBy( + window( col(time_stamp_col), + window_size, slideDuration=win_slide_duration + ), + col("kpi_id") + ) \ + .agg(*get_aggregations(oper_list)) + # Apply thresholds to the aggregated data + thresholded_stream_data = apply_thresholds(windowed_stream_data, thresholds) + + # --- This will write output on console: FOR TESTING PURPOSES + # Start the Spark streaming query + # query = thresholded_stream_data \ + # .writeStream \ + # .outputMode("update") \ + # .format("console") - query = filtered_stream_data \ + # --- This will write output to Kafka: ACTUAL IMPLEMENTATION + query = thresholded_stream_data \ .selectExpr("CAST(kpi_id AS STRING) AS key", "to_json(struct(*)) AS value") \ .writeStream \ .format("kafka") \ .option("kafka.bootstrap.servers", KafkaConfig.get_kafka_address()) \ .option("topic", KafkaTopic.ANALYTICS_RESPONSE.value) \ - .option("checkpointLocation", "/home/tfs/sparkcheckpoint") \ - .outputMode("append") - - # Start the Spark streaming query and write the output to the Kafka topic - # query = filtered_stream_data \ - # .selectExpr("CAST(kpi_id AS STRING) AS key", "to_json(struct(*)) AS value") \ - # .writeStream \ - # .format("kafka") \ - # .option(**kafka_producer_params) \ - # .option("checkpointLocation", "sparkcheckpoint") \ - # .outputMode("append") \ - # .start() - - # Start the Spark streaming query - # query = filtered_stream_data \ - # .writeStream \ - # .outputMode("append") \ - # .format("console") # You can change this to other output modes or sinks + .option("checkpointLocation", "analytics/.spark/checkpoint") \ + .outputMode("update") # Start the query execution query.start().awaitTermination() + except Exception as e: print("Error in Spark streaming process: {:}".format(e)) LOGGER.debug("Error in Spark streaming process: {:}".format(e)) diff --git a/src/analytics/backend/tests/messages.py b/src/analytics/backend/tests/messages.py index 5cf553eaa..c4a26a1ac 100644 --- a/src/analytics/backend/tests/messages.py +++ b/src/analytics/backend/tests/messages.py @@ -13,3 +13,22 @@ # limitations under the License. +def get_kpi_id_list(): + return ["6e22f180-ba28-4641-b190-2287bf448888", "1e22f180-ba28-4641-b190-2287bf446666"] + +def get_operation_list(): + return [ 'avg', 'max' ] # possibilities ['avg', 'min', 'max', 'first', 'last', 'stdev'] + +def get_threshold_dict(): + threshold_dict = { + 'avg_value' : (20, 30), + 'min_value' : (00, 10), + 'max_value' : (45, 50), + 'first_value' : (00, 10), + 'last_value' : (40, 50), + 'stddev_value' : (00, 10), + } + # Filter threshold_dict based on the operation_list + return { + op + '_value': threshold_dict[op+'_value'] for op in get_operation_list() if op + '_value' in threshold_dict + } diff --git a/src/analytics/backend/tests/test_backend.py b/src/analytics/backend/tests/test_backend.py index 426c89e54..9e8a0832d 100644 --- a/src/analytics/backend/tests/test_backend.py +++ b/src/analytics/backend/tests/test_backend.py @@ -15,7 +15,7 @@ import logging from common.tools.kafka.Variables import KafkaTopic from analytics.backend.service.AnalyticsBackendService import AnalyticsBackendService - +from analytics.backend.tests.messages import get_kpi_id_list, get_operation_list, get_threshold_dict LOGGER = logging.getLogger(__name__) @@ -25,10 +25,10 @@ LOGGER = logging.getLogger(__name__) ########################### # --- "test_validate_kafka_topics" should be run before the functionality tests --- -# def test_validate_kafka_topics(): -# LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") -# response = KafkaTopic.create_all_topics() -# assert isinstance(response, bool) +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) def test_RunRequestListener(): LOGGER.info('test_RunRequestListener') @@ -37,8 +37,11 @@ def test_RunRequestListener(): LOGGER.debug(str(response)) -# def test_SparkListener(): -# LOGGER.info('test_RunRequestListener') -# AnalyticsBackendServiceObj = AnalyticsBackendService() -# response = AnalyticsBackendServiceObj.RunSparkStreamer() -# LOGGER.debug(str(response)) +def test_SparkListener(): + LOGGER.info('test_RunRequestListener') + AnalyticsBackendServiceObj = AnalyticsBackendService() + response = AnalyticsBackendServiceObj.RunSparkStreamer( + get_kpi_id_list(), get_operation_list(), get_threshold_dict() + ) + LOGGER.debug(str(response)) + assert isinstance(response, bool) -- GitLab From faa6c684f08a62da7ab5bc871b7680179961375b Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 5 Sep 2024 10:32:09 +0000 Subject: [PATCH 53/70] Changes in Analytics Backend. - Updated function names. --- src/analytics/backend/service/SparkStreaming.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 26d3c26d8..73aa75025 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -44,7 +44,7 @@ def DefiningRequestSchema(): StructField("kpi_value" , DoubleType() , True) ]) -def get_aggregations(oper_list): +def GetAggregations(oper_list): # Define the possible aggregation functions agg_functions = { 'avg' : round(avg ("kpi_value"), 3) .alias("avg_value"), @@ -56,7 +56,7 @@ def get_aggregations(oper_list): } return [agg_functions[op] for op in oper_list if op in agg_functions] # Filter and return only the selected aggregations -def apply_thresholds(aggregated_df, thresholds): +def ApplyThresholds(aggregated_df, thresholds): # Apply thresholds (TH-Fail and TH-RAISE) based on the thresholds dictionary on the aggregated DataFrame. # Loop through each column name and its associated thresholds @@ -114,9 +114,9 @@ def SparkStreamer(kpi_list, oper_list, window_size=None, win_slide_duration=None ), col("kpi_id") ) \ - .agg(*get_aggregations(oper_list)) + .agg(*GetAggregations(oper_list)) # Apply thresholds to the aggregated data - thresholded_stream_data = apply_thresholds(windowed_stream_data, thresholds) + thresholded_stream_data = ApplyThresholds(windowed_stream_data, thresholds) # --- This will write output on console: FOR TESTING PURPOSES # Start the Spark streaming query -- GitLab From df864e661ecc8d548dd015bffa63aa5a96088e9f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 5 Sep 2024 10:32:59 +0000 Subject: [PATCH 54/70] Added pySpark checkpoint path in '.gitignore' file. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 20b98c30c..6a53f106e 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,6 @@ libyang/ # Other logs **/logs/*.log.* + +# PySpark checkpoints +src/analytics/.spark/* \ No newline at end of file -- GitLab From 990395f47c5c6ca47027f036b71edcdb090edb08 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 6 Sep 2024 07:55:54 +0000 Subject: [PATCH 55/70] Changes in Analytics DB. - parameters col added in DB - parameters field added in Analytics.proto - AnalyzerModel class methods changes - changes in messages file --- src/analytics/database/AnalyzerModel.py | 45 ++++++++++--------- src/analytics/frontend/tests/messages.py | 11 ++++- src/analytics/frontend/tests/test_frontend.py | 2 +- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/analytics/database/AnalyzerModel.py b/src/analytics/database/AnalyzerModel.py index 0b66e04d0..783205269 100644 --- a/src/analytics/database/AnalyzerModel.py +++ b/src/analytics/database/AnalyzerModel.py @@ -15,7 +15,7 @@ import logging import enum -from sqlalchemy import Column, String, Float, Enum +from sqlalchemy import Column, String, Float, Enum, BigInteger, JSON from sqlalchemy.orm import registry from common.proto import analytics_frontend_pb2 from common.proto import kpi_manager_pb2 @@ -36,23 +36,25 @@ class AnalyzerOperationMode (enum.Enum): class Analyzer(Base): __tablename__ = 'analyzer' - analyzer_id = Column(UUID(as_uuid=False) , primary_key=True) - algorithm_name = Column(String , nullable=False) - input_kpi_ids = Column(ARRAY(UUID(as_uuid=False)) , nullable=False) - output_kpi_ids = Column(ARRAY(UUID(as_uuid=False)) , nullable=False) - operation_mode = Column(Enum(AnalyzerOperationMode), nullable=False) - batch_min_duration_s = Column(Float , nullable=False) - batch_max_duration_s = Column(Float , nullable=False) - bacth_min_size = Column(Float , nullable=False) - bacth_max_size = Column(Float , nullable=False) + analyzer_id = Column( UUID(as_uuid=False) , primary_key=True) + algorithm_name = Column( String , nullable=False ) + input_kpi_ids = Column( ARRAY(UUID(as_uuid=False)) , nullable=False ) + output_kpi_ids = Column( ARRAY(UUID(as_uuid=False)) , nullable=False ) + operation_mode = Column( Enum(AnalyzerOperationMode), nullable=False ) + parameters = Column( JSON , nullable=True ) + batch_min_duration_s = Column( Float , nullable=False ) + batch_max_duration_s = Column( Float , nullable=False ) + batch_min_size = Column( BigInteger , nullable=False ) + batch_max_size = Column( BigInteger , nullable=False ) # helps in logging the information def __repr__(self): - return (f"") + return (f"") + @classmethod def ConvertAnalyzerToRow(cls, request): @@ -67,10 +69,11 @@ class Analyzer(Base): input_kpi_ids = [k.kpi_id.uuid for k in request.input_kpi_ids], output_kpi_ids = [k.kpi_id.uuid for k in request.output_kpi_ids], operation_mode = AnalyzerOperationMode(request.operation_mode), # converts integer to coresponding Enum class member + parameters = dict(request.parameters), batch_min_duration_s = request.batch_min_duration_s, batch_max_duration_s = request.batch_max_duration_s, - bacth_min_size = request.batch_min_size, - bacth_max_size = request.batch_max_size + batch_min_size = request.batch_min_size, + batch_max_size = request.batch_max_size ) @classmethod @@ -85,17 +88,19 @@ class Analyzer(Base): response.analyzer_id.analyzer_id.uuid = row.analyzer_id response.algorithm_name = row.algorithm_name response.operation_mode = row.operation_mode + response.parameters.update(row.parameters) - _kpi_id = kpi_manager_pb2.KpiId() for input_kpi_id in row.input_kpi_ids: + _kpi_id = kpi_manager_pb2.KpiId() _kpi_id.kpi_id.uuid = input_kpi_id response.input_kpi_ids.append(_kpi_id) for output_kpi_id in row.output_kpi_ids: + _kpi_id = kpi_manager_pb2.KpiId() _kpi_id.kpi_id.uuid = output_kpi_id response.output_kpi_ids.append(_kpi_id) response.batch_min_duration_s = row.batch_min_duration_s response.batch_max_duration_s = row.batch_max_duration_s - response.batch_min_size = row.bacth_min_size - response.batch_max_size = row.bacth_max_size + response.batch_min_size = row.batch_min_size + response.batch_max_size = row.batch_max_size return response diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py index 0a8300436..4ffbb0b8e 100644 --- a/src/analytics/frontend/tests/messages.py +++ b/src/analytics/frontend/tests/messages.py @@ -13,6 +13,7 @@ # limitations under the License. import uuid +import json from common.proto.kpi_manager_pb2 import KpiId from common.proto.analytics_frontend_pb2 import ( AnalyzerOperationMode, AnalyzerId, Analyzer, AnalyzerFilter ) @@ -26,7 +27,7 @@ def create_analyzer_id(): def create_analyzer(): _create_analyzer = Analyzer() _create_analyzer.analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) - _create_analyzer.algorithm_name = "some_algo_name" + _create_analyzer.algorithm_name = "Test_Aggergate_and_Threshold" _create_analyzer.operation_mode = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING _kpi_id = KpiId() @@ -44,6 +45,14 @@ def create_analyzer(): _create_analyzer.output_kpi_ids.append(_kpi_id) _kpi_id.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer.output_kpi_ids.append(_kpi_id) + # parameter + _threshold_dict = { + 'avg_value' :(20, 30), 'min_value' :(00, 10), 'max_value' :(45, 50), + 'first_value' :(00, 10), 'last_value' :(40, 50), 'stddev_value':(00, 10)} + _create_analyzer.parameters['thresholds'] = json.dumps(_threshold_dict) + _create_analyzer.parameters['window_size'] = "60 seconds" # Such as "10 seconds", "2 minutes", "3 hours", "4 days" or "5 weeks" + _create_analyzer.parameters['window_slider'] = "30 seconds" # should be less than window size + _create_analyzer.parameters['store_aggregate'] = str(False) # TRUE to store. No implemented yet return _create_analyzer diff --git a/src/analytics/frontend/tests/test_frontend.py b/src/analytics/frontend/tests/test_frontend.py index ae7875b86..df6ce165e 100644 --- a/src/analytics/frontend/tests/test_frontend.py +++ b/src/analytics/frontend/tests/test_frontend.py @@ -76,7 +76,7 @@ def analyticsFrontend_client(analyticsFrontend_service : AnalyticsFrontendServic ########################### # ----- core funtionality test ----- -def test_StartAnalytic(analyticsFrontend_client): +def test_StartAnalytics(analyticsFrontend_client): LOGGER.info(' >>> test_StartAnalytic START: <<< ') response = analyticsFrontend_client.StartAnalyzer(create_analyzer()) LOGGER.debug(str(response)) -- GitLab From 5eed4743c3195f42e677f9818604fed03c3a0a69 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 6 Sep 2024 09:48:02 +0000 Subject: [PATCH 56/70] Changes in Analytics. - UNSPECIFIED option added in the "AnalyzerOperationMode" enum as a best practice. - In SparkStreamer, changed the thresholds parameter from optional to compulsory. --- proto/analytics_frontend.proto | 5 +++-- src/analytics/backend/service/SparkStreaming.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto index bd28a21bf..bc7420d54 100644 --- a/proto/analytics_frontend.proto +++ b/proto/analytics_frontend.proto @@ -30,8 +30,9 @@ message AnalyzerId { } enum AnalyzerOperationMode { - ANALYZEROPERATIONMODE_BATCH = 0; - ANALYZEROPERATIONMODE_STREAMING = 1; + ANALYZEROPERATIONMODE_UNSPECIFIED = 0; + ANALYZEROPERATIONMODE_BATCH = 1; + ANALYZEROPERATIONMODE_STREAMING = 2; } message Analyzer { diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 73aa75025..175222b59 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -73,7 +73,7 @@ def ApplyThresholds(aggregated_df, thresholds): ) return aggregated_df -def SparkStreamer(kpi_list, oper_list, window_size=None, win_slide_duration=None, thresholds=None, time_stamp_col=None): +def SparkStreamer(kpi_list, oper_list, thresholds, window_size=None, win_slide_duration=None, time_stamp_col=None): """ Method to perform Spark operation Kafka stream. NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session. @@ -86,7 +86,6 @@ def SparkStreamer(kpi_list, oper_list, window_size=None, win_slide_duration=None if window_size is None: window_size = "60 seconds" # default if win_slide_duration is None: win_slide_duration = "30 seconds" # default if time_stamp_col is None: time_stamp_col = "time_stamp" # default - if thresholds is None: thresholds = {} # No threshold will be applied try: # Read data from Kafka -- GitLab From 88d7a20007565a66f7561ec9e9a7811db1c9b0fe Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 7 Sep 2024 05:56:22 +0000 Subject: [PATCH 57/70] Changes in Analytics Service. - Thresholds, window_size, window_slider is added in Frontend and Backend. --- .../service/AnalyticsBackendService.py | 24 ++++++++++++------- .../backend/service/SparkStreaming.py | 3 ++- src/analytics/backend/tests/messages.py | 2 +- .../AnalyticsFrontendServiceServicerImpl.py | 12 ++++++---- src/analytics/frontend/tests/messages.py | 8 +++---- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 2842e2374..4784ef051 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -22,7 +22,7 @@ from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from confluent_kafka import Consumer as KafkaConsumer from confluent_kafka import KafkaError -LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class AnalyticsBackendService(GenericGrpcService): """ @@ -33,11 +33,18 @@ class AnalyticsBackendService(GenericGrpcService): 'group.id' : 'analytics-frontend', 'auto.offset.reset' : 'latest'}) - def RunSparkStreamer(self, kpi_list, oper_list, thresholds_dict): - print ("Received parameters: {:} - {:} - {:}".format(kpi_list, oper_list, thresholds_dict)) - LOGGER.debug ("Received parameters: {:} - {:} - {:}".format(kpi_list, oper_list, thresholds_dict)) + def RunSparkStreamer(self, analyzer): + kpi_list = analyzer['input_kpis'] + oper_list = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())] + thresholds = analyzer['thresholds'] + window_size = analyzer['window_size'] + window_slider = analyzer['window_slider'] + print ("Received parameters: {:} - {:} - {:} - {:} - {:}".format( + kpi_list, oper_list, thresholds, window_size, window_slider)) + LOGGER.debug ("Received parameters: {:} - {:} - {:} - {:} - {:}".format( + kpi_list, oper_list, thresholds, window_size, window_slider)) threading.Thread(target=SparkStreamer, - args=(kpi_list, oper_list, None, None, thresholds_dict, None) + args=(kpi_list, oper_list, thresholds, window_size, window_slider, None) ).start() return True @@ -63,6 +70,7 @@ class AnalyticsBackendService(GenericGrpcService): break analyzer = json.loads(receive_msg.value().decode('utf-8')) analyzer_id = receive_msg.key().decode('utf-8') - LOGGER.debug('Recevied Collector: {:} - {:}'.format(analyzer_id, analyzer)) - print('Recevied Collector: {:} - {:} - {:}'.format(analyzer_id, analyzer, analyzer['input_kpis'])) - self.RunSparkStreamer(analyzer['input_kpis']) # TODO: Add active analyzer to list + LOGGER.debug('Recevied Analyzer: {:} - {:}'.format(analyzer_id, analyzer)) + print('Recevied Analyzer: {:} - {:}'.format(analyzer_id, analyzer)) + # TODO: Add active analyzer to list + self.RunSparkStreamer(analyzer) diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 175222b59..11ec9fe5f 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -73,7 +73,8 @@ def ApplyThresholds(aggregated_df, thresholds): ) return aggregated_df -def SparkStreamer(kpi_list, oper_list, thresholds, window_size=None, win_slide_duration=None, time_stamp_col=None): +def SparkStreamer(kpi_list, oper_list, thresholds, + window_size=None, win_slide_duration=None, time_stamp_col=None): """ Method to perform Spark operation Kafka stream. NOTE: Kafka topic to be processesd should have atleast one row before initiating the spark session. diff --git a/src/analytics/backend/tests/messages.py b/src/analytics/backend/tests/messages.py index c4a26a1ac..9acd6ad9d 100644 --- a/src/analytics/backend/tests/messages.py +++ b/src/analytics/backend/tests/messages.py @@ -26,7 +26,7 @@ def get_threshold_dict(): 'max_value' : (45, 50), 'first_value' : (00, 10), 'last_value' : (40, 50), - 'stddev_value' : (00, 10), + 'stdev_value' : (00, 10), } # Filter threshold_dict based on the operation_list return { diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index 0f9f4e146..2671bfb13 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -64,10 +64,14 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): """ analyzer_uuid = analyzer_obj.analyzer_id.analyzer_id.uuid analyzer_to_generate : Dict = { - "algo_name" : analyzer_obj.algorithm_name, - "input_kpis" : [k.kpi_id.uuid for k in analyzer_obj.input_kpi_ids], - "output_kpis" : [k.kpi_id.uuid for k in analyzer_obj.output_kpi_ids], - "oper_mode" : analyzer_obj.operation_mode + "algo_name" : analyzer_obj.algorithm_name, + "input_kpis" : [k.kpi_id.uuid for k in analyzer_obj.input_kpi_ids], + "output_kpis" : [k.kpi_id.uuid for k in analyzer_obj.output_kpi_ids], + "oper_mode" : analyzer_obj.operation_mode, + "thresholds" : json.loads(analyzer_obj.parameters["thresholds"]), + "window_size" : analyzer_obj.parameters["window_size"], + "window_slider" : analyzer_obj.parameters["window_slider"], + # "store_aggregate" : analyzer_obj.parameters["store_aggregate"] } self.kafka_producer.produce( KafkaTopic.ANALYTICS_REQUEST.value, diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py index 4ffbb0b8e..180fac1f8 100644 --- a/src/analytics/frontend/tests/messages.py +++ b/src/analytics/frontend/tests/messages.py @@ -33,10 +33,10 @@ def create_analyzer(): _kpi_id = KpiId() # input IDs to analyze _kpi_id.kpi_id.uuid = str(uuid.uuid4()) - _kpi_id.kpi_id.uuid = "1e22f180-ba28-4641-b190-2287bf446666" + _kpi_id.kpi_id.uuid = "6e22f180-ba28-4641-b190-2287bf448888" _create_analyzer.input_kpi_ids.append(_kpi_id) _kpi_id.kpi_id.uuid = str(uuid.uuid4()) - _kpi_id.kpi_id.uuid = "6e22f180-ba28-4641-b190-2287bf448888" + _kpi_id.kpi_id.uuid = "1e22f180-ba28-4641-b190-2287bf446666" _create_analyzer.input_kpi_ids.append(_kpi_id) _kpi_id.kpi_id.uuid = str(uuid.uuid4()) _create_analyzer.input_kpi_ids.append(_kpi_id) @@ -47,8 +47,8 @@ def create_analyzer(): _create_analyzer.output_kpi_ids.append(_kpi_id) # parameter _threshold_dict = { - 'avg_value' :(20, 30), 'min_value' :(00, 10), 'max_value' :(45, 50), - 'first_value' :(00, 10), 'last_value' :(40, 50), 'stddev_value':(00, 10)} + # 'avg_value' :(20, 30), 'min_value' :(00, 10), 'max_value' :(45, 50), + 'first_value' :(00, 10), 'last_value' :(40, 50), 'stdev_value':(00, 10)} _create_analyzer.parameters['thresholds'] = json.dumps(_threshold_dict) _create_analyzer.parameters['window_size'] = "60 seconds" # Such as "10 seconds", "2 minutes", "3 hours", "4 days" or "5 weeks" _create_analyzer.parameters['window_slider'] = "30 seconds" # should be less than window size -- GitLab From 88caa1cf6f141b8409c258fe685f192c646a9dea Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 7 Sep 2024 14:20:49 +0000 Subject: [PATCH 58/70] Changes in Analytics **Backend:** - Added a dictionary to manage running analyzers. - Implemented logic to manage running analyzers. - Added the `TerminateAnalyzerBackend` method to handle analyzer termination. **Frontend:** - Modified and invoked the `PublishStopRequestOnKafka` method. --- .../service/AnalyticsBackendService.py | 54 +++++++++++++++---- .../AnalyticsFrontendServiceServicerImpl.py | 12 ++--- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 4784ef051..f9fcf47ec 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -29,13 +29,14 @@ class AnalyticsBackendService(GenericGrpcService): Class listens for ... """ def __init__(self, cls_name : str = __name__) -> None: + self.running_threads = {} # To keep track of all running analyzers self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'analytics-frontend', 'auto.offset.reset' : 'latest'}) - def RunSparkStreamer(self, analyzer): + def RunSparkStreamer(self, analyzer_id, analyzer): kpi_list = analyzer['input_kpis'] - oper_list = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())] + oper_list = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())] # TODO: update this line... thresholds = analyzer['thresholds'] window_size = analyzer['window_size'] window_slider = analyzer['window_slider'] @@ -43,10 +44,19 @@ class AnalyticsBackendService(GenericGrpcService): kpi_list, oper_list, thresholds, window_size, window_slider)) LOGGER.debug ("Received parameters: {:} - {:} - {:} - {:} - {:}".format( kpi_list, oper_list, thresholds, window_size, window_slider)) - threading.Thread(target=SparkStreamer, - args=(kpi_list, oper_list, thresholds, window_size, window_slider, None) - ).start() - return True + try: + stop_event = threading.Event() + thread = threading.Thread(target=SparkStreamer, + args=(kpi_list, oper_list, thresholds, window_size, window_slider, None, + stop_event)) + self.running_threads[analyzer_id] = (thread, stop_event) + # thread.start() + LOGGER.info("Initiated Analyzer backend: {:}".format(analyzer_id)) + return True + except Exception as e: + print ("Failed to initiate Analyzer backend: {:}".format(e)) + LOGGER.error("Failed to initiate Analyzer backend: {:}".format(e)) + return False def RunRequestListener(self)->bool: threading.Thread(target=self.RequestListener).start() @@ -69,8 +79,30 @@ class AnalyticsBackendService(GenericGrpcService): print("Consumer error: {}".format(receive_msg.error())) break analyzer = json.loads(receive_msg.value().decode('utf-8')) - analyzer_id = receive_msg.key().decode('utf-8') - LOGGER.debug('Recevied Analyzer: {:} - {:}'.format(analyzer_id, analyzer)) - print('Recevied Analyzer: {:} - {:}'.format(analyzer_id, analyzer)) - # TODO: Add active analyzer to list - self.RunSparkStreamer(analyzer) + analyzer_uuid = receive_msg.key().decode('utf-8') + LOGGER.debug('Recevied Analyzer: {:} - {:}'.format(analyzer_uuid, analyzer)) + print ('Recevied Analyzer: {:} - {:}'.format(analyzer_uuid, analyzer)) + + if analyzer["algo_name"] is None and analyzer["oper_mode"] is None: + self.TerminateAnalyzerBackend(analyzer_uuid) + else: + self.RunSparkStreamer(analyzer_uuid, analyzer) + + def TerminateAnalyzerBackend(self, analyzer_uuid): + if analyzer_uuid in self.running_threads: + try: + thread, stop_event = self.running_threads[analyzer_uuid] + stop_event.set() + thread.join() + del self.running_threads[analyzer_uuid] + print ("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid)) + LOGGER.info("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid)) + return True + except Exception as e: + LOGGER.error("Failed to terminate. Analyzer Id: {:} - ERROR: ".format(analyzer_uuid, e)) + return False + else: + print ("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid)) + # LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid)) + # generate confirmation towards frontend + diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index 2671bfb13..9c438761c 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -94,21 +94,21 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): self.db_obj.delete_db_row_by_id( AnalyzerModel, "analyzer_id", analyzer_id_to_delete ) + self.PublishStopRequestOnKafka(analyzer_id_to_delete) except Exception as e: - LOGGER.warning('Unable to delete analyzer. Error: {:}'.format(e)) - self.PublishStopRequestOnKafka(request) + LOGGER.error('Unable to delete analyzer. Error: {:}'.format(e)) return Empty() - def PublishStopRequestOnKafka(self, analyzer_id): + def PublishStopRequestOnKafka(self, analyzer_uuid): """ Method to generate stop analyzer request on Kafka. """ - analyzer_uuid = analyzer_id.analyzer_id.uuid + # analyzer_uuid = analyzer_id.analyzer_id.uuid analyzer_to_stop : Dict = { - "algo_name" : -1, + "algo_name" : None, "input_kpis" : [], "output_kpis" : [], - "oper_mode" : -1 + "oper_mode" : None } self.kafka_producer.produce( KafkaTopic.ANALYTICS_REQUEST.value, -- GitLab From 9e2266258226df312ef67b65b781dc3b1dc07459 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 7 Sep 2024 14:30:55 +0000 Subject: [PATCH 59/70] Changes in Telemetry Service **Frontend:** - Deleted irrelevant lines. **Backend:** - Added `delete_db_row_by_id` in the Stop Collector method. --- src/telemetry/backend/service/TelemetryBackendService.py | 2 -- .../service/TelemetryFrontendServiceServicerImpl.py | 9 ++++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index bb9f0a314..5276c2be3 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -32,8 +32,6 @@ from common.tools.service.GenericGrpcService import GenericGrpcService LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('TelemetryBackend', 'backendService') -# EXPORTER_ENDPOINT = "http://10.152.183.2:9100/metrics" - class TelemetryBackendService(GenericGrpcService): """ diff --git a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py index 2b872dba3..b73d9fa95 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py +++ b/src/telemetry/frontend/service/TelemetryFrontendServiceServicerImpl.py @@ -89,7 +89,14 @@ class TelemetryFrontendServiceServicerImpl(TelemetryFrontendServiceServicer): request : CollectorId, grpc_context: grpc.ServicerContext # type: ignore ) -> Empty: # type: ignore LOGGER.info ("gRPC message: {:}".format(request)) - self.PublishStopRequestOnKafka(request) + try: + collector_to_delete = request.collector_id.uuid + self.tele_db_obj.delete_db_row_by_id( + CollectorModel, "collector_id", collector_to_delete + ) + self.PublishStopRequestOnKafka(request) + except Exception as e: + LOGGER.error('Unable to delete collector. Error: {:}'.format(e)) return Empty() def PublishStopRequestOnKafka(self, collector_id): -- GitLab From d6dc91c284e370c3aa64fd854bb0120bd2ca565f Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 7 Sep 2024 16:48:20 +0000 Subject: [PATCH 60/70] Changes in Analytics Backend - Updated the position of the `stop_event` parameter. - Added confirmation for pySpark checkpoint deletion. - Added a PySpark termination script to handle the `StopCollector` event. --- .../backend/service/AnalyticsBackendService.py | 12 ++++++------ .../backend/service/SparkStreaming.py | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index f9fcf47ec..899525fc6 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -47,10 +47,11 @@ class AnalyticsBackendService(GenericGrpcService): try: stop_event = threading.Event() thread = threading.Thread(target=SparkStreamer, - args=(kpi_list, oper_list, thresholds, window_size, window_slider, None, - stop_event)) + args=(kpi_list, oper_list, thresholds, stop_event, + window_size, window_slider, None )) self.running_threads[analyzer_id] = (thread, stop_event) - # thread.start() + thread.start() + print ("Initiated Analyzer backend: {:}".format(analyzer_id)) LOGGER.info("Initiated Analyzer backend: {:}".format(analyzer_id)) return True except Exception as e: @@ -99,10 +100,9 @@ class AnalyticsBackendService(GenericGrpcService): LOGGER.info("Terminating backend (by TerminateBackend): Analyzer Id: {:}".format(analyzer_uuid)) return True except Exception as e: - LOGGER.error("Failed to terminate. Analyzer Id: {:} - ERROR: ".format(analyzer_uuid, e)) + LOGGER.error("Failed to terminate. Analyzer Id: {:} - ERROR: {:}".format(analyzer_uuid, e)) return False else: - print ("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid)) + print ("Analyzer not found in active collectors. Analyzer Id: {:}".format(analyzer_uuid)) # LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid)) # generate confirmation towards frontend - diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 11ec9fe5f..202076ed5 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -13,7 +13,7 @@ # limitations under the License. -import logging +import logging, time from pyspark.sql import SparkSession from pyspark.sql.types import StructType, StructField, StringType, DoubleType, TimestampType from pyspark.sql.functions import from_json, col, window, avg, min, max, first, last, stddev, when, round @@ -25,6 +25,7 @@ def DefiningSparkSession(): # Create a Spark session with specific spark verions (3.5.0) return SparkSession.builder \ .appName("Analytics") \ + .config("spark.sql.streaming.forceDeleteTempCheckpointLocation", "true") \ .config("spark.jars.packages", "org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.0") \ .getOrCreate() @@ -73,7 +74,7 @@ def ApplyThresholds(aggregated_df, thresholds): ) return aggregated_df -def SparkStreamer(kpi_list, oper_list, thresholds, +def SparkStreamer(kpi_list, oper_list, thresholds, stop_event, window_size=None, win_slide_duration=None, time_stamp_col=None): """ Method to perform Spark operation Kafka stream. @@ -136,10 +137,17 @@ def SparkStreamer(kpi_list, oper_list, thresholds, .outputMode("update") # Start the query execution - query.start().awaitTermination() + queryHandler = query.start() + + # Loop to check for stop event flag. To be set by stop collector method. + while True: + if stop_event.is_set(): + print ("Stop Event activated. Terminating in 5 seconds...") + time.sleep(5) + queryHandler.stop() + break + time.sleep(5) except Exception as e: print("Error in Spark streaming process: {:}".format(e)) LOGGER.debug("Error in Spark streaming process: {:}".format(e)) - finally: - spark.stop() -- GitLab From 54e0014b545f6a6d4db0384171ca30189c8a6c8a Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Sat, 7 Sep 2024 16:51:01 +0000 Subject: [PATCH 61/70] This Commit is for Error Correction - Reverted an unintentional change made in the `tfs.sh` file. --- deploy/tfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 1aa32684b..b756ad2d0 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -179,7 +179,7 @@ kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} -- --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \ --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \ --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \ - --from-literSLMODE=require + --from-literal=CRDB_SSLMODE=require printf "\n" echo "Create secret with CockroachDB data for Analytics microservices" -- GitLab From 8ac130c10dade4d58a1de43382d046a203a80af5 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 9 Sep 2024 11:38:19 +0000 Subject: [PATCH 62/70] Changes in Analytics Proto: - Added `Duration_s` field to the proto file. Frontend: - Added `SelectAnalyzer` logic. - Improved message formatting in the `create_analyzer_filter()` function. - Added a test case: `test_SelectAnalytics`. Backend: - Renamed the `RunSparkStreamer` method to `StartSparkStreamer`. - Updated the `StartRequestListener` method to return `(thread, stop_event)`. - Added a `StopRequestListener` method to stop the listener. Database: - Added the `select_with_filter` method with actual logic implementation. - Updated the `ConvertRowToAnalyzer` method to correctly read the `operation_mode` ENUM value. --- proto/analytics_frontend.proto | 18 +++++----- .../service/AnalyticsBackendService.py | 34 ++++++++++++++----- .../backend/service/SparkStreaming.py | 3 +- src/analytics/backend/tests/test_backend.py | 21 ++++++++++-- src/analytics/database/AnalyzerModel.py | 2 +- src/analytics/database/Analyzer_DB.py | 19 +++++++++-- .../AnalyticsFrontendServiceServicerImpl.py | 19 ++++++++--- src/analytics/frontend/tests/messages.py | 25 +++++++------- src/analytics/frontend/tests/test_frontend.py | 21 ++++++++---- 9 files changed, 115 insertions(+), 47 deletions(-) diff --git a/proto/analytics_frontend.proto b/proto/analytics_frontend.proto index bc7420d54..ace0581db 100644 --- a/proto/analytics_frontend.proto +++ b/proto/analytics_frontend.proto @@ -35,18 +35,20 @@ enum AnalyzerOperationMode { ANALYZEROPERATIONMODE_STREAMING = 2; } +// duration field may be added in analyzer... message Analyzer { AnalyzerId analyzer_id = 1; string algorithm_name = 2; // The algorithm to be executed - repeated kpi_manager.KpiId input_kpi_ids = 3; // The KPI Ids to be processed by the analyzer - repeated kpi_manager.KpiId output_kpi_ids = 4; // The KPI Ids produced by the analyzer - AnalyzerOperationMode operation_mode = 5; // Operation mode of the analyzer - map parameters = 6; + float duration_s = 3; // Termiate the data analytics thread after duration (seconds); 0 = infinity time + repeated kpi_manager.KpiId input_kpi_ids = 4; // The KPI Ids to be processed by the analyzer + repeated kpi_manager.KpiId output_kpi_ids = 5; // The KPI Ids produced by the analyzer + AnalyzerOperationMode operation_mode = 6; // Operation mode of the analyzer + map parameters = 7; // Add dictionary of (key, value) pairs such as (window_size, 10) etc. // In batch mode... - float batch_min_duration_s = 7; // ..., min duration to collect before executing batch - float batch_max_duration_s = 8; // ..., max duration collected to execute the batch - uint64 batch_min_size = 9; // ..., min number of samples to collect before executing batch - uint64 batch_max_size = 10; // ..., max number of samples collected to execute the batch + float batch_min_duration_s = 8; // ..., min duration to collect before executing batch + float batch_max_duration_s = 9; // ..., max duration collected to execute the batch + uint64 batch_min_size = 10; // ..., min number of samples to collect before executing batch + uint64 batch_max_size = 11; // ..., max number of samples collected to execute the batch } message AnalyzerFilter { diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 899525fc6..463442f82 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -34,7 +34,7 @@ class AnalyticsBackendService(GenericGrpcService): 'group.id' : 'analytics-frontend', 'auto.offset.reset' : 'latest'}) - def RunSparkStreamer(self, analyzer_id, analyzer): + def StartSparkStreamer(self, analyzer_id, analyzer): kpi_list = analyzer['input_kpis'] oper_list = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())] # TODO: update this line... thresholds = analyzer['thresholds'] @@ -59,17 +59,33 @@ class AnalyticsBackendService(GenericGrpcService): LOGGER.error("Failed to initiate Analyzer backend: {:}".format(e)) return False - def RunRequestListener(self)->bool: - threading.Thread(target=self.RequestListener).start() - return True + def StopRequestListener(self, threadInfo: tuple): + try: + thread, stop_event = threadInfo + stop_event.set() + thread.join() + print ("Terminating Analytics backend RequestListener") + LOGGER.info("Terminating Analytics backend RequestListener") + return True + except Exception as e: + print ("Failed to terminate analytics backend {:}".format(e)) + LOGGER.error("Failed to terminate analytics backend {:}".format(e)) + return False + + def StartRequestListener(self)->tuple: + stop_event = threading.Event() + thread = threading.Thread(target=self.RequestListener, + args=(stop_event,) ) + thread.start() + return (thread, stop_event) - def RequestListener(self): + def RequestListener(self, stop_event): """ listener for requests on Kafka topic. """ consumer = self.kafka_consumer consumer.subscribe([KafkaTopic.ANALYTICS_REQUEST.value]) - while True: + while not stop_event.is_set(): receive_msg = consumer.poll(2.0) if receive_msg is None: continue @@ -87,7 +103,9 @@ class AnalyticsBackendService(GenericGrpcService): if analyzer["algo_name"] is None and analyzer["oper_mode"] is None: self.TerminateAnalyzerBackend(analyzer_uuid) else: - self.RunSparkStreamer(analyzer_uuid, analyzer) + self.StartSparkStreamer(analyzer_uuid, analyzer) + LOGGER.debug("Stop Event activated. Terminating...") + print ("Stop Event activated. Terminating...") def TerminateAnalyzerBackend(self, analyzer_uuid): if analyzer_uuid in self.running_threads: @@ -104,5 +122,5 @@ class AnalyticsBackendService(GenericGrpcService): return False else: print ("Analyzer not found in active collectors. Analyzer Id: {:}".format(analyzer_uuid)) - # LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid)) + LOGGER.warning("Analyzer not found in active collectors: Analyzer Id: {:}".format(analyzer_uuid)) # generate confirmation towards frontend diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index 202076ed5..eaabcfed2 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -142,7 +142,8 @@ def SparkStreamer(kpi_list, oper_list, thresholds, stop_event, # Loop to check for stop event flag. To be set by stop collector method. while True: if stop_event.is_set(): - print ("Stop Event activated. Terminating in 5 seconds...") + LOGGER.debug("Stop Event activated. Terminating in 5 seconds...") + print ("Stop Event activated. Terminating in 5 seconds...") time.sleep(5) queryHandler.stop() break diff --git a/src/analytics/backend/tests/test_backend.py b/src/analytics/backend/tests/test_backend.py index 9e8a0832d..c3e00df35 100644 --- a/src/analytics/backend/tests/test_backend.py +++ b/src/analytics/backend/tests/test_backend.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import time import logging +import threading from common.tools.kafka.Variables import KafkaTopic from analytics.backend.service.AnalyticsBackendService import AnalyticsBackendService from analytics.backend.tests.messages import get_kpi_id_list, get_operation_list, get_threshold_dict @@ -30,12 +32,25 @@ def test_validate_kafka_topics(): response = KafkaTopic.create_all_topics() assert isinstance(response, bool) -def test_RunRequestListener(): +def test_StartRequestListener(): LOGGER.info('test_RunRequestListener') AnalyticsBackendServiceObj = AnalyticsBackendService() - response = AnalyticsBackendServiceObj.RunRequestListener() - LOGGER.debug(str(response)) + response = AnalyticsBackendServiceObj.StartRequestListener() # response is Tuple (thread, stop_event) + LOGGER.debug(str(response)) + assert isinstance(response, tuple) +def test_StopRequestListener(): + LOGGER.info('test_RunRequestListener') + LOGGER.info('Initiating StartRequestListener...') + AnalyticsBackendServiceObj = AnalyticsBackendService() + response_thread = AnalyticsBackendServiceObj.StartRequestListener() # response is Tuple (thread, stop_event) + # LOGGER.debug(str(response_thread)) + time.sleep(10) + LOGGER.info('Initiating StopRequestListener...') + AnalyticsBackendServiceObj = AnalyticsBackendService() + response = AnalyticsBackendServiceObj.StopRequestListener(response_thread) + LOGGER.debug(str(response)) + assert isinstance(response, bool) def test_SparkListener(): LOGGER.info('test_RunRequestListener') diff --git a/src/analytics/database/AnalyzerModel.py b/src/analytics/database/AnalyzerModel.py index 783205269..c33e396e0 100644 --- a/src/analytics/database/AnalyzerModel.py +++ b/src/analytics/database/AnalyzerModel.py @@ -87,7 +87,7 @@ class Analyzer(Base): response = analytics_frontend_pb2.Analyzer() response.analyzer_id.analyzer_id.uuid = row.analyzer_id response.algorithm_name = row.algorithm_name - response.operation_mode = row.operation_mode + response.operation_mode = row.operation_mode.value response.parameters.update(row.parameters) for input_kpi_id in row.input_kpi_ids: diff --git a/src/analytics/database/Analyzer_DB.py b/src/analytics/database/Analyzer_DB.py index 896ba1100..1ba68989a 100644 --- a/src/analytics/database/Analyzer_DB.py +++ b/src/analytics/database/Analyzer_DB.py @@ -15,7 +15,7 @@ import logging import sqlalchemy_utils -from sqlalchemy import inspect +from sqlalchemy import inspect, or_ from sqlalchemy.orm import sessionmaker from analytics.database.AnalyzerModel import Analyzer as AnalyzerModel @@ -120,9 +120,22 @@ class AnalyzerDB: session = self.Session() try: query = session.query(AnalyzerModel) + # Apply filters based on the filter_object - if filter_object.kpi_id: - query = query.filter(AnalyzerModel.kpi_id.in_([k.kpi_id.uuid for k in filter_object.kpi_id])) # Need to be updated + if filter_object.analyzer_id: + query = query.filter(AnalyzerModel.analyzer_id.in_([a.analyzer_id.uuid for a in filter_object.analyzer_id])) + + if filter_object.algorithm_names: + query = query.filter(AnalyzerModel.algorithm_name.in_(filter_object.algorithm_names)) + + if filter_object.input_kpi_ids: + input_kpi_uuids = [k.kpi_id.uuid for k in filter_object.input_kpi_ids] + query = query.filter(AnalyzerModel.input_kpi_ids.op('&&')(input_kpi_uuids)) + + if filter_object.output_kpi_ids: + output_kpi_uuids = [k.kpi_id.uuid for k in filter_object.output_kpi_ids] + query = query.filter(AnalyzerModel.output_kpi_ids.op('&&')(output_kpi_uuids)) + result = query.all() # query should be added to return all rows if result: diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index 9c438761c..f35f035e2 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -126,12 +126,23 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SelectAnalyzers(self, - request : AnalyzerFilter, contextgrpc_context: grpc.ServicerContext # type: ignore + filter : AnalyzerFilter, contextgrpc_context: grpc.ServicerContext # type: ignore ) -> AnalyzerList: # type: ignore - LOGGER.info("At Service gRPC message: {:}".format(request)) + LOGGER.info("At Service gRPC message: {:}".format(filter)) response = AnalyzerList() - - return response + try: + rows = self.db_obj.select_with_filter(AnalyzerModel, filter) + try: + for row in rows: + response.analyzer_list.append( + AnalyzerModel.ConvertRowToAnalyzer(row) + ) + return response + except Exception as e: + LOGGER.info('Unable to process filter response {:}'.format(e)) + except Exception as e: + LOGGER.error('Unable to apply filter on table {:}. ERROR: {:}'.format(AnalyzerModel.__name__, e)) + def delivery_callback(self, err, msg): if err: diff --git a/src/analytics/frontend/tests/messages.py b/src/analytics/frontend/tests/messages.py index 180fac1f8..646de962e 100644 --- a/src/analytics/frontend/tests/messages.py +++ b/src/analytics/frontend/tests/messages.py @@ -21,12 +21,13 @@ from common.proto.analytics_frontend_pb2 import ( AnalyzerOperationMode, Analyze def create_analyzer_id(): _create_analyzer_id = AnalyzerId() # _create_analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) - _create_analyzer_id.analyzer_id.uuid = "9baa306d-d91c-48c2-a92f-76a21bab35b6" + _create_analyzer_id.analyzer_id.uuid = "efef4d95-1cf1-43c4-9742-95c283ddd7a6" return _create_analyzer_id def create_analyzer(): _create_analyzer = Analyzer() - _create_analyzer.analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) + # _create_analyzer.analyzer_id.analyzer_id.uuid = str(uuid.uuid4()) + _create_analyzer.analyzer_id.analyzer_id.uuid = "efef4d95-1cf1-43c4-9742-95c283ddd7a6" _create_analyzer.algorithm_name = "Test_Aggergate_and_Threshold" _create_analyzer.operation_mode = AnalyzerOperationMode.ANALYZEROPERATIONMODE_STREAMING @@ -60,24 +61,24 @@ def create_analyzer_filter(): _create_analyzer_filter = AnalyzerFilter() _analyzer_id_obj = AnalyzerId() - _analyzer_id_obj.analyzer_id.uuid = str(uuid.uuid4()) + # _analyzer_id_obj.analyzer_id.uuid = str(uuid.uuid4()) + _analyzer_id_obj.analyzer_id.uuid = "efef4d95-1cf1-43c4-9742-95c283ddd7a6" _create_analyzer_filter.analyzer_id.append(_analyzer_id_obj) - _create_analyzer_filter.algorithm_names.append('Algorithum1') + _create_analyzer_filter.algorithm_names.append('Test_Aggergate_and_Threshold') - _input_kpi_id_obj = KpiId() - _input_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) - _create_analyzer_filter.input_kpi_ids.append(_input_kpi_id_obj) + # _input_kpi_id_obj = KpiId() + # _input_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + # _create_analyzer_filter.input_kpi_ids.append(_input_kpi_id_obj) # another input kpi Id # _input_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) # _create_analyzer_filter.input_kpi_ids.append(_input_kpi_id_obj) - _output_kpi_id_obj = KpiId() - _output_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) - _create_analyzer_filter.output_kpi_ids.append(_output_kpi_id_obj) - # another output kpi Id + # _output_kpi_id_obj = KpiId() + # _output_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) + # _create_analyzer_filter.output_kpi_ids.append(_output_kpi_id_obj) + # # another output kpi Id # _output_kpi_id_obj.kpi_id.uuid = str(uuid.uuid4()) # _create_analyzer_filter.input_kpi_ids.append(_output_kpi_id_obj) return _create_analyzer_filter - diff --git a/src/analytics/frontend/tests/test_frontend.py b/src/analytics/frontend/tests/test_frontend.py index df6ce165e..b96116d29 100644 --- a/src/analytics/frontend/tests/test_frontend.py +++ b/src/analytics/frontend/tests/test_frontend.py @@ -21,10 +21,11 @@ from common.proto.context_pb2 import Empty from common.Settings import ( get_service_port_grpc, get_env_var_name, ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC ) +from common.tools.kafka.Variables import KafkaTopic from common.proto.analytics_frontend_pb2 import AnalyzerId, AnalyzerList from analytics.frontend.client.AnalyticsFrontendClient import AnalyticsFrontendClient from analytics.frontend.service.AnalyticsFrontendService import AnalyticsFrontendService -from analytics.frontend.tests.messages import ( create_analyzer_id, create_analyzer, +from analytics.frontend.tests.messages import ( create_analyzer_id, create_analyzer, create_analyzer_filter ) ########################### @@ -75,6 +76,12 @@ def analyticsFrontend_client(analyticsFrontend_service : AnalyticsFrontendServic # Tests Implementation of Analytics Frontend ########################### +# --- "test_validate_kafka_topics" should be executed before the functionality tests --- +def test_validate_kafka_topics(): + LOGGER.debug(" >>> test_validate_kafka_topics: START <<< ") + response = KafkaTopic.create_all_topics() + assert isinstance(response, bool) + # ----- core funtionality test ----- def test_StartAnalytics(analyticsFrontend_client): LOGGER.info(' >>> test_StartAnalytic START: <<< ') @@ -82,14 +89,14 @@ def test_StartAnalytics(analyticsFrontend_client): LOGGER.debug(str(response)) assert isinstance(response, AnalyzerId) +def test_SelectAnalytics(analyticsFrontend_client): + LOGGER.info(' >>> test_SelectAnalytics START: <<< ') + response = analyticsFrontend_client.SelectAnalyzers(create_analyzer_filter()) + LOGGER.debug(str(response)) + assert isinstance(response, AnalyzerList) + def test_StopAnalytic(analyticsFrontend_client): LOGGER.info(' >>> test_StopAnalytic START: <<< ') response = analyticsFrontend_client.StopAnalyzer(create_analyzer_id()) LOGGER.debug(str(response)) assert isinstance(response, Empty) - -def test_SelectAnalytics(analyticsFrontend_client): - LOGGER.info(' >>> test_SelectAnalytics START: <<< ') - response = analyticsFrontend_client.SelectAnalyzers(create_analyzer_filter()) - LOGGER.debug(str(response)) - assert isinstance(response, AnalyzerList) \ No newline at end of file -- GitLab From c1ee28db6870f0b0531c2ca42e147afe700c9312 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 13 Sep 2024 09:40:44 +0000 Subject: [PATCH 63/70] Changes in Analytics Backend: - Updated the `StartSparkStreamer` function call to send `Analyzer_uuid` as the key for messages produced on the Kafka topic. - Updated the `SparkStream` definition to receive the key and added the key to the `streamwriter` object. Frontend: - Integrated APScheduler to manage `StreamListener`. - Added `ResponseListener` to consume messages from the `analytics_response_topic` and process them. - Added APScheduler to manage StreamListener. - Added "ResponseListener" to consumer messages from analytics response_topic and process it. --- .../service/AnalyticsBackendService.py | 10 +-- .../backend/service/SparkStreaming.py | 4 +- src/analytics/backend/tests/test_backend.py | 30 +++---- src/analytics/frontend/requirements.in | 1 + .../AnalyticsFrontendServiceServicerImpl.py | 81 ++++++++++++++++--- src/analytics/frontend/tests/test_frontend.py | 62 ++++++++++---- 6 files changed, 142 insertions(+), 46 deletions(-) diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 463442f82..1e0c8a15b 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -34,7 +34,7 @@ class AnalyticsBackendService(GenericGrpcService): 'group.id' : 'analytics-frontend', 'auto.offset.reset' : 'latest'}) - def StartSparkStreamer(self, analyzer_id, analyzer): + def StartSparkStreamer(self, analyzer_uuid, analyzer): kpi_list = analyzer['input_kpis'] oper_list = [s.replace('_value', '') for s in list(analyzer["thresholds"].keys())] # TODO: update this line... thresholds = analyzer['thresholds'] @@ -47,12 +47,12 @@ class AnalyticsBackendService(GenericGrpcService): try: stop_event = threading.Event() thread = threading.Thread(target=SparkStreamer, - args=(kpi_list, oper_list, thresholds, stop_event, + args=(analyzer_uuid, kpi_list, oper_list, thresholds, stop_event, window_size, window_slider, None )) - self.running_threads[analyzer_id] = (thread, stop_event) + self.running_threads[analyzer_uuid] = (thread, stop_event) thread.start() - print ("Initiated Analyzer backend: {:}".format(analyzer_id)) - LOGGER.info("Initiated Analyzer backend: {:}".format(analyzer_id)) + print ("Initiated Analyzer backend: {:}".format(analyzer_uuid)) + LOGGER.info("Initiated Analyzer backend: {:}".format(analyzer_uuid)) return True except Exception as e: print ("Failed to initiate Analyzer backend: {:}".format(e)) diff --git a/src/analytics/backend/service/SparkStreaming.py b/src/analytics/backend/service/SparkStreaming.py index eaabcfed2..96e1aa05d 100644 --- a/src/analytics/backend/service/SparkStreaming.py +++ b/src/analytics/backend/service/SparkStreaming.py @@ -74,7 +74,7 @@ def ApplyThresholds(aggregated_df, thresholds): ) return aggregated_df -def SparkStreamer(kpi_list, oper_list, thresholds, stop_event, +def SparkStreamer(key, kpi_list, oper_list, thresholds, stop_event, window_size=None, win_slide_duration=None, time_stamp_col=None): """ Method to perform Spark operation Kafka stream. @@ -128,7 +128,7 @@ def SparkStreamer(kpi_list, oper_list, thresholds, stop_event, # --- This will write output to Kafka: ACTUAL IMPLEMENTATION query = thresholded_stream_data \ - .selectExpr("CAST(kpi_id AS STRING) AS key", "to_json(struct(*)) AS value") \ + .selectExpr(f"'{key}' AS key", "to_json(struct(*)) AS value") \ .writeStream \ .format("kafka") \ .option("kafka.bootstrap.servers", KafkaConfig.get_kafka_address()) \ diff --git a/src/analytics/backend/tests/test_backend.py b/src/analytics/backend/tests/test_backend.py index c3e00df35..2f40faba9 100644 --- a/src/analytics/backend/tests/test_backend.py +++ b/src/analytics/backend/tests/test_backend.py @@ -32,13 +32,14 @@ def test_validate_kafka_topics(): response = KafkaTopic.create_all_topics() assert isinstance(response, bool) -def test_StartRequestListener(): - LOGGER.info('test_RunRequestListener') - AnalyticsBackendServiceObj = AnalyticsBackendService() - response = AnalyticsBackendServiceObj.StartRequestListener() # response is Tuple (thread, stop_event) - LOGGER.debug(str(response)) - assert isinstance(response, tuple) +# def test_StartRequestListener(): +# LOGGER.info('test_RunRequestListener') +# AnalyticsBackendServiceObj = AnalyticsBackendService() +# response = AnalyticsBackendServiceObj.StartRequestListener() # response is Tuple (thread, stop_event) +# LOGGER.debug(str(response)) +# assert isinstance(response, tuple) +# To test START and STOP communication together def test_StopRequestListener(): LOGGER.info('test_RunRequestListener') LOGGER.info('Initiating StartRequestListener...') @@ -52,11 +53,12 @@ def test_StopRequestListener(): LOGGER.debug(str(response)) assert isinstance(response, bool) -def test_SparkListener(): - LOGGER.info('test_RunRequestListener') - AnalyticsBackendServiceObj = AnalyticsBackendService() - response = AnalyticsBackendServiceObj.RunSparkStreamer( - get_kpi_id_list(), get_operation_list(), get_threshold_dict() - ) - LOGGER.debug(str(response)) - assert isinstance(response, bool) +# To independently tests the SparkListener functionality +# def test_SparkListener(): +# LOGGER.info('test_RunRequestListener') +# AnalyticsBackendServiceObj = AnalyticsBackendService() +# response = AnalyticsBackendServiceObj.RunSparkStreamer( +# get_kpi_id_list(), get_operation_list(), get_threshold_dict() +# ) +# LOGGER.debug(str(response)) +# assert isinstance(response, bool) diff --git a/src/analytics/frontend/requirements.in b/src/analytics/frontend/requirements.in index 1d22df11b..6bf3d7c26 100644 --- a/src/analytics/frontend/requirements.in +++ b/src/analytics/frontend/requirements.in @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +apscheduler==3.10.* # .4 confluent-kafka==2.3.* psycopg2-binary==2.9.* SQLAlchemy==1.4.* diff --git a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py index f35f035e2..8bb6a17af 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py +++ b/src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py @@ -13,7 +13,7 @@ # limitations under the License. -import logging, grpc, json +import logging, grpc, json, queue from typing import Dict from confluent_kafka import Consumer as KafkaConsumer @@ -27,22 +27,24 @@ from common.proto.analytics_frontend_pb2 import Analyzer, AnalyzerId, Analy from common.proto.analytics_frontend_pb2_grpc import AnalyticsFrontendServiceServicer from analytics.database.Analyzer_DB import AnalyzerDB from analytics.database.AnalyzerModel import Analyzer as AnalyzerModel - +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.interval import IntervalTrigger LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('AnalyticsFrontend', 'NBIgRPC') -ACTIVE_ANALYZERS = [] # In case of sevice restarts, the list can be populated from the DB. class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): def __init__(self): LOGGER.info('Init AnalyticsFrontendService') + self.listener_topic = KafkaTopic.ANALYTICS_RESPONSE.value self.db_obj = AnalyzerDB() + self.result_queue = queue.Queue() + self.scheduler = BackgroundScheduler() self.kafka_producer = KafkaProducer({'bootstrap.servers' : KafkaConfig.get_kafka_address()}) self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'analytics-frontend', 'auto.offset.reset' : 'latest'}) - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StartAnalyzer(self, request : Analyzer, grpc_context: grpc.ServicerContext # type: ignore @@ -80,9 +82,64 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): callback = self.delivery_callback ) LOGGER.info("Analyzer Start Request Generated: Analyzer Id: {:}, Value: {:}".format(analyzer_uuid, analyzer_to_generate)) - ACTIVE_ANALYZERS.append(analyzer_uuid) self.kafka_producer.flush() + + # self.StartResponseListener(analyzer_uuid) + def StartResponseListener(self, filter_key=None): + """ + Start the Kafka response listener with APScheduler and return key-value pairs periodically. + """ + LOGGER.info("Starting StartResponseListener") + # Schedule the ResponseListener at fixed intervals + self.scheduler.add_job( + self.response_listener, + trigger=IntervalTrigger(seconds=5), + args=[filter_key], + id=f"response_listener_{self.listener_topic}", + replace_existing=True + ) + self.scheduler.start() + LOGGER.info(f"Started Kafka listener for topic {self.listener_topic}...") + try: + while True: + LOGGER.info("entering while...") + key, value = self.result_queue.get() # Wait until a result is available + LOGGER.info("In while true ...") + yield key, value # Yield the result to the calling function + except KeyboardInterrupt: + LOGGER.warning("Listener stopped manually.") + finally: + self.StopListener() + + def response_listener(self, filter_key=None): + """ + Poll Kafka messages and put key-value pairs into the queue. + """ + LOGGER.info(f"Polling Kafka topic {self.listener_topic}...") + + consumer = self.kafka_consumer + consumer.subscribe([self.listener_topic]) + msg = consumer.poll(2.0) + if msg is None: + return + elif msg.error(): + if msg.error().code() != KafkaError._PARTITION_EOF: + LOGGER.error(f"Kafka error: {msg.error()}") + return + + try: + key = msg.key().decode('utf-8') if msg.key() else None + if filter_key is not None and key == filter_key: + value = json.loads(msg.value().decode('utf-8')) + LOGGER.info(f"Received key: {key}, value: {value}") + self.result_queue.put((key, value)) + else: + LOGGER.info(f"Skipping message with unmatched key: {key}") + # value = json.loads(msg.value().decode('utf-8')) # Added for debugging + # self.result_queue.put((filter_key, value)) # Added for debugging + except Exception as e: + LOGGER.error(f"Error processing Kafka message: {e}") @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def StopAnalyzer(self, @@ -118,11 +175,15 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): ) LOGGER.info("Analyzer Stop Request Generated: Analyzer Id: {:}".format(analyzer_uuid)) self.kafka_producer.flush() - try: - ACTIVE_ANALYZERS.remove(analyzer_uuid) - except ValueError: - LOGGER.warning('Analyzer ID {:} not found in active analyzers'.format(analyzer_uuid)) + self.StopListener() + def StopListener(self): + """ + Gracefully stop the Kafka listener and the scheduler. + """ + LOGGER.info("Stopping Kafka listener...") + self.scheduler.shutdown() + LOGGER.info("Kafka listener stopped.") @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SelectAnalyzers(self, @@ -147,7 +208,7 @@ class AnalyticsFrontendServiceServicerImpl(AnalyticsFrontendServiceServicer): def delivery_callback(self, err, msg): if err: LOGGER.debug('Message delivery failed: {:}'.format(err)) - print('Message delivery failed: {:}'.format(err)) + print ('Message delivery failed: {:}'.format(err)) # else: # LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) # print('Message delivered to topic {:}'.format(msg.topic())) diff --git a/src/analytics/frontend/tests/test_frontend.py b/src/analytics/frontend/tests/test_frontend.py index b96116d29..d2428c01f 100644 --- a/src/analytics/frontend/tests/test_frontend.py +++ b/src/analytics/frontend/tests/test_frontend.py @@ -13,8 +13,11 @@ # limitations under the License. import os +import time +import json import pytest import logging +import threading from common.Constants import ServiceNameEnum from common.proto.context_pb2 import Empty @@ -27,6 +30,10 @@ from analytics.frontend.client.AnalyticsFrontendClient import AnalyticsFronten from analytics.frontend.service.AnalyticsFrontendService import AnalyticsFrontendService from analytics.frontend.tests.messages import ( create_analyzer_id, create_analyzer, create_analyzer_filter ) +from analytics.frontend.service.AnalyticsFrontendServiceServicerImpl import AnalyticsFrontendServiceServicerImpl +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.interval import IntervalTrigger + ########################### # Tests Setup @@ -83,20 +90,45 @@ def test_validate_kafka_topics(): assert isinstance(response, bool) # ----- core funtionality test ----- -def test_StartAnalytics(analyticsFrontend_client): - LOGGER.info(' >>> test_StartAnalytic START: <<< ') - response = analyticsFrontend_client.StartAnalyzer(create_analyzer()) - LOGGER.debug(str(response)) - assert isinstance(response, AnalyzerId) - -def test_SelectAnalytics(analyticsFrontend_client): - LOGGER.info(' >>> test_SelectAnalytics START: <<< ') - response = analyticsFrontend_client.SelectAnalyzers(create_analyzer_filter()) +# def test_StartAnalytics(analyticsFrontend_client): +# LOGGER.info(' >>> test_StartAnalytic START: <<< ') +# response = analyticsFrontend_client.StartAnalyzer(create_analyzer()) +# LOGGER.debug(str(response)) +# assert isinstance(response, AnalyzerId) + +# To test start and stop listener together +def test_StartStopAnalyzers(analyticsFrontend_client): + LOGGER.info(' >>> test_StartStopAnalyzers START: <<< ') + LOGGER.info('--> StartAnalyzer') + added_analyzer_id = analyticsFrontend_client.StartAnalyzer(create_analyzer()) + LOGGER.debug(str(added_analyzer_id)) + LOGGER.info(' --> Calling StartResponseListener... ') + class_obj = AnalyticsFrontendServiceServicerImpl() + response = class_obj.StartResponseListener(added_analyzer_id.analyzer_id._uuid) + LOGGER.debug(response) + LOGGER.info("waiting for timer to comlete ...") + time.sleep(3) + LOGGER.info('--> StopAnalyzer') + response = analyticsFrontend_client.StopAnalyzer(added_analyzer_id) LOGGER.debug(str(response)) - assert isinstance(response, AnalyzerList) -def test_StopAnalytic(analyticsFrontend_client): - LOGGER.info(' >>> test_StopAnalytic START: <<< ') - response = analyticsFrontend_client.StopAnalyzer(create_analyzer_id()) - LOGGER.debug(str(response)) - assert isinstance(response, Empty) +# def test_SelectAnalytics(analyticsFrontend_client): +# LOGGER.info(' >>> test_SelectAnalytics START: <<< ') +# response = analyticsFrontend_client.SelectAnalyzers(create_analyzer_filter()) +# LOGGER.debug(str(response)) +# assert isinstance(response, AnalyzerList) + +# def test_StopAnalytic(analyticsFrontend_client): +# LOGGER.info(' >>> test_StopAnalytic START: <<< ') +# response = analyticsFrontend_client.StopAnalyzer(create_analyzer_id()) +# LOGGER.debug(str(response)) +# assert isinstance(response, Empty) + +# def test_ResponseListener(): +# LOGGER.info(' >>> test_ResponseListener START <<< ') +# analyzer_id = create_analyzer_id() +# LOGGER.debug("Starting Response Listener for Analyzer ID: {:}".format(analyzer_id.analyzer_id.uuid)) +# class_obj = AnalyticsFrontendServiceServicerImpl() +# for response in class_obj.StartResponseListener(analyzer_id.analyzer_id.uuid): +# LOGGER.debug(response) +# assert isinstance(response, tuple) \ No newline at end of file -- GitLab From 2c1fc9c8e466f628ee5ed0920cb650614be80151 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 13 Sep 2024 15:45:28 +0000 Subject: [PATCH 64/70] Changes in Analytic, manifest files and deployment script Deployment Script - Analytics is added in new montioring component to be deployed with TFS. - condition is added in analytics component TFS deployment script. - Analytics show logs files are added. Manifest - modifed KAFKA_ADVERTISED_LISTENERS with internal kafka service address in kafka.yml - analyticsservice.yml added. Analytics - grpc service is added in backend - Docker file for backend and frontend is added. - main file is updaed - small changes in requirements.in files Kafka Variables - get_kafka_address method logic is updated. --- deploy/all.sh | 2 +- deploy/kafka.sh | 2 +- deploy/tfs.sh | 8 +- manifests/analyticsservice.yaml | 128 ++++++++++++++++++ manifests/kafka/02-kafka.yaml | 4 +- .../run_tests_locally-telemetry-backend.sh | 4 +- scripts/show_logs_analytics_backend.sh | 27 ++++ scripts/show_logs_analytics_frontend.sh | 27 ++++ src/analytics/backend/Dockerfile | 69 ++++++++++ src/analytics/backend/requirements.in | 1 - .../service/AnalyticsBackendService.py | 8 +- src/analytics/frontend/Dockerfile | 2 +- src/analytics/frontend/requirements.in | 2 +- src/analytics/frontend/service/__main__.py | 11 +- src/common/tools/kafka/Variables.py | 11 +- 15 files changed, 283 insertions(+), 23 deletions(-) create mode 100644 manifests/analyticsservice.yaml create mode 100755 scripts/show_logs_analytics_backend.sh create mode 100755 scripts/show_logs_analytics_frontend.sh create mode 100644 src/analytics/backend/Dockerfile diff --git a/deploy/all.sh b/deploy/all.sh index e9b33b469..06b8ee701 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -33,7 +33,7 @@ export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice n #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate Monitoring Framework (new) -#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api" +#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics" # Uncomment to activate BGP-LS Speaker #export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker" diff --git a/deploy/kafka.sh b/deploy/kafka.sh index f86108011..0483bce15 100755 --- a/deploy/kafka.sh +++ b/deploy/kafka.sh @@ -78,7 +78,7 @@ function kafka_deploy() { echo "Apache Kafka" echo ">>> Checking if Apache Kafka is deployed ... " -if [ "$KFK_REDEPLOY" = "YES" ]; then +if [ "$KFK_REDEPLOY" == "YES" ]; then echo ">>> Redeploying kafka namespace" kafka_deploy elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then diff --git a/deploy/tfs.sh b/deploy/tfs.sh index b756ad2d0..1dceae1c1 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -194,7 +194,7 @@ kubectl create secret generic crdb-analytics --namespace ${TFS_K8S_NAMESPACE} -- --from-literal=CRDB_SSLMODE=require printf "\n" -echo "Create secret with Apache Kafka data for KPI and Telemetry microservices" +echo "Create secret with Apache Kafka data for KPI, Telemetry and Analytics microservices" KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}') kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \ --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \ @@ -276,7 +276,7 @@ for COMPONENT in $TFS_COMPONENTS; do if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" - elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then + elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" @@ -299,7 +299,7 @@ for COMPONENT in $TFS_COMPONENTS; do echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..." - if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then + if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ] ; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" @@ -350,7 +350,7 @@ for COMPONENT in $TFS_COMPONENTS; do cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST" fi - if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ]; then + if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" diff --git a/manifests/analyticsservice.yaml b/manifests/analyticsservice.yaml new file mode 100644 index 000000000..9fbdc642f --- /dev/null +++ b/manifests/analyticsservice.yaml @@ -0,0 +1,128 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: analyticsservice +spec: + selector: + matchLabels: + app: analyticsservice + #replicas: 1 + template: + metadata: + labels: + app: analyticsservice + spec: + terminationGracePeriodSeconds: 5 + containers: + - name: frontend + image: labs.etsi.org:5050/tfs/controller/analytics-frontend:latest + imagePullPolicy: Always + ports: + - containerPort: 30080 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + envFrom: + - secretRef: + name: crdb-analytics + - secretRef: + name: kfk-kpi-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30050"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30050"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi + - name: backend + image: labs.etsi.org:5050/tfs/controller/analytics-backend:latest + imagePullPolicy: Always + ports: + - containerPort: 30090 + - containerPort: 9192 + env: + - name: LOG_LEVEL + value: "INFO" + envFrom: + - secretRef: + name: kfk-kpi-data + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30060"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:30060"] + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: analyticsservice + labels: + app: analyticsservice +spec: + type: ClusterIP + selector: + app: analyticsservice + ports: + - name: frontend-grpc + protocol: TCP + port: 30080 + targetPort: 30080 + - name: backend-grpc + protocol: TCP + port: 30090 + targetPort: 30090 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: analyticsservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: analyticsservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/manifests/kafka/02-kafka.yaml b/manifests/kafka/02-kafka.yaml index 8e4562e6e..8400f5944 100644 --- a/manifests/kafka/02-kafka.yaml +++ b/manifests/kafka/02-kafka.yaml @@ -53,9 +53,9 @@ spec: - name: KAFKA_LISTENERS value: PLAINTEXT://:9092 - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://localhost:9092 + value: PLAINTEXT://kafka-service.kafka.svc.cluster.local:9092 image: wurstmeister/kafka imagePullPolicy: IfNotPresent name: kafka-broker ports: - - containerPort: 9092 \ No newline at end of file + - containerPort: 9092 diff --git a/scripts/run_tests_locally-telemetry-backend.sh b/scripts/run_tests_locally-telemetry-backend.sh index 9cf404ffc..79db05fcf 100755 --- a/scripts/run_tests_locally-telemetry-backend.sh +++ b/scripts/run_tests_locally-telemetry-backend.sh @@ -24,5 +24,5 @@ cd $PROJECTDIR/src # python3 kpi_manager/tests/test_unitary.py RCFILE=$PROJECTDIR/coverage/.coveragerc -python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \ - telemetry/backend/tests/testTelemetryBackend.py +python3 -m pytest --log-level=INFO --log-cli-level=debug --verbose \ + telemetry/backend/tests/test_TelemetryBackend.py diff --git a/scripts/show_logs_analytics_backend.sh b/scripts/show_logs_analytics_backend.sh new file mode 100755 index 000000000..afb58567c --- /dev/null +++ b/scripts/show_logs_analytics_backend.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/analyticsservice -c backend diff --git a/scripts/show_logs_analytics_frontend.sh b/scripts/show_logs_analytics_frontend.sh new file mode 100755 index 000000000..6d3fae10b --- /dev/null +++ b/scripts/show_logs_analytics_frontend.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################################################################## +# Define your deployment settings here +######################################################################################################################## + +# If not already set, set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} + +######################################################################################################################## +# Automated steps start here +######################################################################################################################## + +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/analyticsservice -c frontend diff --git a/src/analytics/backend/Dockerfile b/src/analytics/backend/Dockerfile new file mode 100644 index 000000000..17adcd3ab --- /dev/null +++ b/src/analytics/backend/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/analytics/backend +WORKDIR /var/teraflow/analytics/backend +COPY src/analytics/backend/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/analytics/__init__.py analytics/__init__.py +COPY src/analytics/backend/. analytics/backend/ + +# Start the service +ENTRYPOINT ["python", "-m", "analytics.backend.service"] diff --git a/src/analytics/backend/requirements.in b/src/analytics/backend/requirements.in index 5c2280c5d..e2917029e 100644 --- a/src/analytics/backend/requirements.in +++ b/src/analytics/backend/requirements.in @@ -12,6 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -java==11.0.* pyspark==3.5.2 confluent-kafka==2.3.* \ No newline at end of file diff --git a/src/analytics/backend/service/AnalyticsBackendService.py b/src/analytics/backend/service/AnalyticsBackendService.py index 1e0c8a15b..595603567 100755 --- a/src/analytics/backend/service/AnalyticsBackendService.py +++ b/src/analytics/backend/service/AnalyticsBackendService.py @@ -21,6 +21,9 @@ from analytics.backend.service.SparkStreaming import SparkStreamer from common.tools.kafka.Variables import KafkaConfig, KafkaTopic from confluent_kafka import Consumer as KafkaConsumer from confluent_kafka import KafkaError +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc + LOGGER = logging.getLogger(__name__) @@ -29,6 +32,9 @@ class AnalyticsBackendService(GenericGrpcService): Class listens for ... """ def __init__(self, cls_name : str = __name__) -> None: + LOGGER.info('Init AnalyticsBackendService') + port = get_service_port_grpc(ServiceNameEnum.ANALYTICSBACKEND) + super().__init__(port, cls_name=cls_name) self.running_threads = {} # To keep track of all running analyzers self.kafka_consumer = KafkaConsumer({'bootstrap.servers' : KafkaConfig.get_kafka_address(), 'group.id' : 'analytics-frontend', @@ -72,7 +78,7 @@ class AnalyticsBackendService(GenericGrpcService): LOGGER.error("Failed to terminate analytics backend {:}".format(e)) return False - def StartRequestListener(self)->tuple: + def install_services(self): stop_event = threading.Event() thread = threading.Thread(target=self.RequestListener, args=(stop_event,) ) diff --git a/src/analytics/frontend/Dockerfile b/src/analytics/frontend/Dockerfile index f3b8838b2..10499713f 100644 --- a/src/analytics/frontend/Dockerfile +++ b/src/analytics/frontend/Dockerfile @@ -55,7 +55,7 @@ RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/analytics/frontend -WORKDIR /var/analyticstelemetry/frontend +WORKDIR /var/teraflow/analytics/frontend COPY src/analytics/frontend/requirements.in requirements.in RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt diff --git a/src/analytics/frontend/requirements.in b/src/analytics/frontend/requirements.in index 6bf3d7c26..20000420c 100644 --- a/src/analytics/frontend/requirements.in +++ b/src/analytics/frontend/requirements.in @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -apscheduler==3.10.* # .4 +apscheduler==3.10.4 confluent-kafka==2.3.* psycopg2-binary==2.9.* SQLAlchemy==1.4.* diff --git a/src/analytics/frontend/service/__main__.py b/src/analytics/frontend/service/__main__.py index e33a4c62b..3fa2ca875 100644 --- a/src/analytics/frontend/service/__main__.py +++ b/src/analytics/frontend/service/__main__.py @@ -13,7 +13,8 @@ # limitations under the License. import logging, signal, sys, threading -from common.Settings import get_log_level +from prometheus_client import start_http_server +from common.Settings import get_log_level, get_metrics_port from .AnalyticsFrontendService import AnalyticsFrontendService @@ -28,13 +29,17 @@ def main(): global LOGGER # pylint: disable=global-statement log_level = get_log_level() - logging.basicConfig(level=log_level) + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") LOGGER = logging.getLogger(__name__) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - LOGGER.debug('Starting...') + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) grpc_service = AnalyticsFrontendService() grpc_service.start() diff --git a/src/common/tools/kafka/Variables.py b/src/common/tools/kafka/Variables.py index 215913c0e..fc43c3151 100644 --- a/src/common/tools/kafka/Variables.py +++ b/src/common/tools/kafka/Variables.py @@ -14,7 +14,6 @@ import logging from enum import Enum -from confluent_kafka import KafkaException from confluent_kafka.admin import AdminClient, NewTopic from common.Settings import get_setting @@ -26,11 +25,11 @@ class KafkaConfig(Enum): @staticmethod def get_kafka_address() -> str: - kafka_server_address = get_setting('KFK_SERVER_ADDRESS', default=None) - if kafka_server_address is None: - KFK_NAMESPACE = get_setting('KFK_NAMESPACE') - KFK_PORT = get_setting('KFK_SERVER_PORT') - kafka_server_address = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) + # kafka_server_address = get_setting('KFK_SERVER_ADDRESS', default=None) + # if kafka_server_address is None: + KFK_NAMESPACE = get_setting('KFK_NAMESPACE') + KFK_PORT = get_setting('KFK_SERVER_PORT') + kafka_server_address = KFK_SERVER_ADDRESS_TEMPLATE.format(KFK_NAMESPACE, KFK_PORT) return kafka_server_address @staticmethod -- GitLab From 8e0f060131795d50b5e14a1671dc96600bc272ff Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 16:00:37 +0000 Subject: [PATCH 65/70] Analytics component: - Corrected Liveness/Readiness probes --- manifests/analyticsservice.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/analyticsservice.yaml b/manifests/analyticsservice.yaml index 9fbdc642f..0fa3ed0be 100644 --- a/manifests/analyticsservice.yaml +++ b/manifests/analyticsservice.yaml @@ -44,10 +44,10 @@ spec: name: kfk-kpi-data readinessProbe: exec: - command: ["/bin/grpc_health_probe", "-addr=:30050"] + command: ["/bin/grpc_health_probe", "-addr=:30080"] livenessProbe: exec: - command: ["/bin/grpc_health_probe", "-addr=:30050"] + command: ["/bin/grpc_health_probe", "-addr=:30080"] resources: requests: cpu: 250m @@ -69,10 +69,10 @@ spec: name: kfk-kpi-data readinessProbe: exec: - command: ["/bin/grpc_health_probe", "-addr=:30060"] + command: ["/bin/grpc_health_probe", "-addr=:30090"] livenessProbe: exec: - command: ["/bin/grpc_health_probe", "-addr=:30060"] + command: ["/bin/grpc_health_probe", "-addr=:30090"] resources: requests: cpu: 250m -- GitLab From 059027081c0ab9169594653e083a501ee2f836ce Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 16:19:22 +0000 Subject: [PATCH 66/70] Pre-merge commit --- ... run_tests_locally-analytics-backend__.sh} | 0 scripts/run_tests_locally-telemetry-mgtDB.sh | 26 +++++++++++++++++++ src/analytics/__init__.py | 2 -- src/telemetry/database/tests/__init__.py | 14 ++++++++++ src/telemetry/frontend/tests/__init__.py | 15 +++++++++++ 5 files changed, 55 insertions(+), 2 deletions(-) rename scripts/{run_tests_locally-analytics-backend.sh => run_tests_locally-analytics-backend__.sh} (100%) create mode 100755 scripts/run_tests_locally-telemetry-mgtDB.sh create mode 100644 src/telemetry/database/tests/__init__.py create mode 100644 src/telemetry/frontend/tests/__init__.py diff --git a/scripts/run_tests_locally-analytics-backend.sh b/scripts/run_tests_locally-analytics-backend__.sh similarity index 100% rename from scripts/run_tests_locally-analytics-backend.sh rename to scripts/run_tests_locally-analytics-backend__.sh diff --git a/scripts/run_tests_locally-telemetry-mgtDB.sh b/scripts/run_tests_locally-telemetry-mgtDB.sh new file mode 100755 index 000000000..4b9a41760 --- /dev/null +++ b/scripts/run_tests_locally-telemetry-mgtDB.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc +# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ +# kpi_manager/tests/test_unitary.py + +RCFILE=$PROJECTDIR/coverage/.coveragerc +python3 -m pytest --log-level=DEBUG --log-cli-level=debug --verbose \ + telemetry/tests/test_telemetryDB.py diff --git a/src/analytics/__init__.py b/src/analytics/__init__.py index 234a1af65..bbfc943b6 100644 --- a/src/analytics/__init__.py +++ b/src/analytics/__init__.py @@ -1,4 +1,3 @@ - # Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/src/telemetry/database/tests/__init__.py b/src/telemetry/database/tests/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/telemetry/database/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/telemetry/frontend/tests/__init__.py b/src/telemetry/frontend/tests/__init__.py new file mode 100644 index 000000000..234a1af65 --- /dev/null +++ b/src/telemetry/frontend/tests/__init__.py @@ -0,0 +1,15 @@ + +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -- GitLab From 08b15bed236bb4b9ffa9a5ba9e08074a22a20c5b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 16:28:18 +0000 Subject: [PATCH 67/70] Analytics: fixed main.py --- src/analytics/backend/service/__main__.py | 17 +++++++++++------ src/analytics/frontend/service/__main__.py | 5 ++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/analytics/backend/service/__main__.py b/src/analytics/backend/service/__main__.py index 371b5a7ca..3c4c36b7c 100644 --- a/src/analytics/backend/service/__main__.py +++ b/src/analytics/backend/service/__main__.py @@ -13,8 +13,9 @@ # limitations under the License. import logging, signal, sys, threading -from common.Settings import get_log_level -from analytics.backend.service.AnalyticsBackendService import AnalyticsBackendService +from prometheus_client import start_http_server +from common.Settings import get_log_level, get_metrics_port +from .AnalyticsBackendService import AnalyticsBackendService terminate = threading.Event() LOGGER = None @@ -27,13 +28,17 @@ def main(): global LOGGER # pylint: disable=global-statement log_level = get_log_level() - logging.basicConfig(level=log_level) + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") LOGGER = logging.getLogger(__name__) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - LOGGER.debug('Starting...') + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) grpc_service = AnalyticsBackendService() grpc_service.start() @@ -41,10 +46,10 @@ def main(): # Wait for Ctrl+C or termination signal while not terminate.wait(timeout=1.0): pass - LOGGER.debug('Terminating...') + LOGGER.info('Terminating...') grpc_service.stop() - LOGGER.debug('Bye') + LOGGER.info('Bye') return 0 if __name__ == '__main__': diff --git a/src/analytics/frontend/service/__main__.py b/src/analytics/frontend/service/__main__.py index 3fa2ca875..6c331844f 100644 --- a/src/analytics/frontend/service/__main__.py +++ b/src/analytics/frontend/service/__main__.py @@ -17,7 +17,6 @@ from prometheus_client import start_http_server from common.Settings import get_log_level, get_metrics_port from .AnalyticsFrontendService import AnalyticsFrontendService - terminate = threading.Event() LOGGER = None @@ -47,10 +46,10 @@ def main(): # Wait for Ctrl+C or termination signal while not terminate.wait(timeout=1.0): pass - LOGGER.debug('Terminating...') + LOGGER.info('Terminating...') grpc_service.stop() - LOGGER.debug('Bye') + LOGGER.info('Bye') return 0 if __name__ == '__main__': -- GitLab From 368cbf299b8b05e2bb7c15ed1c9ee5776ec98913 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 16:38:46 +0000 Subject: [PATCH 68/70] Pre-merge Cosmetic changes --- .gitignore | 2 +- src/analytics/backend/requirements.in | 2 +- src/analytics/frontend/__init__.py | 1 - .../frontend/client/AnalyticsFrontendClient.py | 2 +- src/analytics/frontend/requirements.in | 2 +- .../frontend/service/AnalyticsFrontendService.py | 2 +- src/analytics/frontend/service/__init__.py | 1 - src/analytics/frontend/tests/__init__.py | 14 ++++++++++++++ src/analytics/requirements.in | 2 +- src/analytics/tests/__init__.py | 14 ++++++++++++++ src/kpi_manager/database/KpiEngine.py | 6 +----- 11 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 src/analytics/frontend/tests/__init__.py create mode 100644 src/analytics/tests/__init__.py diff --git a/.gitignore b/.gitignore index 6a53f106e..e1f87cfd3 100644 --- a/.gitignore +++ b/.gitignore @@ -178,4 +178,4 @@ libyang/ **/logs/*.log.* # PySpark checkpoints -src/analytics/.spark/* \ No newline at end of file +src/analytics/.spark/* diff --git a/src/analytics/backend/requirements.in b/src/analytics/backend/requirements.in index e2917029e..9df678fe8 100644 --- a/src/analytics/backend/requirements.in +++ b/src/analytics/backend/requirements.in @@ -13,4 +13,4 @@ # limitations under the License. pyspark==3.5.2 -confluent-kafka==2.3.* \ No newline at end of file +confluent-kafka==2.3.* diff --git a/src/analytics/frontend/__init__.py b/src/analytics/frontend/__init__.py index 234a1af65..3ee6f7071 100644 --- a/src/analytics/frontend/__init__.py +++ b/src/analytics/frontend/__init__.py @@ -1,4 +1,3 @@ - # Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/analytics/frontend/client/AnalyticsFrontendClient.py b/src/analytics/frontend/client/AnalyticsFrontendClient.py index bfa8cae45..90e95d661 100644 --- a/src/analytics/frontend/client/AnalyticsFrontendClient.py +++ b/src/analytics/frontend/client/AnalyticsFrontendClient.py @@ -65,4 +65,4 @@ class AnalyticsFrontendClient: LOGGER.debug('SelectAnalyzers: {:s}'.format(grpc_message_to_json_string(request))) response = self.stub.SelectAnalyzers(request) LOGGER.debug('SelectAnalyzers result: {:s}'.format(grpc_message_to_json_string(response))) - return response \ No newline at end of file + return response diff --git a/src/analytics/frontend/requirements.in b/src/analytics/frontend/requirements.in index 20000420c..d81b9ddbe 100644 --- a/src/analytics/frontend/requirements.in +++ b/src/analytics/frontend/requirements.in @@ -17,4 +17,4 @@ confluent-kafka==2.3.* psycopg2-binary==2.9.* SQLAlchemy==1.4.* sqlalchemy-cockroachdb==1.4.* -SQLAlchemy-Utils==0.38.* \ No newline at end of file +SQLAlchemy-Utils==0.38.* diff --git a/src/analytics/frontend/service/AnalyticsFrontendService.py b/src/analytics/frontend/service/AnalyticsFrontendService.py index e702c0144..42a7fc9b6 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendService.py +++ b/src/analytics/frontend/service/AnalyticsFrontendService.py @@ -25,4 +25,4 @@ class AnalyticsFrontendService(GenericGrpcService): self.analytics_frontend_servicer = AnalyticsFrontendServiceServicerImpl() def install_servicers(self): - add_AnalyticsFrontendServiceServicer_to_server(self.analytics_frontend_servicer, self.server) \ No newline at end of file + add_AnalyticsFrontendServiceServicer_to_server(self.analytics_frontend_servicer, self.server) diff --git a/src/analytics/frontend/service/__init__.py b/src/analytics/frontend/service/__init__.py index 234a1af65..3ee6f7071 100644 --- a/src/analytics/frontend/service/__init__.py +++ b/src/analytics/frontend/service/__init__.py @@ -1,4 +1,3 @@ - # Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/analytics/frontend/tests/__init__.py b/src/analytics/frontend/tests/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/analytics/frontend/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/analytics/requirements.in b/src/analytics/requirements.in index 98cf96710..8ff30ddaa 100644 --- a/src/analytics/requirements.in +++ b/src/analytics/requirements.in @@ -18,4 +18,4 @@ confluent-kafka==2.3.* psycopg2-binary==2.9.* SQLAlchemy==1.4.* sqlalchemy-cockroachdb==1.4.* -SQLAlchemy-Utils==0.38.* \ No newline at end of file +SQLAlchemy-Utils==0.38.* diff --git a/src/analytics/tests/__init__.py b/src/analytics/tests/__init__.py new file mode 100644 index 000000000..3ee6f7071 --- /dev/null +++ b/src/analytics/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/kpi_manager/database/KpiEngine.py b/src/kpi_manager/database/KpiEngine.py index dff406de6..0fce7e3d3 100644 --- a/src/kpi_manager/database/KpiEngine.py +++ b/src/kpi_manager/database/KpiEngine.py @@ -16,8 +16,6 @@ import logging, sqlalchemy from common.Settings import get_setting LOGGER = logging.getLogger(__name__) - -# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' class KpiEngine: @@ -33,12 +31,10 @@ class KpiEngine: CRDB_SSLMODE = get_setting('CRDB_SSLMODE') crdb_uri = CRDB_URI_TEMPLATE.format( CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) - # crdb_uri = CRDB_URI_TEMPLATE.format( - # CRDB_USERNAME, CRDB_PASSWORD, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: engine = sqlalchemy.create_engine(crdb_uri, echo=False) LOGGER.info(' KpiDBmanager initalized with DB URL: {:}'.format(crdb_uri)) except: # pylint: disable=bare-except # pragma: no cover LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) return None # type: ignore - return engine + return engine -- GitLab From 25c86b1776640614d51898d710c88f1290f8f23d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 16:39:07 +0000 Subject: [PATCH 69/70] Added Analytics and Telemetry dynamic DB Engine configuration --- src/analytics/database/AnalyzerEngine.py | 20 ++++++++++---------- src/telemetry/database/TelemetryEngine.py | 20 +++++++++----------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/analytics/database/AnalyzerEngine.py b/src/analytics/database/AnalyzerEngine.py index 4bed9f93a..9294e0996 100644 --- a/src/analytics/database/AnalyzerEngine.py +++ b/src/analytics/database/AnalyzerEngine.py @@ -22,19 +22,19 @@ class AnalyzerEngine: @staticmethod def get_engine() -> sqlalchemy.engine.Engine: crdb_uri = get_setting('CRDB_URI', default=None) - if crdb_uri is None: - CRDB_NAMESPACE = "crdb" - CRDB_SQL_PORT = "26257" - CRDB_DATABASE = "tfs-analyzer" - CRDB_USERNAME = "tfs" - CRDB_PASSWORD = "tfs123" - CRDB_SSLMODE = "require" + if crdb_uri is None: + CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') + CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') + CRDB_DATABASE = "tfs-analyzer" # TODO: define variable get_setting('CRDB_DATABASE_KPI_MGMT') + CRDB_USERNAME = get_setting('CRDB_USERNAME') + CRDB_PASSWORD = get_setting('CRDB_PASSWORD') + CRDB_SSLMODE = get_setting('CRDB_SSLMODE') crdb_uri = CRDB_URI_TEMPLATE.format( - CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: engine = sqlalchemy.create_engine(crdb_uri, echo=False) LOGGER.info(' AnalyzerDB initalized with DB URL: {:}'.format(crdb_uri)) - except: + except: # pylint: disable=bare-except # pragma: no cover LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) - return None + return None # type: ignore return engine diff --git a/src/telemetry/database/TelemetryEngine.py b/src/telemetry/database/TelemetryEngine.py index 18ec2ddbc..7c8620faf 100644 --- a/src/telemetry/database/TelemetryEngine.py +++ b/src/telemetry/database/TelemetryEngine.py @@ -16,27 +16,25 @@ import logging, sqlalchemy from common.Settings import get_setting LOGGER = logging.getLogger(__name__) - -# CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@127.0.0.1:{:s}/{:s}?sslmode={:s}' CRDB_URI_TEMPLATE = 'cockroachdb://{:s}:{:s}@cockroachdb-public.{:s}.svc.cluster.local:{:s}/{:s}?sslmode={:s}' class TelemetryEngine: @staticmethod def get_engine() -> sqlalchemy.engine.Engine: crdb_uri = get_setting('CRDB_URI', default=None) - if crdb_uri is None: - CRDB_NAMESPACE = "crdb" - CRDB_SQL_PORT = "26257" - CRDB_DATABASE = "tfs-telemetry" - CRDB_USERNAME = "tfs" - CRDB_PASSWORD = "tfs123" - CRDB_SSLMODE = "require" + if crdb_uri is None: + CRDB_NAMESPACE = get_setting('CRDB_NAMESPACE') + CRDB_SQL_PORT = get_setting('CRDB_SQL_PORT') + CRDB_DATABASE = "tfs-telemetry" # TODO: define variable get_setting('CRDB_DATABASE_KPI_MGMT') + CRDB_USERNAME = get_setting('CRDB_USERNAME') + CRDB_PASSWORD = get_setting('CRDB_PASSWORD') + CRDB_SSLMODE = get_setting('CRDB_SSLMODE') crdb_uri = CRDB_URI_TEMPLATE.format( - CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) + CRDB_USERNAME, CRDB_PASSWORD, CRDB_NAMESPACE, CRDB_SQL_PORT, CRDB_DATABASE, CRDB_SSLMODE) try: engine = sqlalchemy.create_engine(crdb_uri, echo=False) LOGGER.info(' TelemetryDB initalized with DB URL: {:}'.format(crdb_uri)) except: # pylint: disable=bare-except # pragma: no cover LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri))) return None # type: ignore - return engine # type: ignore + return engine -- GitLab From a94fe1fdfe078db28a729adf2b177f17632e5aaa Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 13 Sep 2024 16:43:13 +0000 Subject: [PATCH 70/70] Pre-merge Cosmetic changes --- src/telemetry/backend/service/__main__.py | 15 ++++++++++----- src/telemetry/frontend/service/__main__.py | 8 ++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/telemetry/backend/service/__main__.py b/src/telemetry/backend/service/__main__.py index 4ad867331..9ec9e191f 100644 --- a/src/telemetry/backend/service/__main__.py +++ b/src/telemetry/backend/service/__main__.py @@ -13,7 +13,8 @@ # limitations under the License. import logging, signal, sys, threading -from common.Settings import get_log_level +from prometheus_client import start_http_server +from common.Settings import get_log_level, get_metrics_port from .TelemetryBackendService import TelemetryBackendService terminate = threading.Event() @@ -27,13 +28,17 @@ def main(): global LOGGER # pylint: disable=global-statement log_level = get_log_level() - logging.basicConfig(level=log_level) + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") LOGGER = logging.getLogger(__name__) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - LOGGER.debug('Starting...') + LOGGER.info('Starting...') + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) grpc_service = TelemetryBackendService() grpc_service.start() @@ -41,10 +46,10 @@ def main(): # Wait for Ctrl+C or termination signal while not terminate.wait(timeout=1.0): pass - LOGGER.debug('Terminating...') + LOGGER.info('Terminating...') grpc_service.stop() - LOGGER.debug('Bye') + LOGGER.info('Bye') return 0 if __name__ == '__main__': diff --git a/src/telemetry/frontend/service/__main__.py b/src/telemetry/frontend/service/__main__.py index 74bc6f500..2a6c5dbcf 100644 --- a/src/telemetry/frontend/service/__main__.py +++ b/src/telemetry/frontend/service/__main__.py @@ -28,13 +28,13 @@ def main(): global LOGGER # pylint: disable=global-statement log_level = get_log_level() - logging.basicConfig(level=log_level) + logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") LOGGER = logging.getLogger(__name__) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - LOGGER.debug('Starting...') + LOGGER.info('Starting...') # Start metrics server metrics_port = get_metrics_port() @@ -46,10 +46,10 @@ def main(): # Wait for Ctrl+C or termination signal while not terminate.wait(timeout=1.0): pass - LOGGER.debug('Terminating...') + LOGGER.info('Terminating...') grpc_service.stop() - LOGGER.debug('Bye') + LOGGER.info('Bye') return 0 if __name__ == '__main__': -- GitLab