From 31b46ce434e261b5518fa02594fd30eb8f25205e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 17:40:07 +0000 Subject: [PATCH 01/14] Scripts: - Corrected show logs automation - Cosmetic change in deploy crdb --- deploy/crdb.sh | 2 ++ scripts/show_logs_automation.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/crdb.sh b/deploy/crdb.sh index 3fcbb5cfa..42b49fe98 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -175,6 +175,7 @@ function crdb_drop_databases_single() { --execute "SHOW DATABASES;" --format=tsv | awk '{print $1}' | grep "^tfs" ) echo "Found TFS databases: ${DATABASES}" | tr '\n' ' ' + echo for DB_NAME in $DATABASES; do echo "Dropping TFS database: $DB_NAME" @@ -369,6 +370,7 @@ function crdb_drop_databases_cluster() { --execute "SHOW DATABASES;" --format=tsv | awk '{print $1}' | grep "^tfs" ) echo "Found TFS databases: ${DATABASES}" | tr '\n' ' ' + echo for DB_NAME in $DATABASES; do echo "Dropping TFS database: $DB_NAME" diff --git a/scripts/show_logs_automation.sh b/scripts/show_logs_automation.sh index 8a0e417d9..266842980 100755 --- a/scripts/show_logs_automation.sh +++ b/scripts/show_logs_automation.sh @@ -24,4 +24,4 @@ export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"} # Automated steps start here ######################################################################################################################## -kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/ztpservice +kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/automationservice -- GitLab From 5e23f50f23c4113a35c84c9b1a824bc91746aff5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 17:40:56 +0000 Subject: [PATCH 02/14] Common/tools/grps: - Added new Event Collection and Dispatching framework --- src/common/tools/grpc/BaseEventCollector.py | 136 +++++++++++++++++++ src/common/tools/grpc/BaseEventDispatcher.py | 119 ++++++++++++++++ src/common/tools/grpc/ExampleEventEngine.py | 101 ++++++++++++++ 3 files changed, 356 insertions(+) create mode 100644 src/common/tools/grpc/BaseEventCollector.py create mode 100644 src/common/tools/grpc/BaseEventDispatcher.py create mode 100644 src/common/tools/grpc/ExampleEventEngine.py diff --git a/src/common/tools/grpc/BaseEventCollector.py b/src/common/tools/grpc/BaseEventCollector.py new file mode 100644 index 000000000..04dfb6549 --- /dev/null +++ b/src/common/tools/grpc/BaseEventCollector.py @@ -0,0 +1,136 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See usage example below + +import grpc, logging, queue, threading, time +from typing import Any, Callable, List, Optional +from common.proto.context_pb2 import Empty +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.client.ContextClient import ContextClient + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +class CollectorThread(threading.Thread): + def __init__( + self, subscription_func : Callable, events_queue = queue.PriorityQueue, + terminate = threading.Event, log_events_received: bool = False + ) -> None: + super().__init__(daemon=False) + self._subscription_func = subscription_func + self._events_queue = events_queue + self._terminate = terminate + self._log_events_received = log_events_received + self._stream = None + + def cancel(self) -> None: + if self._stream is None: return + self._stream.cancel() + + def run(self) -> None: + while not self._terminate.is_set(): + self._stream = self._subscription_func() + try: + for event in self._stream: + if self._log_events_received: + str_event = grpc_message_to_json_string(event) + LOGGER.info('[_collect] event: {:s}'.format(str_event)) + timestamp = event.event.timestamp.timestamp + self._events_queue.put_nowait((timestamp, event)) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.UNAVAILABLE: # pylint: disable=no-member + LOGGER.info('[_collect] UNAVAILABLE... retrying...') + time.sleep(0.5) + continue + elif e.code() == grpc.StatusCode.CANCELLED: # pylint: disable=no-member + break + else: + raise # pragma: no cover + +class BaseEventCollector: + def __init__( + self, terminate : Optional[threading.Event] = None + ) -> None: + self._events_queue = queue.PriorityQueue() + self._terminate = threading.Event() if terminate is None else terminate + self._collector_threads : List[CollectorThread] = list() + + def install_collector( + self, subscription_method : Callable, request_message : Any, + log_events_received : bool = False + ) -> None: + self._collector_threads.append(CollectorThread( + lambda: subscription_method(request_message), + self._events_queue, self._terminate, log_events_received + )) + + def start(self): + self._terminate.clear() + for collector_thread in self._collector_threads: + collector_thread.start() + + def stop(self): + self._terminate.set() + + for collector_thread in self._collector_threads: + collector_thread.cancel() + + for collector_thread in self._collector_threads: + collector_thread.join() + + def get_events_queue(self) -> queue.PriorityQueue: + return self._events_queue + + def get_event(self, block : bool = True, timeout : float = 0.1): + try: + _,event = self._events_queue.get(block=block, timeout=timeout) + return event + except queue.Empty: # pylint: disable=catching-non-exception + return None + + def get_events(self, block : bool = True, timeout : float = 0.1, count : int = None): + events = [] + if count is None: + while not self._terminate.is_set(): + event = self.get_event(block=block, timeout=timeout) + if event is None: break + events.append(event) + else: + while len(events) < count: + if self._terminate.is_set(): break + event = self.get_event(block=block, timeout=timeout) + if event is None: continue + events.append(event) + return sorted(events, key=lambda e: e.event.timestamp.timestamp) + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + context_client = ContextClient() + context_client.connect() + + event_collector = BaseEventCollector() + event_collector.install_collector(context_client.GetDeviceEvents, Empty(), log_events_received=True) + event_collector.install_collector(context_client.GetLinkEvents, Empty(), log_events_received=True) + event_collector.install_collector(context_client.GetServiceEvents, Empty(), log_events_received=True) + event_collector.start() + + time.sleep(60) + + event_collector.stop() + context_client.close() + +if __name__ == '__main__': + main() diff --git a/src/common/tools/grpc/BaseEventDispatcher.py b/src/common/tools/grpc/BaseEventDispatcher.py new file mode 100644 index 000000000..c9ec292c9 --- /dev/null +++ b/src/common/tools/grpc/BaseEventDispatcher.py @@ -0,0 +1,119 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See usage example below + +import logging, queue, threading, time +from typing import Any, Callable, Optional +from common.proto.context_pb2 import DeviceEvent, Empty, EventTypeEnum, LinkEvent +from common.tools.grpc.BaseEventCollector import BaseEventCollector +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.client.ContextClient import ContextClient + +LOGGER = logging.getLogger(__name__) + +class BaseEventDispatcher(threading.Thread): + def __init__( + self, events_queue : queue.PriorityQueue, + terminate : Optional[threading.Event] = None + ) -> None: + super().__init__(daemon=True) + self._events_queue = events_queue + self._terminate = threading.Event() if terminate is None else terminate + + def stop(self): + self._terminate.set() + + def _get_event(self, block : bool = True, timeout : Optional[float] = 0.5) -> Optional[Any]: + try: + _, event = self._events_queue.get(block=block, timeout=timeout) + return event + except queue.Empty: + return None + + def _get_dispatcher(self, event : Any) -> Optional[Callable]: + object_name = str(event.__class__.__name__).lower().replace('event', '') + event_type = EventTypeEnum.Name(event.event.event_type).lower().replace('eventtype_', '') + + method_name = 'dispatch_{:s}_{:s}'.format(object_name, event_type) + dispatcher = getattr(self, method_name, None) + if dispatcher is not None: return dispatcher + + method_name = 'dispatch_{:s}'.format(object_name) + dispatcher = getattr(self, method_name, None) + if dispatcher is not None: return dispatcher + + method_name = 'dispatch' + dispatcher = getattr(self, method_name, None) + if dispatcher is not None: return dispatcher + + return None + + def run(self) -> None: + while not self._terminate.is_set(): + event = self._get_event() + if event is None: continue + + dispatcher = self._get_dispatcher(event) + if dispatcher is None: + MSG = 'No dispatcher available for Event({:s})' + LOGGER.warning(MSG.format(grpc_message_to_json_string(event))) + continue + + dispatcher(event) + +class MyEventDispatcher(BaseEventDispatcher): + def dispatch_device_create(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Create: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_device_update(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Update: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_device_remove(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Remove: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_link(self, link_event : LinkEvent) -> None: + MSG = 'Processing Link Create/Update/Remove: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(link_event))) + + def dispatch(self, event : Any) -> None: + MSG = 'Processing any other Event: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(event))) + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + context_client = ContextClient() + context_client.connect() + + event_collector = BaseEventCollector() + event_collector.install_collector(context_client.GetDeviceEvents, Empty(), log_events_received=True) + event_collector.install_collector(context_client.GetLinkEvents, Empty(), log_events_received=True) + event_collector.install_collector(context_client.GetServiceEvents, Empty(), log_events_received=True) + event_collector.start() + + event_dispatcher = MyEventDispatcher(event_collector.get_events_queue()) + event_dispatcher.start() + + time.sleep(60) + + event_dispatcher.stop() + event_collector.stop() + context_client.close() + +if __name__ == '__main__': + main() diff --git a/src/common/tools/grpc/ExampleEventEngine.py b/src/common/tools/grpc/ExampleEventEngine.py new file mode 100644 index 000000000..f27792497 --- /dev/null +++ b/src/common/tools/grpc/ExampleEventEngine.py @@ -0,0 +1,101 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, threading, time +from typing import Optional +from common.proto.context_pb2 import DeviceEvent, Empty, ServiceEvent +from common.tools.grpc.BaseEventCollector import BaseEventCollector +from common.tools.grpc.BaseEventDispatcher import BaseEventDispatcher +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.client.ContextClient import ContextClient + +LOGGER = logging.getLogger(__name__) + +class EventCollector(BaseEventCollector): + pass + +class EventDispatcher(BaseEventDispatcher): + def dispatch_device_create(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Create: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_device_update(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Update: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_device_remove(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Remove: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_service_create(self, service_event : ServiceEvent) -> None: + MSG = 'Processing Service Create: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(service_event))) + + def dispatch_service_update(self, service_event : ServiceEvent) -> None: + MSG = 'Processing Service Update: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(service_event))) + + def dispatch_service_remove(self, service_event : ServiceEvent) -> None: + MSG = 'Processing Service Remove: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(service_event))) + +class ExampleEventEngine: + def __init__( + self, terminate : Optional[threading.Event] = None + ) -> None: + self._terminate = threading.Event() if terminate is None else terminate + + self._context_client = ContextClient() + self._event_collector = EventCollector(terminate=self._terminate) + self._event_collector.install_collector( + self._context_client.GetDeviceEvents, Empty(), + log_events_received=True + ) + self._event_collector.install_collector( + self._context_client.GetLinkEvents, Empty(), + log_events_received=True + ) + self._event_collector.install_collector( + self._context_client.GetServiceEvents, Empty(), + log_events_received=True + ) + + self._event_dispatcher = EventDispatcher( + self._event_collector.get_events_queue(), + terminate=self._terminate + ) + + def start(self) -> None: + self._context_client.connect() + self._event_collector.start() + self._event_dispatcher.start() + + def stop(self) -> None: + self._terminate.set() + self._event_dispatcher.stop() + self._event_collector.stop() + self._context_client.close() + +def main() -> None: + logging.basicConfig(level=logging.INFO) + + event_engine = ExampleEventEngine() + event_engine.start() + + time.sleep(60) + + event_engine.stop() + +if __name__ == '__main__': + main() -- GitLab From d92148dafae75a88eda98cc0b84d5017da830c8c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 17:42:05 +0000 Subject: [PATCH 03/14] Corrected ServiceName Constants for Analytics and Telemetry --- src/analytics/frontend/client/AnalyticsFrontendClient.py | 4 ++-- .../frontend/service/AnalyticsFrontendService.py | 2 +- src/analytics/frontend/tests/test_frontend.py | 6 +++--- src/common/Constants.py | 8 ++++---- src/telemetry/frontend/client/TelemetryFrontendClient.py | 4 ++-- .../frontend/service/TelemetryFrontendService.py | 2 +- src/telemetry/frontend/tests/test_frontend.py | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/analytics/frontend/client/AnalyticsFrontendClient.py b/src/analytics/frontend/client/AnalyticsFrontendClient.py index 90e95d661..809c957ea 100644 --- a/src/analytics/frontend/client/AnalyticsFrontendClient.py +++ b/src/analytics/frontend/client/AnalyticsFrontendClient.py @@ -28,8 +28,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class AnalyticsFrontendClient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.ANALYTICSFRONTEND) - if not port: port = get_service_port_grpc(ServiceNameEnum.ANALYTICSFRONTEND) + if not host: host = get_service_host(ServiceNameEnum.ANALYTICS) + if not port: port = get_service_port_grpc(ServiceNameEnum.ANALYTICS) self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) self.channel = None diff --git a/src/analytics/frontend/service/AnalyticsFrontendService.py b/src/analytics/frontend/service/AnalyticsFrontendService.py index 42a7fc9b6..8d2536fe0 100644 --- a/src/analytics/frontend/service/AnalyticsFrontendService.py +++ b/src/analytics/frontend/service/AnalyticsFrontendService.py @@ -20,7 +20,7 @@ from analytics.frontend.service.AnalyticsFrontendServiceServicerImpl import Anal class AnalyticsFrontendService(GenericGrpcService): def __init__(self, cls_name: str = __name__): - port = get_service_port_grpc(ServiceNameEnum.ANALYTICSFRONTEND) + port = get_service_port_grpc(ServiceNameEnum.ANALYTICS) super().__init__(port, cls_name=cls_name) self.analytics_frontend_servicer = AnalyticsFrontendServiceServicerImpl() diff --git a/src/analytics/frontend/tests/test_frontend.py b/src/analytics/frontend/tests/test_frontend.py index 48ab4dac5..74fef6c79 100644 --- a/src/analytics/frontend/tests/test_frontend.py +++ b/src/analytics/frontend/tests/test_frontend.py @@ -41,9 +41,9 @@ from apscheduler.triggers.interval import IntervalTrigger LOCAL_HOST = '127.0.0.1' -ANALYTICS_FRONTEND_PORT = str(get_service_port_grpc(ServiceNameEnum.ANALYTICSFRONTEND)) -os.environ[get_env_var_name(ServiceNameEnum.ANALYTICSFRONTEND, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.ANALYTICSFRONTEND, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(ANALYTICS_FRONTEND_PORT) +ANALYTICS_FRONTEND_PORT = str(get_service_port_grpc(ServiceNameEnum.ANALYTICS)) +os.environ[get_env_var_name(ServiceNameEnum.ANALYTICS, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.ANALYTICS, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(ANALYTICS_FRONTEND_PORT) LOGGER = logging.getLogger(__name__) diff --git a/src/common/Constants.py b/src/common/Constants.py index 33a5d5047..bd4b9031f 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -66,9 +66,9 @@ class ServiceNameEnum(Enum): KPIMANAGER = 'kpi-manager' KPIVALUEAPI = 'kpi-value-api' KPIVALUEWRITER = 'kpi-value-writer' - TELEMETRYFRONTEND = 'telemetry-frontend' + TELEMETRY = 'telemetry' TELEMETRYBACKEND = 'telemetry-backend' - ANALYTICSFRONTEND = 'analytics-frontend' + ANALYTICS = 'analytics' ANALYTICSBACKEND = 'analytics-backend' QOSPROFILE = 'qos-profile' @@ -105,9 +105,9 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.KPIMANAGER .value : 30010, ServiceNameEnum.KPIVALUEAPI .value : 30020, ServiceNameEnum.KPIVALUEWRITER .value : 30030, - ServiceNameEnum.TELEMETRYFRONTEND .value : 30050, + ServiceNameEnum.TELEMETRY .value : 30050, ServiceNameEnum.TELEMETRYBACKEND .value : 30060, - ServiceNameEnum.ANALYTICSFRONTEND .value : 30080, + ServiceNameEnum.ANALYTICS .value : 30080, ServiceNameEnum.ANALYTICSBACKEND .value : 30090, ServiceNameEnum.AUTOMATION .value : 30200, diff --git a/src/telemetry/frontend/client/TelemetryFrontendClient.py b/src/telemetry/frontend/client/TelemetryFrontendClient.py index cd36ecd45..afcf24153 100644 --- a/src/telemetry/frontend/client/TelemetryFrontendClient.py +++ b/src/telemetry/frontend/client/TelemetryFrontendClient.py @@ -29,8 +29,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class TelemetryFrontendClient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.TELEMETRYFRONTEND) - if not port: port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) + if not host: host = get_service_host(ServiceNameEnum.TELEMETRY) + if not port: port = get_service_port_grpc(ServiceNameEnum.TELEMETRY) self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) self.channel = None diff --git a/src/telemetry/frontend/service/TelemetryFrontendService.py b/src/telemetry/frontend/service/TelemetryFrontendService.py index abd361aa0..49def20a1 100644 --- a/src/telemetry/frontend/service/TelemetryFrontendService.py +++ b/src/telemetry/frontend/service/TelemetryFrontendService.py @@ -21,7 +21,7 @@ from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import Tele class TelemetryFrontendService(GenericGrpcService): def __init__(self, cls_name: str = __name__) -> None: - port = get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND) + port = get_service_port_grpc(ServiceNameEnum.TELEMETRY) super().__init__(port, cls_name=cls_name) self.telemetry_frontend_servicer = TelemetryFrontendServiceServicerImpl() diff --git a/src/telemetry/frontend/tests/test_frontend.py b/src/telemetry/frontend/tests/test_frontend.py index c3f8091c8..988d76af0 100644 --- a/src/telemetry/frontend/tests/test_frontend.py +++ b/src/telemetry/frontend/tests/test_frontend.py @@ -36,9 +36,9 @@ from telemetry.frontend.service.TelemetryFrontendServiceServicerImpl import Tele LOCAL_HOST = '127.0.0.1' -TELEMETRY_FRONTEND_PORT = str(get_service_port_grpc(ServiceNameEnum.TELEMETRYFRONTEND)) -os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.TELEMETRYFRONTEND, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(TELEMETRY_FRONTEND_PORT) +TELEMETRY_FRONTEND_PORT = str(get_service_port_grpc(ServiceNameEnum.TELEMETRY)) +os.environ[get_env_var_name(ServiceNameEnum.TELEMETRY, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.TELEMETRY, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(TELEMETRY_FRONTEND_PORT) LOGGER = logging.getLogger(__name__) -- GitLab From 58d5f61904db5665098b242604c5a88388fb869d Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 17:43:09 +0000 Subject: [PATCH 04/14] Restructured NBI Dockerfile to prevent rebilding libyang too frequently --- src/nbi/Dockerfile | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/nbi/Dockerfile b/src/nbi/Dockerfile index c5fc8d324..ad111c283 100644 --- a/src/nbi/Dockerfile +++ b/src/nbi/Dockerfile @@ -16,9 +16,24 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ git && \ + apt-get --yes --quiet --quiet install wget g++ git build-essential cmake libpcre2-dev python3-dev python3-cffi && \ rm -rf /var/lib/apt/lists/* +# Download, build and install libyang. Note that APT package is outdated +# - Ref: https://github.com/CESNET/libyang +# - Ref: https://github.com/CESNET/libyang-python/ +RUN mkdir -p /var/libyang +RUN git clone https://github.com/CESNET/libyang.git /var/libyang +WORKDIR /var/libyang +RUN git fetch +RUN git checkout v2.1.148 +RUN mkdir -p /var/libyang/build +WORKDIR /var/libyang/build +RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. +RUN make +RUN make install +RUN ldconfig + # Set Python to show logs as they occur ENV PYTHONUNBUFFERED=0 @@ -53,24 +68,6 @@ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -# Download, build and install libyang. Note that APT package is outdated -# - Ref: https://github.com/CESNET/libyang -# - Ref: https://github.com/CESNET/libyang-python/ -RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install build-essential cmake libpcre2-dev python3-dev python3-cffi && \ - rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/libyang -RUN git clone https://github.com/CESNET/libyang.git /var/libyang -WORKDIR /var/libyang -RUN git fetch -RUN git checkout v2.1.148 -RUN mkdir -p /var/libyang/build -WORKDIR /var/libyang/build -RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. -RUN make -RUN make install -RUN ldconfig - # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/nbi WORKDIR /var/teraflow/nbi -- GitLab From 7196288b9e856c9d7adee01ac85df61a50efffd8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 17:43:53 +0000 Subject: [PATCH 05/14] Automation: - Added new EventEngine to activate automatically monitoring of device endpoints --- src/automation/service/EventEngine.py | 169 ++++++++++++++++++++++++++ src/automation/service/Tools.py | 49 ++++++++ src/automation/service/__main__.py | 31 ++++- 3 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 src/automation/service/EventEngine.py create mode 100644 src/automation/service/Tools.py diff --git a/src/automation/service/EventEngine.py b/src/automation/service/EventEngine.py new file mode 100644 index 000000000..26c2b28cb --- /dev/null +++ b/src/automation/service/EventEngine.py @@ -0,0 +1,169 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, logging, queue, threading +from typing import Dict, Optional +from automation.service.Tools import create_kpi_descriptor, start_collector +from common.proto.context_pb2 import ( + ConfigActionEnum, DeviceEvent, DeviceOperationalStatusEnum, Empty, ServiceEvent +) +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.tools.grpc.BaseEventCollector import BaseEventCollector +from common.tools.grpc.BaseEventDispatcher import BaseEventDispatcher +from common.tools.grpc.Tools import grpc_message_to_json_string +from context.client.ContextClient import ContextClient +from kpi_manager.client.KpiManagerClient import KpiManagerClient +from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendClient + +LOGGER = logging.getLogger(__name__) + +DEVICE_OP_STATUS_UNDEFINED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_UNDEFINED +DEVICE_OP_STATUS_DISABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_DISABLED +DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED +DEVICE_OP_STATUS_NOT_ENABLED = {DEVICE_OP_STATUS_UNDEFINED, DEVICE_OP_STATUS_DISABLED} + +KPISAMPLETYPE_UNKNOWN = KpiSampleType.KPISAMPLETYPE_UNKNOWN + +class EventCollector(BaseEventCollector): + pass + +class EventDispatcher(BaseEventDispatcher): + def __init__( + self, events_queue : queue.PriorityQueue, + terminate : Optional[threading.Event] = None + ) -> None: + super().__init__(events_queue, terminate) + self._context_client = ContextClient() + self._kpi_manager_client = KpiManagerClient() + self._telemetry_client = TelemetryFrontendClient() + self._device_endpoint_monitored : Dict[str, Dict[str, bool]] = dict() + + def dispatch_device_create(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Create: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + self._device_activate_monitoring(device_event) + + def dispatch_device_update(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Update: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + self._device_activate_monitoring(device_event) + + def dispatch_device_remove(self, device_event : DeviceEvent) -> None: + MSG = 'Processing Device Remove: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(device_event))) + + def dispatch_service_create(self, service_event : ServiceEvent) -> None: + MSG = 'Processing Service Create: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(service_event))) + + def dispatch_service_update(self, service_event : ServiceEvent) -> None: + MSG = 'Processing Service Update: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(service_event))) + + def dispatch_service_remove(self, service_event : ServiceEvent) -> None: + MSG = 'Processing Service Remove: {:s}' + LOGGER.info(MSG.format(grpc_message_to_json_string(service_event))) + + def _device_activate_monitoring(self, device_event : DeviceEvent) -> None: + device_id = device_event.device_id + device_uuid = device_id.device_uuid.uuid + device = self._context_client.GetDevice(device_id) + + device_op_status = device.device_operational_status + if device_op_status != DEVICE_OP_STATUS_ENABLED: + LOGGER.debug('Ignoring Device not enabled: {:s}'.format(grpc_message_to_json_string(device))) + return + + enabled_endpoint_names = set() + for config_rule in device.device_config.config_rules: + if config_rule.action != ConfigActionEnum.CONFIGACTION_SET: continue + if config_rule.WhichOneof('config_rule') != 'custom': continue + str_resource_key = str(config_rule.custom.resource_key) + if not str_resource_key.startswith('/interface['): continue + json_resource_value = json.loads(config_rule.custom.resource_value) + if 'name' not in json_resource_value: continue + if 'enabled' not in json_resource_value: continue + if not json_resource_value['enabled']: continue + enabled_endpoint_names.add(json_resource_value['name']) + + endpoints_monitored = self._device_endpoint_monitored.setdefault(device_uuid, dict()) + for endpoint in device.device_endpoints: + endpoint_uuid = endpoint.endpoint_id.endpoint_uuid.uuid + endpoint_name_or_uuid = endpoint.name + if endpoint_name_or_uuid is None or len(endpoint_name_or_uuid) == 0: + endpoint_name_or_uuid = endpoint_uuid + + endpoint_was_monitored = endpoints_monitored.get(endpoint_uuid, False) + endpoint_is_enabled = (endpoint_name_or_uuid in enabled_endpoint_names) + + if not endpoint_was_monitored and endpoint_is_enabled: + # activate + for kpi_sample_type in endpoint.kpi_sample_types: + if kpi_sample_type == KPISAMPLETYPE_UNKNOWN: continue + + kpi_id = create_kpi_descriptor( + self._kpi_manager_client, kpi_sample_type, + device_id=device.device_id, + endpoint_id=endpoint.endpoint_id, + ) + + duration_seconds = 86400 + interval_seconds = 10 + collector_id = start_collector( + self._telemetry_client, kpi_id, + duration_seconds, interval_seconds + ) + + endpoints_monitored[endpoint_uuid] = True + else: + MSG = 'Not implemented condition: event={:s} device={:s} endpoint={:s}' + \ + ' endpoint_was_monitored={:s} endpoint_is_enabled={:s}' + LOGGER.warning(MSG.format( + grpc_message_to_json_string(device_event), grpc_message_to_json_string(device), + grpc_message_to_json_string(endpoint), str(endpoint_was_monitored), + str(endpoint_is_enabled) + )) + +class EventEngine: + def __init__( + self, terminate : Optional[threading.Event] = None + ) -> None: + self._terminate = threading.Event() if terminate is None else terminate + + self._context_client = ContextClient() + self._event_collector = EventCollector(terminate=self._terminate) + self._event_collector.install_collector( + self._context_client.GetDeviceEvents, Empty(), + log_events_received=True + ) + self._event_collector.install_collector( + self._context_client.GetServiceEvents, Empty(), + log_events_received=True + ) + + self._event_dispatcher = EventDispatcher( + self._event_collector.get_events_queue(), + terminate=self._terminate + ) + + def start(self) -> None: + self._context_client.connect() + self._event_collector.start() + self._event_dispatcher.start() + + def stop(self) -> None: + self._terminate.set() + self._event_dispatcher.stop() + self._event_collector.stop() + self._context_client.close() diff --git a/src/automation/service/Tools.py b/src/automation/service/Tools.py new file mode 100644 index 000000000..4f86dfafe --- /dev/null +++ b/src/automation/service/Tools.py @@ -0,0 +1,49 @@ +import logging, uuid +from typing import Optional +from common.proto.context_pb2 import ConnectionId, DeviceId, EndPointId, LinkId, ServiceId, SliceId +from common.proto.kpi_manager_pb2 import KpiDescriptor, KpiId +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.proto.telemetry_frontend_pb2 import Collector, CollectorId +from kpi_manager.client.KpiManagerClient import KpiManagerClient +from telemetry.frontend.client.TelemetryFrontendClient import TelemetryFrontendClient + +LOGGER = logging.getLogger(__name__) + +def create_kpi_descriptor( + kpi_manager_client : KpiManagerClient, + kpi_sample_type : KpiSampleType, + device_id : Optional[DeviceId ] = None, + endpoint_id : Optional[EndPointId ] = None, + service_id : Optional[ServiceId ] = None, + slice_id : Optional[SliceId ] = None, + connection_id : Optional[ConnectionId] = None, + link_id : Optional[LinkId ] = None, +) -> KpiId: + kpi_descriptor = KpiDescriptor() + kpi_descriptor.kpi_id.kpi_id.uuid = str(uuid.uuid4()) + kpi_descriptor.kpi_description = '' + kpi_descriptor.kpi_sample_type = kpi_sample_type + + if device_id is not None: kpi_descriptor.device_id .CopyFrom(device_id ) + if endpoint_id is not None: kpi_descriptor.endpoint_id .CopyFrom(endpoint_id ) + if service_id is not None: kpi_descriptor.service_id .CopyFrom(service_id ) + if slice_id is not None: kpi_descriptor.slice_id .CopyFrom(slice_id ) + if connection_id is not None: kpi_descriptor.connection_id.CopyFrom(connection_id) + if link_id is not None: kpi_descriptor.link_id .CopyFrom(link_id ) + + kpi_id : KpiId = kpi_manager_client.SetKpiDescriptor(kpi_descriptor) + return kpi_id + +def start_collector( + telemetry_client : TelemetryFrontendClient, + kpi_id : KpiId, + duration_seconds : float, + interval_seconds : float +) -> CollectorId: + collector = Collector() + collector.collector_id.collector_id.uuid = str(uuid.uuid4()) + collector.kpi_id.CopyFrom(kpi_id) + collector.duration_s = duration_seconds + collector.interval_s = interval_seconds + collector_id : CollectorId = telemetry_client.StartCollector(collector) + return collector_id diff --git a/src/automation/service/__main__.py b/src/automation/service/__main__.py index 39d8beaff..3baa0bd30 100644 --- a/src/automation/service/__main__.py +++ b/src/automation/service/__main__.py @@ -14,7 +14,13 @@ import logging, signal, sys, threading from prometheus_client import start_http_server -from common.Settings import get_log_level, get_metrics_port +from automation.service.EventEngine import EventEngine +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, + get_env_var_name, get_log_level, get_metrics_port, + wait_for_environment_variables +) from .AutomationService import AutomationService LOG_LEVEL = get_log_level() @@ -29,6 +35,22 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name,unused def main(): LOGGER.info('Starting...') + + wait_for_environment_variables([ + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.KPIMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.TELEMETRY, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.TELEMETRY, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.ANALYTICS, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.ANALYTICS, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + get_env_var_name(ServiceNameEnum.POLICY, ENVVAR_SUFIX_SERVICE_HOST ), + get_env_var_name(ServiceNameEnum.POLICY, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ]) + signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) @@ -36,7 +58,11 @@ def main(): metrics_port = get_metrics_port() start_http_server(metrics_port) - # Starting context service + # Start Event Collection+Dispatching Engine + event_engine = EventEngine(terminate=terminate) + event_engine.start() + + # Starting Automation service grpc_service = AutomationService() grpc_service.start() @@ -45,6 +71,7 @@ def main(): LOGGER.info('Terminating...') grpc_service.stop() + event_engine.stop() LOGGER.info('Bye') return 0 -- GitLab From 244c8a430291cca9bfbed77498b95079a156eba0 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 17:45:46 +0000 Subject: [PATCH 06/14] Corrected analytics and telemetry manifests --- manifests/analyticsservice.yaml | 4 ++-- manifests/telemetryservice.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/analyticsservice.yaml b/manifests/analyticsservice.yaml index 61666ead9..536bb1852 100644 --- a/manifests/analyticsservice.yaml +++ b/manifests/analyticsservice.yaml @@ -98,11 +98,11 @@ spec: selector: app: analyticsservice ports: - - name: frontend-grpc + - name: grpc protocol: TCP port: 30080 targetPort: 30080 - - name: backend-grpc + - name: grpc-backend protocol: TCP port: 30090 targetPort: 30090 diff --git a/manifests/telemetryservice.yaml b/manifests/telemetryservice.yaml index cd35d2698..86d864157 100644 --- a/manifests/telemetryservice.yaml +++ b/manifests/telemetryservice.yaml @@ -98,11 +98,11 @@ spec: selector: app: telemetryservice ports: - - name: frontend-grpc + - name: grpc protocol: TCP port: 30050 targetPort: 30050 - - name: backend-grpc + - name: grpc-backend protocol: TCP port: 30060 targetPort: 30060 -- GitLab From 4959720a650ba0f45b31f9f593c2bdf7980184f4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 8 Oct 2024 18:00:08 +0000 Subject: [PATCH 07/14] Device component: - Improved procedure to update operational status --- src/device/service/DeviceServiceServicerImpl.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index ebbf19607..0596aa76b 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -251,8 +251,14 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): device_id = context_client.SetDevice(device) device = context_client.GetDevice(device_id) - if request.device_operational_status != DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_UNDEFINED: - device.device_operational_status = request.device_operational_status + ztp_service_host = get_env_var_name(ServiceNameEnum.ZTP, ENVVAR_SUFIX_SERVICE_HOST) + environment_variables = set(os.environ.keys()) + if ztp_service_host in environment_variables: + # ZTP component is deployed; accept status updates + if request.device_operational_status != DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_UNDEFINED: + device.device_operational_status = request.device_operational_status + else: + # ZTP is not deployed; activated during AddDevice and not modified t4 = time.time() # TODO: use of datastores (might be virtual ones) to enable rollbacks -- GitLab From 1cec3c552eb3604770c72e0b6f3a8c5ffa329974 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 10:19:28 +0000 Subject: [PATCH 08/14] Device component: - Minor code fix --- src/device/service/DeviceServiceServicerImpl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index 0596aa76b..7546c225e 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -259,6 +259,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): device.device_operational_status = request.device_operational_status else: # ZTP is not deployed; activated during AddDevice and not modified + pass t4 = time.time() # TODO: use of datastores (might be virtual ones) to enable rollbacks -- GitLab From 886295d0fb81ac5214e33cf05623ec212e292367 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 10:39:35 +0000 Subject: [PATCH 09/14] Automation component: - Added missing header --- src/automation/service/Tools.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/automation/service/Tools.py b/src/automation/service/Tools.py index 4f86dfafe..6a63475ca 100644 --- a/src/automation/service/Tools.py +++ b/src/automation/service/Tools.py @@ -1,3 +1,18 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import logging, uuid from typing import Optional from common.proto.context_pb2 import ConnectionId, DeviceId, EndPointId, LinkId, ServiceId, SliceId -- GitLab From 0dcb42b472c70550e0c5bccdaec776dfbd0e20b4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 10:53:03 +0000 Subject: [PATCH 10/14] Deploy scripts: - Added test my_deploy.sh - Updated tfs.sh version image inference --- deploy/tfs.sh | 14 +++++++------- my_deploy.sh | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/deploy/tfs.sh b/deploy/tfs.sh index 65c1e8de2..df7b00e2e 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -344,13 +344,13 @@ for COMPONENT in $TFS_COMPONENTS; do VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f4) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" else - if [ "$TFS_SKIP_BUILD" != "YES" ]; then - IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') - VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) - else - VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) - IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$VERSION" | sed 's,//,/,g' | sed 's,http:/,,g') - fi + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + #if [ "$TFS_SKIP_BUILD" != "YES" ]; then + # IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + #else + # IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$VERSION" | sed 's,//,/,g' | sed 's,http:/,,g') + #fi + VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" fi diff --git a/my_deploy.sh b/my_deploy.sh index 344ca44ee..6692803f5 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,13 +20,13 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service nbi webui" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate Monitoring Framework (new) -#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" +export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" # Uncomment to activate QoS Profiles #export TFS_COMPONENTS="${TFS_COMPONENTS} qos_profile" @@ -48,7 +48,7 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene #export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" # Uncomment to activate Policy Manager -#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" +export TFS_COMPONENTS="${TFS_COMPONENTS} policy" # Uncomment to activate Optical CyberSecurity #export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" @@ -103,7 +103,7 @@ export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" export TFS_GRAFANA_PASSWORD="admin123+" # Disable skip-build flag to rebuild the Docker images. -export TFS_SKIP_BUILD="" +export TFS_SKIP_BUILD="YES" # ----- CockroachDB ------------------------------------------------------------ @@ -131,7 +131,7 @@ export CRDB_DATABASE="tfs" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="" +export CRDB_DROP_DATABASE_IF_EXISTS="YES" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" @@ -183,7 +183,7 @@ export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" # Disable flag for dropping tables if they exist. -export QDB_DROP_TABLES_IF_EXIST="" +export QDB_DROP_TABLES_IF_EXIST="YES" # Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" -- GitLab From cf71aede130bcc98d95be757cd396ba7a9ba33bc Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 10:53:29 +0000 Subject: [PATCH 11/14] Manifests: - Activated DEBUG in relevant components - Reduced initial resource requests --- manifests/analyticsservice.yaml | 8 ++++---- manifests/automationservice.yaml | 4 ++-- manifests/contextservice.yaml | 4 ++-- manifests/deviceservice.yaml | 4 ++-- manifests/kpi_managerservice.yaml | 4 ++-- manifests/kpi_value_apiservice.yaml | 2 +- manifests/kpi_value_writerservice.yaml | 2 +- manifests/pathcompservice.yaml | 2 +- manifests/serviceservice.yaml | 2 +- manifests/telemetryservice.yaml | 8 ++++---- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/manifests/analyticsservice.yaml b/manifests/analyticsservice.yaml index 536bb1852..1c1b9f034 100644 --- a/manifests/analyticsservice.yaml +++ b/manifests/analyticsservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: CRDB_DATABASE value: "tfs_analytics" - name: METRICS_PORT @@ -54,7 +54,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30080"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m @@ -67,7 +67,7 @@ spec: - containerPort: 9193 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: METRICS_PORT value: "9193" envFrom: @@ -81,7 +81,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30090"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/automationservice.yaml b/manifests/automationservice.yaml index f6c97f7fb..d2113cecf 100644 --- a/manifests/automationservice.yaml +++ b/manifests/automationservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:30200"] @@ -53,7 +53,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30200"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index 0fc8a1c44..e687687c0 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -40,7 +40,7 @@ spec: - name: MB_BACKEND value: "nats" - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY value: "FALSE" - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY @@ -60,7 +60,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:1010"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index e49ba2399..f12faed11 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:2020"] @@ -53,7 +53,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:2020"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/kpi_managerservice.yaml b/manifests/kpi_managerservice.yaml index f8ee8ff82..69046af89 100644 --- a/manifests/kpi_managerservice.yaml +++ b/manifests/kpi_managerservice.yaml @@ -38,7 +38,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: CRDB_DATABASE value: "tfs_kpi_mgmt" envFrom: @@ -52,7 +52,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30010"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/kpi_value_apiservice.yaml b/manifests/kpi_value_apiservice.yaml index e4dcb0054..def3879b6 100644 --- a/manifests/kpi_value_apiservice.yaml +++ b/manifests/kpi_value_apiservice.yaml @@ -50,7 +50,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30020"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/kpi_value_writerservice.yaml b/manifests/kpi_value_writerservice.yaml index e21e36f48..c2b3d9507 100644 --- a/manifests/kpi_value_writerservice.yaml +++ b/manifests/kpi_value_writerservice.yaml @@ -50,7 +50,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30030"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/pathcompservice.yaml b/manifests/pathcompservice.yaml index 1f54c569c..690c8232c 100644 --- a/manifests/pathcompservice.yaml +++ b/manifests/pathcompservice.yaml @@ -47,7 +47,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:10020"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 1dd383d61..bcd69f140 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -45,7 +45,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:3030"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/telemetryservice.yaml b/manifests/telemetryservice.yaml index 86d864157..44ee500c2 100644 --- a/manifests/telemetryservice.yaml +++ b/manifests/telemetryservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: CRDB_DATABASE value: "tfs_telemetry" - name: METRICS_PORT @@ -54,7 +54,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30050"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m @@ -67,7 +67,7 @@ spec: - containerPort: 9193 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: METRICS_PORT value: "9193" envFrom: @@ -81,7 +81,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30060"] resources: requests: - cpu: 250m + cpu: 100m memory: 128Mi limits: cpu: 1000m -- GitLab From 80078b217f3d480dab6d23afcecfe4305bf9da02 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 11:05:43 +0000 Subject: [PATCH 12/14] Telemetry - Backend: - Fixed few code issues --- .../backend/service/TelemetryBackendService.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/telemetry/backend/service/TelemetryBackendService.py b/src/telemetry/backend/service/TelemetryBackendService.py index 79a35d343..81ef24481 100755 --- a/src/telemetry/backend/service/TelemetryBackendService.py +++ b/src/telemetry/backend/service/TelemetryBackendService.py @@ -106,7 +106,7 @@ class TelemetryBackendService(GenericGrpcService): Method receives collector request and initiates collecter backend. """ # print("Initiating backend for collector: ", collector_id) - LOGGER.info("Initiating backend for collector: ", collector_id) + LOGGER.info("Initiating backend for collector: {:s}".format(str(collector_id))) start_time = time.time() while not stop_event.is_set(): if int(collector['duration']) != -1 and time.time() - start_time >= collector['duration']: # condition to terminate backend @@ -165,9 +165,9 @@ class TelemetryBackendService(GenericGrpcService): Args: err (KafkaError): Kafka error object. msg (Message): Kafka message object. """ - if err: - LOGGER.debug('Message delivery failed: {:}'.format(err)) + if err: + LOGGER.error('Message delivery failed: {:}'.format(err)) # print(f'Message delivery failed: {err}') - else: - LOGGER.info('Message delivered to topic {:}'.format(msg.topic())) - # print(f'Message delivered to topic {msg.topic()}') + #else: + # LOGGER.debug('Message delivered to topic {:}'.format(msg.topic())) + # # print(f'Message delivered to topic {msg.topic()}') -- GitLab From 1b5d9536e2db9ddf3c1d4a06b4517a8e3b3876b4 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 16:09:55 +0000 Subject: [PATCH 13/14] Recover default settings and specs --- manifests/analyticsservice.yaml | 8 ++++---- manifests/automationservice.yaml | 4 ++-- manifests/contextservice.yaml | 4 ++-- manifests/deviceservice.yaml | 4 ++-- manifests/kpi_managerservice.yaml | 4 ++-- manifests/kpi_value_apiservice.yaml | 2 +- manifests/kpi_value_writerservice.yaml | 2 +- manifests/pathcompservice.yaml | 2 +- manifests/serviceservice.yaml | 2 +- manifests/telemetryservice.yaml | 8 ++++---- my_deploy.sh | 15 +++++++++------ 11 files changed, 29 insertions(+), 26 deletions(-) diff --git a/manifests/analyticsservice.yaml b/manifests/analyticsservice.yaml index 1c1b9f034..536bb1852 100644 --- a/manifests/analyticsservice.yaml +++ b/manifests/analyticsservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: CRDB_DATABASE value: "tfs_analytics" - name: METRICS_PORT @@ -54,7 +54,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30080"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m @@ -67,7 +67,7 @@ spec: - containerPort: 9193 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: METRICS_PORT value: "9193" envFrom: @@ -81,7 +81,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30090"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/automationservice.yaml b/manifests/automationservice.yaml index d2113cecf..f6c97f7fb 100644 --- a/manifests/automationservice.yaml +++ b/manifests/automationservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:30200"] @@ -53,7 +53,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30200"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index e687687c0..0fc8a1c44 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -40,7 +40,7 @@ spec: - name: MB_BACKEND value: "nats" - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: ALLOW_EXPLICIT_ADD_DEVICE_TO_TOPOLOGY value: "FALSE" - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY @@ -60,7 +60,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:1010"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index f12faed11..e49ba2399 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -39,7 +39,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" startupProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:2020"] @@ -53,7 +53,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:2020"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/kpi_managerservice.yaml b/manifests/kpi_managerservice.yaml index 69046af89..f8ee8ff82 100644 --- a/manifests/kpi_managerservice.yaml +++ b/manifests/kpi_managerservice.yaml @@ -38,7 +38,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: CRDB_DATABASE value: "tfs_kpi_mgmt" envFrom: @@ -52,7 +52,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30010"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/kpi_value_apiservice.yaml b/manifests/kpi_value_apiservice.yaml index def3879b6..e4dcb0054 100644 --- a/manifests/kpi_value_apiservice.yaml +++ b/manifests/kpi_value_apiservice.yaml @@ -50,7 +50,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30020"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/kpi_value_writerservice.yaml b/manifests/kpi_value_writerservice.yaml index c2b3d9507..e21e36f48 100644 --- a/manifests/kpi_value_writerservice.yaml +++ b/manifests/kpi_value_writerservice.yaml @@ -50,7 +50,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30030"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/pathcompservice.yaml b/manifests/pathcompservice.yaml index 690c8232c..1f54c569c 100644 --- a/manifests/pathcompservice.yaml +++ b/manifests/pathcompservice.yaml @@ -47,7 +47,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:10020"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index bcd69f140..1dd383d61 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -45,7 +45,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:3030"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/manifests/telemetryservice.yaml b/manifests/telemetryservice.yaml index 44ee500c2..86d864157 100644 --- a/manifests/telemetryservice.yaml +++ b/manifests/telemetryservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: CRDB_DATABASE value: "tfs_telemetry" - name: METRICS_PORT @@ -54,7 +54,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30050"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m @@ -67,7 +67,7 @@ spec: - containerPort: 9193 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: METRICS_PORT value: "9193" envFrom: @@ -81,7 +81,7 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:30060"] resources: requests: - cpu: 100m + cpu: 250m memory: 128Mi limits: cpu: 1000m diff --git a/my_deploy.sh b/my_deploy.sh index 6692803f5..555173cf5 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,13 +20,13 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service nbi webui" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" # Uncomment to activate Monitoring (old) #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" # Uncomment to activate Monitoring Framework (new) -export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" +#export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_api telemetry analytics automation" # Uncomment to activate QoS Profiles #export TFS_COMPONENTS="${TFS_COMPONENTS} qos_profile" @@ -48,7 +48,7 @@ export TFS_COMPONENTS="${TFS_COMPONENTS} kpi_manager kpi_value_writer kpi_value_ #export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" # Uncomment to activate Policy Manager -export TFS_COMPONENTS="${TFS_COMPONENTS} policy" +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" # Uncomment to activate Optical CyberSecurity #export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" @@ -83,6 +83,9 @@ export TFS_COMPONENTS="${TFS_COMPONENTS} policy" # export TFS_COMPONENTS="${BEFORE} qkd_app service ${AFTER}" #fi +# Uncomment to activate Load Generator +#export TFS_COMPONENTS="${TFS_COMPONENTS} load_generator" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" @@ -103,7 +106,7 @@ export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" export TFS_GRAFANA_PASSWORD="admin123+" # Disable skip-build flag to rebuild the Docker images. -export TFS_SKIP_BUILD="YES" +export TFS_SKIP_BUILD="" # ----- CockroachDB ------------------------------------------------------------ @@ -131,7 +134,7 @@ export CRDB_DATABASE="tfs" export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="YES" +export CRDB_DROP_DATABASE_IF_EXISTS="" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" @@ -183,7 +186,7 @@ export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" # Disable flag for dropping tables if they exist. -export QDB_DROP_TABLES_IF_EXIST="YES" +export QDB_DROP_TABLES_IF_EXIST="" # Disable flag for re-deploying QuestDB from scratch. export QDB_REDEPLOY="" -- GitLab From 69d1a5ee6be063f2c424ffd5e2340a5c7b80312c Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 9 Oct 2024 16:17:05 +0000 Subject: [PATCH 14/14] Pre-merge code cleanup --- deploy/tfs.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/deploy/tfs.sh b/deploy/tfs.sh index df7b00e2e..6c0ddcb63 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -344,13 +344,12 @@ for COMPONENT in $TFS_COMPONENTS; do VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f4) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" else - IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') - #if [ "$TFS_SKIP_BUILD" != "YES" ]; then - # IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') - #else - # IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$VERSION" | sed 's,//,/,g' | sed 's,http:/,,g') - #fi VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4) + if [ "$TFS_SKIP_BUILD" != "YES" ]; then + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') + else + IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$VERSION" | sed 's,//,/,g' | sed 's,http:/,,g') + fi sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" fi -- GitLab