From f62d28c746bf3cde9511f2f2d6e358820ec3c72e Mon Sep 17 00:00:00 2001
From: gifrerenom <lluis.gifre@cttc.es>
Date: Wed, 18 Sep 2024 16:26:37 +0000
Subject: [PATCH] Pre-merge code cleanup
---
deploy/tfs.sh | 114 ++++++++----------
manifests/dltservice.yaml | 12 ++
.../connector/client/DltConnectorClient.py | 1 -
.../client/DltConnectorClientAsync.py | 1 -
.../connector/client/DltEventsCollector.py | 1 -
src/dlt/connector/client/DltGatewayClient.py | 1 -
.../connector/client/DltGatewayClientAsync.py | 1 -
.../DltConnectorServiceServicerImpl.py | 1 -
.../topology_abstractor/DltRecordSender.py | 1 -
.../topology_abstractor/DltRecorder.py | 1 -
10 files changed, 65 insertions(+), 69 deletions(-)
diff --git a/deploy/tfs.sh b/deploy/tfs.sh
index 293a56f0f..32605143f 100755
--- a/deploy/tfs.sh
+++ b/deploy/tfs.sh
@@ -45,10 +45,6 @@ export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}
# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.
export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""}
-# If not already set, disable build-if-exists flag to skip building Docker images if they already exist.
-# If TFS_BUILD_IF_EXISTS is "NO", the containers are not rebuilt if they already exist.
-export TFS_BUILD_IF_EXISTS=${TFS_BUILD_IF_EXISTS:-"YES"}
-
# ----- CockroachDB ------------------------------------------------------------
@@ -118,6 +114,7 @@ export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"}
# If not already set, set the external port Grafana HTTP Dashboards will be exposed to.
export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"}
+
########################################################################################################################
# Automated steps start here
########################################################################################################################
@@ -247,77 +244,72 @@ for COMPONENT in $TFS_COMPONENTS; do
echo "Processing '$COMPONENT' component..."
if [ "$TFS_SKIP_BUILD" != "YES" ]; then
- IMAGE_EXISTS=$(docker images -q "$COMPONENT:$TFS_IMAGE_TAG")
- if [ -z "$IMAGE_EXISTS" ] || [ "$TFS_BUILD_IF_EXISTS" == "YES" ]; then
- echo " Building Docker image..."
- BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
-
- if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then
- $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
- elif [ "$COMPONENT" == "pathcomp" ]; then
- BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
- $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"
-
- BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"
- $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"
- # next command is redundant, but helpful to keep cache updated between rebuilds
- IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"
- $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
- elif [ "$COMPONENT" == "dlt" ]; then
- BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"
- $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
-
- BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"
- $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG"
- else
- $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"
- fi
+ echo " Building Docker image..."
+ BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
+
+ if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then
+ $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
+ elif [ "$COMPONENT" == "pathcomp" ]; then
+ BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
+ $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"
+
+ BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"
+ $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"
+ # next command is redundant, but helpful to keep cache updated between rebuilds
+ IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"
+ $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
+ elif [ "$COMPONENT" == "dlt" ]; then
+ BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"
+ $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
+
+ BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"
+ $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG"
+ else
+ $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"
+ fi
- echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."
+ echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."
- if [ "$COMPONENT" == "pathcomp" ]; then
- IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+ if [ "$COMPONENT" == "pathcomp" ]; then
+ IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
- TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
- docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+ TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
+ docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
- PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"
- docker push "$IMAGE_URL" > "$PUSH_LOG"
+ PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"
+ docker push "$IMAGE_URL" > "$PUSH_LOG"
- IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+ IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
- TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"
- docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+ TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"
+ docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
- PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"
- docker push "$IMAGE_URL" > "$PUSH_LOG"
- elif [ "$COMPONENT" == "dlt" ]; then
- IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+ PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"
+ docker push "$IMAGE_URL" > "$PUSH_LOG"
+ elif [ "$COMPONENT" == "dlt" ]; then
+ IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
- TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"
- docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+ TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"
+ docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
- PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"
- docker push "$IMAGE_URL" > "$PUSH_LOG"
+ PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"
+ docker push "$IMAGE_URL" > "$PUSH_LOG"
- IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+ IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
- TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"
- docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+ TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"
+ docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
- PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"
- docker push "$IMAGE_URL" > "$PUSH_LOG"
- else
- IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+ PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"
+ docker push "$IMAGE_URL" > "$PUSH_LOG"
+ else
+ IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
- TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
- docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+ TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
+ docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
- PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
- docker push "$IMAGE_URL" > "$PUSH_LOG"
- fi
- else
- echo " Skipping Docker build for '$COMPONENT' as the image already exists and TFS_BUILD_IF_EXISTS is set to 'NO'."
+ PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
+ docker push "$IMAGE_URL" > "$PUSH_LOG"
fi
fi
diff --git a/manifests/dltservice.yaml b/manifests/dltservice.yaml
index 35ab9919d..c79158801 100644
--- a/manifests/dltservice.yaml
+++ b/manifests/dltservice.yaml
@@ -74,6 +74,18 @@ spec:
imagePullPolicy: Always
ports:
- containerPort: 50051
+ #readinessProbe:
+ # httpGet:
+ # path: /health
+ # port: 8081
+ # initialDelaySeconds: 5
+ # timeoutSeconds: 5
+ #livenessProbe:
+ # httpGet:
+ # path: /health
+ # port: 8081
+ # initialDelaySeconds: 5
+ # timeoutSeconds: 5
resources:
requests:
cpu: 200m
diff --git a/src/dlt/connector/client/DltConnectorClient.py b/src/dlt/connector/client/DltConnectorClient.py
index e383217d8..c3101d65e 100644
--- a/src/dlt/connector/client/DltConnectorClient.py
+++ b/src/dlt/connector/client/DltConnectorClient.py
@@ -22,7 +22,6 @@ from common.tools.client.RetryDecorator import retry, delay_exponential
from common.tools.grpc.Tools import grpc_message_to_json_string
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
MAX_RETRIES = 15
DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')
diff --git a/src/dlt/connector/client/DltConnectorClientAsync.py b/src/dlt/connector/client/DltConnectorClientAsync.py
index 23cd63fa0..b9ed8a4d6 100644
--- a/src/dlt/connector/client/DltConnectorClientAsync.py
+++ b/src/dlt/connector/client/DltConnectorClientAsync.py
@@ -23,7 +23,6 @@ from common.tools.client.RetryDecorator import retry, delay_exponential
from common.tools.grpc.Tools import grpc_message_to_json_string
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
MAX_RETRIES = 15
DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')
diff --git a/src/dlt/connector/client/DltEventsCollector.py b/src/dlt/connector/client/DltEventsCollector.py
index e59784a4d..ce7d01480 100644
--- a/src/dlt/connector/client/DltEventsCollector.py
+++ b/src/dlt/connector/client/DltEventsCollector.py
@@ -19,7 +19,6 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
from dlt.connector.client.DltGatewayClient import DltGatewayClient
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
# This class accepts an event_handler method as attribute that can be used to pre-process and
# filter events before they reach the events_queue. Depending on the handler, the supported
diff --git a/src/dlt/connector/client/DltGatewayClient.py b/src/dlt/connector/client/DltGatewayClient.py
index 71f336866..5e0328380 100644
--- a/src/dlt/connector/client/DltGatewayClient.py
+++ b/src/dlt/connector/client/DltGatewayClient.py
@@ -22,7 +22,6 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
from dlt.connector.Config import DLT_GATEWAY_HOST, DLT_GATEWAY_PORT
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
MAX_RETRIES = 15
DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')
diff --git a/src/dlt/connector/client/DltGatewayClientAsync.py b/src/dlt/connector/client/DltGatewayClientAsync.py
index 84c753e03..3f1cf5396 100644
--- a/src/dlt/connector/client/DltGatewayClientAsync.py
+++ b/src/dlt/connector/client/DltGatewayClientAsync.py
@@ -23,7 +23,6 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
from dlt.connector.Config import DLT_GATEWAY_HOST, DLT_GATEWAY_PORT
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
MAX_RETRIES = 15
DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')
diff --git a/src/dlt/connector/service/DltConnectorServiceServicerImpl.py b/src/dlt/connector/service/DltConnectorServiceServicerImpl.py
index edba01fac..2222f5a29 100644
--- a/src/dlt/connector/service/DltConnectorServiceServicerImpl.py
+++ b/src/dlt/connector/service/DltConnectorServiceServicerImpl.py
@@ -26,7 +26,6 @@ from dlt.connector.client.DltGatewayClientAsync import DltGatewayClientAsync
from .tools.Checkers import record_exists
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
METRICS_POOL = MetricsPool('DltConnector', 'RPC')
diff --git a/src/interdomain/service/topology_abstractor/DltRecordSender.py b/src/interdomain/service/topology_abstractor/DltRecordSender.py
index f91d6d547..ae9fd440b 100644
--- a/src/interdomain/service/topology_abstractor/DltRecordSender.py
+++ b/src/interdomain/service/topology_abstractor/DltRecordSender.py
@@ -23,7 +23,6 @@ from context.client.ContextClient import ContextClient
from dlt.connector.client.DltConnectorClientAsync import DltConnectorClientAsync
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
class DltRecordSender:
def __init__(self, context_client: ContextClient) -> None:
diff --git a/src/interdomain/service/topology_abstractor/DltRecorder.py b/src/interdomain/service/topology_abstractor/DltRecorder.py
index 79067072c..ae869b7c0 100644
--- a/src/interdomain/service/topology_abstractor/DltRecorder.py
+++ b/src/interdomain/service/topology_abstractor/DltRecorder.py
@@ -26,7 +26,6 @@ from .DltRecordSender import DltRecordSender
from .Types import EventTypes
LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME))
INTERDOMAIN_TOPOLOGY_ID = TopologyId(**json_topology_id(INTERDOMAIN_TOPOLOGY_NAME, context_id=ADMIN_CONTEXT_ID))
--
GitLab