Pre-merge code cleanup

deploy/tfs.sh

@@ -45,10 +45,6 @@ export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}
 # If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.
 export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-""}
 
-# If not already set, disable build-if-exists flag to skip building Docker images if they already exist.
-# If TFS_BUILD_IF_EXISTS is "NO", the containers are not rebuilt if they already exist.
-export TFS_BUILD_IF_EXISTS=${TFS_BUILD_IF_EXISTS:-"YES"}
-
 
 # ----- CockroachDB ------------------------------------------------------------
 
@@ -118,6 +114,7 @@ export PROM_EXT_PORT_HTTP=${PROM_EXT_PORT_HTTP:-"9090"}
 # If not already set, set the external port Grafana HTTP Dashboards will be exposed to.
 export GRAF_EXT_PORT_HTTP=${GRAF_EXT_PORT_HTTP:-"3000"}
 
+
 ########################################################################################################################
 # Automated steps start here
 ########################################################################################################################
@@ -247,77 +244,72 @@ for COMPONENT in $TFS_COMPONENTS; do
     echo "Processing '$COMPONENT' component..."
 
     if [ "$TFS_SKIP_BUILD" != "YES" ]; then
-        IMAGE_EXISTS=$(docker images -q "$COMPONENT:$TFS_IMAGE_TAG")
-        if [ -z "$IMAGE_EXISTS" ] || [ "$TFS_BUILD_IF_EXISTS" == "YES" ]; then
-            echo "  Building Docker image..."
-            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
-
-            if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then
-                $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
-            elif [ "$COMPONENT" == "pathcomp" ]; then
-                BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
-                $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"
-
-                BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"
-                $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"
-                # next command is redundant, but helpful to keep cache updated between rebuilds
-                IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"
-                $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
-            elif [ "$COMPONENT" == "dlt" ]; then
-                BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"
-                $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
-
-                BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"
-                $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG"
-            else
-                $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"
-            fi
+        echo "  Building Docker image..."
+        BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
+
+        if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then
+            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
+        elif [ "$COMPONENT" == "pathcomp" ]; then
+            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
+            $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"
+
+            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"
+            $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"
+            # next command is redundant, but helpful to keep cache updated between rebuilds
+            IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"
+            $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
+        elif [ "$COMPONENT" == "dlt" ]; then
+            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"
+            $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
+
+            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"
+            $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG"
+        else
+            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"
+        fi
 
-            echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."
+        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."
 
-            if [ "$COMPONENT" == "pathcomp" ]; then
-                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+        if [ "$COMPONENT" == "pathcomp" ]; then
+            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
 
-                TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
-                docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
+            docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
 
-                PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"
-                docker push "$IMAGE_URL" > "$PUSH_LOG"
+            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"
+            docker push "$IMAGE_URL" > "$PUSH_LOG"
 
-                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
 
-                TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"
-                docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"
+            docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
 
-                PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"
-                docker push "$IMAGE_URL" > "$PUSH_LOG"
-            elif [ "$COMPONENT" == "dlt" ]; then
-                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"
+            docker push "$IMAGE_URL" > "$PUSH_LOG"
+        elif [ "$COMPONENT" == "dlt" ]; then
+            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
 
-                TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"
-                docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"
+            docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
 
-                PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"
-                docker push "$IMAGE_URL" > "$PUSH_LOG"
+            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"
+            docker push "$IMAGE_URL" > "$PUSH_LOG"
 
-                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
 
-                TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"
-                docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"
+            docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
 
-                PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"
-                docker push "$IMAGE_URL" > "$PUSH_LOG"
-            else
-                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
+            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"
+            docker push "$IMAGE_URL" > "$PUSH_LOG"
+        else
+            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
 
-                TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
-                docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
+            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
+            docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
 
-                PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
-                docker push "$IMAGE_URL" > "$PUSH_LOG"
-            fi
-        else
-            echo "  Skipping Docker build for '$COMPONENT' as the image already exists and TFS_BUILD_IF_EXISTS is set to 'NO'."
+            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
+            docker push "$IMAGE_URL" > "$PUSH_LOG"
         fi
     fi
 

manifests/dltservice.yaml

@@ -74,6 +74,18 @@ spec:
           imagePullPolicy: Always
           ports:
             - containerPort: 50051
+          #readinessProbe:
+          #  httpGet:
+          #    path: /health
+          #    port: 8081
+          #  initialDelaySeconds: 5
+          #  timeoutSeconds: 5
+          #livenessProbe:
+          #  httpGet:
+          #    path: /health
+          #    port: 8081
+          #  initialDelaySeconds: 5
+          #  timeoutSeconds: 5
           resources:
             requests:
               cpu: 200m

src/dlt/connector/client/DltConnectorClient.py

@@ -22,7 +22,6 @@ from common.tools.client.RetryDecorator import retry, delay_exponential
 from common.tools.grpc.Tools import grpc_message_to_json_string
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 MAX_RETRIES = 15
 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
 RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')

src/dlt/connector/client/DltConnectorClientAsync.py

@@ -23,7 +23,6 @@ from common.tools.client.RetryDecorator import retry, delay_exponential
 from common.tools.grpc.Tools import grpc_message_to_json_string
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 MAX_RETRIES = 15
 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
 RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')

src/dlt/connector/client/DltEventsCollector.py

@@ -19,7 +19,6 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
 from dlt.connector.client.DltGatewayClient import DltGatewayClient
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 
 # This class accepts an event_handler method as attribute that can be used to pre-process and
 # filter events before they reach the events_queue. Depending on the handler, the supported

src/dlt/connector/client/DltGatewayClient.py

@@ -22,7 +22,6 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
 from dlt.connector.Config import DLT_GATEWAY_HOST, DLT_GATEWAY_PORT
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 MAX_RETRIES = 15
 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
 RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')

src/dlt/connector/client/DltGatewayClientAsync.py

@@ -23,7 +23,6 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
 from dlt.connector.Config import DLT_GATEWAY_HOST, DLT_GATEWAY_PORT
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 MAX_RETRIES = 15
 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
 RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')

src/dlt/connector/service/DltConnectorServiceServicerImpl.py

@@ -26,7 +26,6 @@ from dlt.connector.client.DltGatewayClientAsync import DltGatewayClientAsync
 from .tools.Checkers import record_exists
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 
 METRICS_POOL = MetricsPool('DltConnector', 'RPC')
 

src/interdomain/service/topology_abstractor/DltRecordSender.py

@@ -23,7 +23,6 @@ from context.client.ContextClient import ContextClient
 from dlt.connector.client.DltConnectorClientAsync import DltConnectorClientAsync
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 
 class DltRecordSender:
     def __init__(self, context_client: ContextClient) -> None:

src/interdomain/service/topology_abstractor/DltRecorder.py

@@ -26,7 +26,6 @@ from .DltRecordSender import DltRecordSender
 from .Types import EventTypes
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.DEBUG)
 
 ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME))
 INTERDOMAIN_TOPOLOGY_ID = TopologyId(**json_topology_id(INTERDOMAIN_TOPOLOGY_NAME, context_id=ADMIN_CONTEXT_ID))