Commit b235c131 authored by Pablo Armingol's avatar Pablo Armingol
Browse files

Merge branch... 

Merge branch 'feat/419-tid-clean-mk8s-on-deploy-deploy-components-in-parallel-detect-components-to-deploy' of https://labs.etsi.org/rep/tfs/controller into feat/416-tid-new-put-call-for-ipowdm-nbi
parents 81cd8cb4 158b156e

deploy/tfs.sh

+150 −71
Original line number Diff line number Diff line
@@ -111,8 +111,31 @@ export LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""} 
# Automated steps start here

########################################################################################################################



# Function to calculate the hash of a component's source directories

get_component_hash() {

    local COMP=$1

    local files="./src/$COMP"

    # If the component depends on the common python library, include common files and dependencies

    if [ -d "./src/common" ]; then

        files="$files ./src/common ./common_requirements.in"

    fi

    find $files -type f -not -name "*.pyc" -not -path "*/__pycache__/*" 2>/dev/null | sort | xargs md5sum 2>/dev/null | md5sum | cut -d' ' -f1

}



# Function to clean up Docker buildx cache and images on exit

cleanup() {

    echo "Pruning Docker Images..."

    docker image prune --force

    if [ "$DOCKER_BUILD" == "docker buildx build" ]; then

        echo "Pruning Docker Buildx Cache..."

        docker buildx prune --force

    fi

}

trap cleanup EXIT



bash scripts/dockerhub_login.sh





# Constants

GITLAB_REPO_URL="labs.etsi.org:5050/tfs/controller"

TMP_FOLDER="./tmp"
@@ -123,9 +146,11 @@ mkdir -p $TMP_MANIFESTS_FOLDER 
TMP_LOGS_FOLDER="${TMP_FOLDER}/${TFS_K8S_NAMESPACE}/logs"

mkdir -p $TMP_LOGS_FOLDER



echo "Deleting and Creating a new namespace..."

kubectl delete namespace $TFS_K8S_NAMESPACE --ignore-not-found

kubectl create namespace $TFS_K8S_NAMESPACE

echo "Cleaning up existing deployments and services (fast)..."

# Ensure the namespace exists first (failsafe)

kubectl create namespace $TFS_K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f -

# Asynchronously delete existing resources to start clean without blocking

kubectl delete deployments,services,ingresses,horizontalpodautoscalers --all --namespace $TFS_K8S_NAMESPACE --grace-period=0 --force --wait=false >/dev/null 2>&1

bash scripts/dockerhub_k8s_secret.sh "$TFS_K8S_NAMESPACE"

sleep 2

printf "\n"
@@ -235,81 +260,134 @@ if [[ "$TFS_COMPONENTS" == *"telemetry"* ]]; then 
    sed -i "s|_LOAD_BALANCER_IP_|$LOAD_BALANCER_IP|g" manifests/telemetryservice.yaml

fi



echo ">>> Phase 1: Building and pushing Docker images (in parallel)..."

# Parallel Build & Push Phase

MAX_CONCURRENT=${MAX_CONCURRENT:-$(($(nproc) / 4))}

if [ $MAX_CONCURRENT -lt 2 ]; then MAX_CONCURRENT=2; fi

CURRENT_JOBS=0

BUILD_PIDS=()

declare -A COMPONENT_BUILD_STATUS



for COMPONENT in $TFS_COMPONENTS; do

    echo "Processing '$COMPONENT' component..."

    # Calculate source code hash

    HASH_FILE="${TMP_LOGS_FOLDER}/build_${COMPONENT}.hash"

    CURRENT_HASH=$(get_component_hash "$COMPONENT")



    # Check if we should build

    SKIP_BUILD_FOR_COMPONENT="NO"

    if [ "$TFS_SKIP_BUILD" == "YES" ]; then

        SKIP_BUILD_FOR_COMPONENT="YES"

    elif [ -f "$HASH_FILE" ] && [ "$(cat "$HASH_FILE")" == "$CURRENT_HASH" ]; then

        echo "  No changes detected in '$COMPONENT' source code. Skipping build."

        SKIP_BUILD_FOR_COMPONENT="YES"

    fi



    if [ "$TFS_SKIP_BUILD" != "YES" ]; then

        echo "  Building Docker image..."

    if [ "$SKIP_BUILD_FOR_COMPONENT" != "YES" ]; then

        (

            echo "  [BG] Starting build for '$COMPONENT'..."

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"



            if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then

            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"

                $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" 2>&1

            elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"

            $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"

                BUILD_LOG_FT="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"

                $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG_FT" 2>&1 &

                PID_FT=$!

                

                BUILD_LOG_BK="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"

                $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG_BK" 2>&1 &

                PID_BK=$!

                

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"

            $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"

                wait $PID_FT && wait $PID_BK

                if [ "$COMPONENT" == "pathcomp" ]; then

                # next command is redundant, but helpful to keep cache updated between rebuilds

                    IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"

                $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

                    $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG_BK" 2>&1

                fi

            elif [ "$COMPONENT" == "dlt" ]; then

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"

            $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"

                BUILD_LOG_CONN="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"

                $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG_CONN" 2>&1 &

                PID_CONN=$!

                

            BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"

            $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG"

                BUILD_LOG_GW="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"

                $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG_GW" 2>&1 &

                PID_GW=$!

                

                wait $PID_CONN && wait $PID_GW

            else

            $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"

                $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" 2>&1

            fi



        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."

            if [ $? -ne 0 ]; then

                echo "  [BG] Build FAILED for '$COMPONENT'. Check logs in $TMP_LOGS_FOLDER."

                exit 1

            fi



            echo "  [BG] Pushing '$COMPONENT' image..."

            if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ] ; then

                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"

            docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

                docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1

                docker push "$IMAGE_URL" > /dev/null 2>&1

                docker rmi "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true



                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"

            docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

                docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1

                docker push "$IMAGE_URL" > /dev/null 2>&1

                docker rmi "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true

            elif [ "$COMPONENT" == "dlt" ]; then

                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"

            docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

                docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1

                docker push "$IMAGE_URL" > /dev/null 2>&1

                docker rmi "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true



                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"

            docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

                docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1

                docker push "$IMAGE_URL" > /dev/null 2>&1

                docker rmi "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true

            else

                IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

                docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1

                docker push "$IMAGE_URL" > /dev/null 2>&1

                docker rmi "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true

            fi



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"

            docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"

            # Save the successful build hash

            echo "$CURRENT_HASH" > "$HASH_FILE"

            echo "  [BG] Build and push finished for '$COMPONENT'."

        ) &

        BUILD_PIDS+=($!)

        COMPONENT_BUILD_STATUS[$!]=$COMPONENT



        # Throttle concurrency

        CURRENT_JOBS=$((CURRENT_JOBS + 1))

        if [ $CURRENT_JOBS -ge $MAX_CONCURRENT ]; then

            wait -n 2>/dev/null || wait

            CURRENT_JOBS=$((CURRENT_JOBS - 1))

        fi

    fi

done



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

# Wait for all background builds to finish

if [ ${#BUILD_PIDS[@]} -gt 0 ]; then

    echo ">>> Waiting for all parallel builds to complete..."

    FAILED_BUILDS=0

    for pid in "${BUILD_PIDS[@]}"; do

        wait $pid

        exit_code=$?

        if [ $exit_code -ne 0 ]; then

            echo "  Error: Build for component '${COMPONENT_BUILD_STATUS[$pid]}' failed."

            FAILED_BUILDS=$((FAILED_BUILDS + 1))

        fi

    done

    if [ $FAILED_BUILDS -gt 0 ]; then

        echo "Error: One or more parallel builds failed. Exiting deployment."

        exit 1

    fi

fi



echo ">>> Phase 2: Deploying components to Kubernetes (sequentially)..."

for COMPONENT in $TFS_COMPONENTS; do

    echo "Processing '$COMPONENT' component deployment..."



    echo "  Adapting '$COMPONENT' manifest file..."

    MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"

    if [[ $linkerd_status =~ "enabled" ]]; then
@@ -348,8 +426,6 @@ for COMPONENT in $TFS_COMPONENTS; do 
    DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"

    kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"

    COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/g")

    #kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"

    #kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"



    echo "  Collecting env-vars for '$COMPONENT' component..."
@@ -633,3 +709,6 @@ if [ "$DOCKER_BUILD" == "docker buildx build" ]; then 
    docker buildx prune --force

    printf "\n\n"

fi



echo ">>> Running MicroK8s cleanup in the background..."

sudo -n ./scripts/clean_microk8s.sh >/dev/null 2>&1 &

scripts/clean_microk8s.sh

0 → 100755
+49 −0
Original line number Diff line number Diff line 
#!/bin/bash



echo "=== Starting MicroK8s cleanup ($(date)) ==="



# 1. Get the images that are currently in use by active pods in Kubernetes

echo "Searching for images in use..."

RUNNING_IMAGES=$(kubectl get pods -A -o jsonpath='{.items[*].spec.containers[*].image} {.items[*].spec.initContainers[*].image}' 2>/dev/null | tr ' ' '\n' | sort -u)



if [ -z "$RUNNING_IMAGES" ]; then

    echo "Error: Could not retrieve the list of active images from Kubernetes (is the cluster down?). Aborting image cleanup to avoid deleting everything."

else

    # 2. List and delete inactive containerd images

    echo "Cleaning up inactive images from containerd..."

    /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io images list -q | while read -r img; do

        if ! echo "$RUNNING_IMAGES" | grep -F -q "$img"; then

            echo "Deleting unused image: $img"

            /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io images rm "$img" >/dev/null 2>&1

        fi

    done



    # 3. Force the release of physical space (orphaned layers/blobs)

    echo "Forcing the release of physical space (orphaned layers/blobs)..."

    /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io content prune references >/dev/null 2>&1



    # 3b. Run Garbage Collector in the private image registry

    echo "Running Garbage Collector in the private image registry..."

    kubectl exec -n container-registry deployment/registry -- registry garbage-collect /etc/docker/registry/config.yml >/dev/null 2>&1

fi



# 4. Cleaning up orphaned persistent volumes (default-storage)

echo "Searching for orphaned volumes in default-storage..."

ACTIVE_PATHS=$(kubectl get pv -o jsonpath='{range .items[*]}{.spec.hostPath.path}{"\n"}{end}' 2>/dev/null)



if [ -n "$ACTIVE_PATHS" ]; then

    for dir in /var/snap/microk8s/common/default-storage/*; do

        if [ -d "$dir" ]; then

            folder_name=$(basename "$dir")

            # Ignore the registry's own folder if active, otherwise validate

            if ! echo "$ACTIVE_PATHS" | grep -q "$folder_name"; then

                echo "Deleting orphaned volume: $dir"

                rm -rf "$dir"

            fi

        fi

    done

fi



echo "=== Cleanup completed ($(date)) ==="

echo "Current disk space:"

df -h /

src/analytics/backend/Dockerfile

+19 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



FROM python:3.9-slim

FROM python:3.9-slim AS builder



# Install dependencies

RUN apt-get --yes --quiet --quiet update && \
@@ -66,4 +66,22 @@ COPY src/analytics/__init__.py analytics/__init__.py 
COPY src/analytics/backend/. analytics/backend/



# Start the service



# ==========================================

# Stage 2: Runner image

# ==========================================

FROM python:3.9-slim



ENV PYTHONUNBUFFERED=0



# Copy health probe

COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe



# Copy installed libraries and binaries

COPY --from=builder /usr/local /usr/local



# Copy app code

WORKDIR /var/teraflow

COPY --from=builder /var/teraflow /var/teraflow



ENTRYPOINT ["python", "-m", "analytics.backend.service"]

src/analytics/frontend/Dockerfile

+19 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



FROM python:3.9-slim

FROM python:3.9-slim AS builder



# Install dependencies

RUN apt-get --yes --quiet --quiet update && \
@@ -67,4 +67,22 @@ COPY src/analytics/frontend/. analytics/frontend/ 
COPY src/analytics/database/. analytics/database/



# Start the service



# ==========================================

# Stage 2: Runner image

# ==========================================

FROM python:3.9-slim



ENV PYTHONUNBUFFERED=0



# Copy health probe

COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe



# Copy installed libraries and binaries

COPY --from=builder /usr/local /usr/local



# Copy app code

WORKDIR /var/teraflow

COPY --from=builder /var/teraflow /var/teraflow



ENTRYPOINT ["python", "-m", "analytics.frontend.service"]

src/automation/Dockerfile

+19 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



FROM python:3.9-slim

FROM python:3.9-slim AS builder



# Install dependencies

RUN apt-get --yes --quiet --quiet update && \
@@ -75,4 +75,22 @@ COPY src/analytics/frontend/client/. analytics/frontend/client/ 
COPY src/automation/. automation/



# Start the service



# ==========================================

# Stage 2: Runner image

# ==========================================

FROM python:3.9-slim



ENV PYTHONUNBUFFERED=0



# Copy health probe

COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe



# Copy installed libraries and binaries

COPY --from=builder /usr/local /usr/local



# Copy app code

WORKDIR /var/teraflow

COPY --from=builder /var/teraflow /var/teraflow



ENTRYPOINT ["python", "-m", "automation.service"]