Loading deploy/tfs.sh +150 −71 Original line number Diff line number Diff line Loading @@ -111,8 +111,31 @@ export LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""} # Automated steps start here ######################################################################################################################## # Function to calculate the hash of a component's source directories get_component_hash() { local COMP=$1 local files="./src/$COMP" # If the component depends on the common python library, include common files and dependencies if [ -d "./src/common" ]; then files="$files ./src/common ./common_requirements.in" fi find $files -type f -not -name "*.pyc" -not -path "*/__pycache__/*" 2>/dev/null | sort | xargs md5sum 2>/dev/null | md5sum | cut -d' ' -f1 } # Function to clean up Docker buildx cache and images on exit cleanup() { echo "Pruning Docker Images..." docker image prune --force if [ "$DOCKER_BUILD" == "docker buildx build" ]; then echo "Pruning Docker Buildx Cache..." docker buildx prune --force fi } trap cleanup EXIT bash scripts/dockerhub_login.sh # Constants GITLAB_REPO_URL="labs.etsi.org:5050/tfs/controller" TMP_FOLDER="./tmp" Loading @@ -123,9 +146,11 @@ mkdir -p $TMP_MANIFESTS_FOLDER TMP_LOGS_FOLDER="${TMP_FOLDER}/${TFS_K8S_NAMESPACE}/logs" mkdir -p $TMP_LOGS_FOLDER echo "Deleting and Creating a new namespace..." kubectl delete namespace $TFS_K8S_NAMESPACE --ignore-not-found kubectl create namespace $TFS_K8S_NAMESPACE echo "Cleaning up existing deployments and services (fast)..." # Ensure the namespace exists first (failsafe) kubectl create namespace $TFS_K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - # Asynchronously delete existing resources to start clean without blocking kubectl delete deployments,services,ingresses,horizontalpodautoscalers --all --namespace $TFS_K8S_NAMESPACE --grace-period=0 --force --wait=false >/dev/null 2>&1 bash scripts/dockerhub_k8s_secret.sh "$TFS_K8S_NAMESPACE" sleep 2 printf "\n" Loading Loading @@ -235,81 +260,134 @@ if [[ "$TFS_COMPONENTS" == *"telemetry"* ]]; then sed -i "s|_LOAD_BALANCER_IP_|$LOAD_BALANCER_IP|g" manifests/telemetryservice.yaml fi echo ">>> Phase 1: Building and pushing Docker images (in parallel)..." # Parallel Build & Push Phase MAX_CONCURRENT=${MAX_CONCURRENT:-$(($(nproc) / 4))} if [ $MAX_CONCURRENT -lt 2 ]; then MAX_CONCURRENT=2; fi CURRENT_JOBS=0 BUILD_PIDS=() declare -A COMPONENT_BUILD_STATUS for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." # Calculate source code hash HASH_FILE="${TMP_LOGS_FOLDER}/build_${COMPONENT}.hash" CURRENT_HASH=$(get_component_hash "$COMPONENT") # Check if we should build SKIP_BUILD_FOR_COMPONENT="NO" if [ "$TFS_SKIP_BUILD" == "YES" ]; then SKIP_BUILD_FOR_COMPONENT="YES" elif [ -f "$HASH_FILE" ] && [ "$(cat "$HASH_FILE")" == "$CURRENT_HASH" ]; then echo " No changes detected in '$COMPONENT' source code. Skipping build." SKIP_BUILD_FOR_COMPONENT="YES" fi if [ "$TFS_SKIP_BUILD" != "YES" ]; then echo " Building Docker image..." if [ "$SKIP_BUILD_FOR_COMPONENT" != "YES" ]; then ( echo " [BG] Starting build for '$COMPONENT'..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" 2>&1 elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG_FT="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG_FT" 2>&1 & PID_FT=$! BUILD_LOG_BK="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG_BK" 2>&1 & PID_BK=$! BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" wait $PID_FT && wait $PID_BK if [ "$COMPONENT" == "pathcomp" ]; then # next command is redundant, but helpful to keep cache updated between rebuilds IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG_BK" 2>&1 fi elif [ "$COMPONENT" == "dlt" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" BUILD_LOG_CONN="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG_CONN" 2>&1 & PID_CONN=$! BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" BUILD_LOG_GW="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG_GW" 2>&1 & PID_GW=$! wait $PID_CONN && wait $PID_GW else $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" 2>&1 fi echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..." if [ $? -ne 0 ]; then echo " [BG] Build FAILED for '$COMPONENT'. Check logs in $TMP_LOGS_FOLDER." exit 1 fi echo " [BG] Pushing '$COMPONENT' image..." if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ] ; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true elif [ "$COMPONENT" == "dlt" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log" docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log" docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true else IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true fi TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" # Save the successful build hash echo "$CURRENT_HASH" > "$HASH_FILE" echo " [BG] Build and push finished for '$COMPONENT'." ) & BUILD_PIDS+=($!) COMPONENT_BUILD_STATUS[$!]=$COMPONENT # Throttle concurrency CURRENT_JOBS=$((CURRENT_JOBS + 1)) if [ $CURRENT_JOBS -ge $MAX_CONCURRENT ]; then wait -n 2>/dev/null || wait CURRENT_JOBS=$((CURRENT_JOBS - 1)) fi fi done PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" # Wait for all background builds to finish if [ ${#BUILD_PIDS[@]} -gt 0 ]; then echo ">>> Waiting for all parallel builds to complete..." FAILED_BUILDS=0 for pid in "${BUILD_PIDS[@]}"; do wait $pid exit_code=$? if [ $exit_code -ne 0 ]; then echo " Error: Build for component '${COMPONENT_BUILD_STATUS[$pid]}' failed." FAILED_BUILDS=$((FAILED_BUILDS + 1)) fi done if [ $FAILED_BUILDS -gt 0 ]; then echo "Error: One or more parallel builds failed. Exiting deployment." exit 1 fi fi echo ">>> Phase 2: Deploying components to Kubernetes (sequentially)..." for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component deployment..." echo " Adapting '$COMPONENT' manifest file..." MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml" if [[ $linkerd_status =~ "enabled" ]]; then Loading Loading @@ -348,8 +426,6 @@ for COMPONENT in $TFS_COMPONENTS; do DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log" kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG" COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/g") #kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG" #kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG" echo " Collecting env-vars for '$COMPONENT' component..." Loading Loading @@ -633,3 +709,6 @@ if [ "$DOCKER_BUILD" == "docker buildx build" ]; then docker buildx prune --force printf "\n\n" fi echo ">>> Running MicroK8s cleanup in the background..." sudo -n ./scripts/clean_microk8s.sh >/dev/null 2>&1 & scripts/clean_microk8s.sh 0 → 100755 +49 −0 Original line number Diff line number Diff line #!/bin/bash echo "=== Starting MicroK8s cleanup ($(date)) ===" # 1. Get the images that are currently in use by active pods in Kubernetes echo "Searching for images in use..." RUNNING_IMAGES=$(kubectl get pods -A -o jsonpath='{.items[*].spec.containers[*].image} {.items[*].spec.initContainers[*].image}' 2>/dev/null | tr ' ' '\n' | sort -u) if [ -z "$RUNNING_IMAGES" ]; then echo "Error: Could not retrieve the list of active images from Kubernetes (is the cluster down?). Aborting image cleanup to avoid deleting everything." else # 2. List and delete inactive containerd images echo "Cleaning up inactive images from containerd..." /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io images list -q | while read -r img; do if ! echo "$RUNNING_IMAGES" | grep -F -q "$img"; then echo "Deleting unused image: $img" /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io images rm "$img" >/dev/null 2>&1 fi done # 3. Force the release of physical space (orphaned layers/blobs) echo "Forcing the release of physical space (orphaned layers/blobs)..." /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io content prune references >/dev/null 2>&1 # 3b. Run Garbage Collector in the private image registry echo "Running Garbage Collector in the private image registry..." kubectl exec -n container-registry deployment/registry -- registry garbage-collect /etc/docker/registry/config.yml >/dev/null 2>&1 fi # 4. Cleaning up orphaned persistent volumes (default-storage) echo "Searching for orphaned volumes in default-storage..." ACTIVE_PATHS=$(kubectl get pv -o jsonpath='{range .items[*]}{.spec.hostPath.path}{"\n"}{end}' 2>/dev/null) if [ -n "$ACTIVE_PATHS" ]; then for dir in /var/snap/microk8s/common/default-storage/*; do if [ -d "$dir" ]; then folder_name=$(basename "$dir") # Ignore the registry's own folder if active, otherwise validate if ! echo "$ACTIVE_PATHS" | grep -q "$folder_name"; then echo "Deleting orphaned volume: $dir" rm -rf "$dir" fi fi done fi echo "=== Cleanup completed ($(date)) ===" echo "Current disk space:" df -h / src/analytics/backend/Dockerfile +19 −1 Original line number Diff line number Diff line Loading @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.9-slim FROM python:3.9-slim AS builder # Install dependencies RUN apt-get --yes --quiet --quiet update && \ Loading Loading @@ -66,4 +66,22 @@ COPY src/analytics/__init__.py analytics/__init__.py COPY src/analytics/backend/. analytics/backend/ # Start the service # ========================================== # Stage 2: Runner image # ========================================== FROM python:3.9-slim ENV PYTHONUNBUFFERED=0 # Copy health probe COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe # Copy installed libraries and binaries COPY --from=builder /usr/local /usr/local # Copy app code WORKDIR /var/teraflow COPY --from=builder /var/teraflow /var/teraflow ENTRYPOINT ["python", "-m", "analytics.backend.service"] src/analytics/frontend/Dockerfile +19 −1 Original line number Diff line number Diff line Loading @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.9-slim FROM python:3.9-slim AS builder # Install dependencies RUN apt-get --yes --quiet --quiet update && \ Loading Loading @@ -67,4 +67,22 @@ COPY src/analytics/frontend/. analytics/frontend/ COPY src/analytics/database/. analytics/database/ # Start the service # ========================================== # Stage 2: Runner image # ========================================== FROM python:3.9-slim ENV PYTHONUNBUFFERED=0 # Copy health probe COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe # Copy installed libraries and binaries COPY --from=builder /usr/local /usr/local # Copy app code WORKDIR /var/teraflow COPY --from=builder /var/teraflow /var/teraflow ENTRYPOINT ["python", "-m", "analytics.frontend.service"] src/automation/Dockerfile +19 −1 Original line number Diff line number Diff line Loading @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.9-slim FROM python:3.9-slim AS builder # Install dependencies RUN apt-get --yes --quiet --quiet update && \ Loading Loading @@ -75,4 +75,22 @@ COPY src/analytics/frontend/client/. analytics/frontend/client/ COPY src/automation/. automation/ # Start the service # ========================================== # Stage 2: Runner image # ========================================== FROM python:3.9-slim ENV PYTHONUNBUFFERED=0 # Copy health probe COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe # Copy installed libraries and binaries COPY --from=builder /usr/local /usr/local # Copy app code WORKDIR /var/teraflow COPY --from=builder /var/teraflow /var/teraflow ENTRYPOINT ["python", "-m", "automation.service"] Loading
deploy/tfs.sh +150 −71 Original line number Diff line number Diff line Loading @@ -111,8 +111,31 @@ export LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""} # Automated steps start here ######################################################################################################################## # Function to calculate the hash of a component's source directories get_component_hash() { local COMP=$1 local files="./src/$COMP" # If the component depends on the common python library, include common files and dependencies if [ -d "./src/common" ]; then files="$files ./src/common ./common_requirements.in" fi find $files -type f -not -name "*.pyc" -not -path "*/__pycache__/*" 2>/dev/null | sort | xargs md5sum 2>/dev/null | md5sum | cut -d' ' -f1 } # Function to clean up Docker buildx cache and images on exit cleanup() { echo "Pruning Docker Images..." docker image prune --force if [ "$DOCKER_BUILD" == "docker buildx build" ]; then echo "Pruning Docker Buildx Cache..." docker buildx prune --force fi } trap cleanup EXIT bash scripts/dockerhub_login.sh # Constants GITLAB_REPO_URL="labs.etsi.org:5050/tfs/controller" TMP_FOLDER="./tmp" Loading @@ -123,9 +146,11 @@ mkdir -p $TMP_MANIFESTS_FOLDER TMP_LOGS_FOLDER="${TMP_FOLDER}/${TFS_K8S_NAMESPACE}/logs" mkdir -p $TMP_LOGS_FOLDER echo "Deleting and Creating a new namespace..." kubectl delete namespace $TFS_K8S_NAMESPACE --ignore-not-found kubectl create namespace $TFS_K8S_NAMESPACE echo "Cleaning up existing deployments and services (fast)..." # Ensure the namespace exists first (failsafe) kubectl create namespace $TFS_K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - # Asynchronously delete existing resources to start clean without blocking kubectl delete deployments,services,ingresses,horizontalpodautoscalers --all --namespace $TFS_K8S_NAMESPACE --grace-period=0 --force --wait=false >/dev/null 2>&1 bash scripts/dockerhub_k8s_secret.sh "$TFS_K8S_NAMESPACE" sleep 2 printf "\n" Loading Loading @@ -235,81 +260,134 @@ if [[ "$TFS_COMPONENTS" == *"telemetry"* ]]; then sed -i "s|_LOAD_BALANCER_IP_|$LOAD_BALANCER_IP|g" manifests/telemetryservice.yaml fi echo ">>> Phase 1: Building and pushing Docker images (in parallel)..." # Parallel Build & Push Phase MAX_CONCURRENT=${MAX_CONCURRENT:-$(($(nproc) / 4))} if [ $MAX_CONCURRENT -lt 2 ]; then MAX_CONCURRENT=2; fi CURRENT_JOBS=0 BUILD_PIDS=() declare -A COMPONENT_BUILD_STATUS for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." # Calculate source code hash HASH_FILE="${TMP_LOGS_FOLDER}/build_${COMPONENT}.hash" CURRENT_HASH=$(get_component_hash "$COMPONENT") # Check if we should build SKIP_BUILD_FOR_COMPONENT="NO" if [ "$TFS_SKIP_BUILD" == "YES" ]; then SKIP_BUILD_FOR_COMPONENT="YES" elif [ -f "$HASH_FILE" ] && [ "$(cat "$HASH_FILE")" == "$CURRENT_HASH" ]; then echo " No changes detected in '$COMPONENT' source code. Skipping build." SKIP_BUILD_FOR_COMPONENT="YES" fi if [ "$TFS_SKIP_BUILD" != "YES" ]; then echo " Building Docker image..." if [ "$SKIP_BUILD_FOR_COMPONENT" != "YES" ]; then ( echo " [BG] Starting build for '$COMPONENT'..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" 2>&1 elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG_FT="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG_FT" 2>&1 & PID_FT=$! BUILD_LOG_BK="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG_BK" 2>&1 & PID_BK=$! BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" wait $PID_FT && wait $PID_BK if [ "$COMPONENT" == "pathcomp" ]; then # next command is redundant, but helpful to keep cache updated between rebuilds IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG_BK" 2>&1 fi elif [ "$COMPONENT" == "dlt" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" BUILD_LOG_CONN="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG_CONN" 2>&1 & PID_CONN=$! BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" BUILD_LOG_GW="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG_GW" 2>&1 & PID_GW=$! wait $PID_CONN && wait $PID_GW else $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" 2>&1 fi echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..." if [ $? -ne 0 ]; then echo " [BG] Build FAILED for '$COMPONENT'. Check logs in $TMP_LOGS_FOLDER." exit 1 fi echo " [BG] Pushing '$COMPONENT' image..." if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ] ; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true elif [ "$COMPONENT" == "dlt" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log" docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log" docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log" docker push "$IMAGE_URL" > "$PUSH_LOG" docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true else IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 docker push "$IMAGE_URL" > /dev/null 2>&1 docker rmi "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > /dev/null 2>&1 || true fi TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" # Save the successful build hash echo "$CURRENT_HASH" > "$HASH_FILE" echo " [BG] Build and push finished for '$COMPONENT'." ) & BUILD_PIDS+=($!) COMPONENT_BUILD_STATUS[$!]=$COMPONENT # Throttle concurrency CURRENT_JOBS=$((CURRENT_JOBS + 1)) if [ $CURRENT_JOBS -ge $MAX_CONCURRENT ]; then wait -n 2>/dev/null || wait CURRENT_JOBS=$((CURRENT_JOBS - 1)) fi fi done PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" # Wait for all background builds to finish if [ ${#BUILD_PIDS[@]} -gt 0 ]; then echo ">>> Waiting for all parallel builds to complete..." FAILED_BUILDS=0 for pid in "${BUILD_PIDS[@]}"; do wait $pid exit_code=$? if [ $exit_code -ne 0 ]; then echo " Error: Build for component '${COMPONENT_BUILD_STATUS[$pid]}' failed." FAILED_BUILDS=$((FAILED_BUILDS + 1)) fi done if [ $FAILED_BUILDS -gt 0 ]; then echo "Error: One or more parallel builds failed. Exiting deployment." exit 1 fi fi echo ">>> Phase 2: Deploying components to Kubernetes (sequentially)..." for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component deployment..." echo " Adapting '$COMPONENT' manifest file..." MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml" if [[ $linkerd_status =~ "enabled" ]]; then Loading Loading @@ -348,8 +426,6 @@ for COMPONENT in $TFS_COMPONENTS; do DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log" kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG" COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/g") #kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG" #kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG" echo " Collecting env-vars for '$COMPONENT' component..." Loading Loading @@ -633,3 +709,6 @@ if [ "$DOCKER_BUILD" == "docker buildx build" ]; then docker buildx prune --force printf "\n\n" fi echo ">>> Running MicroK8s cleanup in the background..." sudo -n ./scripts/clean_microk8s.sh >/dev/null 2>&1 &
scripts/clean_microk8s.sh 0 → 100755 +49 −0 Original line number Diff line number Diff line #!/bin/bash echo "=== Starting MicroK8s cleanup ($(date)) ===" # 1. Get the images that are currently in use by active pods in Kubernetes echo "Searching for images in use..." RUNNING_IMAGES=$(kubectl get pods -A -o jsonpath='{.items[*].spec.containers[*].image} {.items[*].spec.initContainers[*].image}' 2>/dev/null | tr ' ' '\n' | sort -u) if [ -z "$RUNNING_IMAGES" ]; then echo "Error: Could not retrieve the list of active images from Kubernetes (is the cluster down?). Aborting image cleanup to avoid deleting everything." else # 2. List and delete inactive containerd images echo "Cleaning up inactive images from containerd..." /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io images list -q | while read -r img; do if ! echo "$RUNNING_IMAGES" | grep -F -q "$img"; then echo "Deleting unused image: $img" /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io images rm "$img" >/dev/null 2>&1 fi done # 3. Force the release of physical space (orphaned layers/blobs) echo "Forcing the release of physical space (orphaned layers/blobs)..." /snap/microk8s/current/bin/ctr -a /var/snap/microk8s/common/run/containerd.sock -n k8s.io content prune references >/dev/null 2>&1 # 3b. Run Garbage Collector in the private image registry echo "Running Garbage Collector in the private image registry..." kubectl exec -n container-registry deployment/registry -- registry garbage-collect /etc/docker/registry/config.yml >/dev/null 2>&1 fi # 4. Cleaning up orphaned persistent volumes (default-storage) echo "Searching for orphaned volumes in default-storage..." ACTIVE_PATHS=$(kubectl get pv -o jsonpath='{range .items[*]}{.spec.hostPath.path}{"\n"}{end}' 2>/dev/null) if [ -n "$ACTIVE_PATHS" ]; then for dir in /var/snap/microk8s/common/default-storage/*; do if [ -d "$dir" ]; then folder_name=$(basename "$dir") # Ignore the registry's own folder if active, otherwise validate if ! echo "$ACTIVE_PATHS" | grep -q "$folder_name"; then echo "Deleting orphaned volume: $dir" rm -rf "$dir" fi fi done fi echo "=== Cleanup completed ($(date)) ===" echo "Current disk space:" df -h /
src/analytics/backend/Dockerfile +19 −1 Original line number Diff line number Diff line Loading @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.9-slim FROM python:3.9-slim AS builder # Install dependencies RUN apt-get --yes --quiet --quiet update && \ Loading Loading @@ -66,4 +66,22 @@ COPY src/analytics/__init__.py analytics/__init__.py COPY src/analytics/backend/. analytics/backend/ # Start the service # ========================================== # Stage 2: Runner image # ========================================== FROM python:3.9-slim ENV PYTHONUNBUFFERED=0 # Copy health probe COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe # Copy installed libraries and binaries COPY --from=builder /usr/local /usr/local # Copy app code WORKDIR /var/teraflow COPY --from=builder /var/teraflow /var/teraflow ENTRYPOINT ["python", "-m", "analytics.backend.service"]
src/analytics/frontend/Dockerfile +19 −1 Original line number Diff line number Diff line Loading @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.9-slim FROM python:3.9-slim AS builder # Install dependencies RUN apt-get --yes --quiet --quiet update && \ Loading Loading @@ -67,4 +67,22 @@ COPY src/analytics/frontend/. analytics/frontend/ COPY src/analytics/database/. analytics/database/ # Start the service # ========================================== # Stage 2: Runner image # ========================================== FROM python:3.9-slim ENV PYTHONUNBUFFERED=0 # Copy health probe COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe # Copy installed libraries and binaries COPY --from=builder /usr/local /usr/local # Copy app code WORKDIR /var/teraflow COPY --from=builder /var/teraflow /var/teraflow ENTRYPOINT ["python", "-m", "analytics.frontend.service"]
src/automation/Dockerfile +19 −1 Original line number Diff line number Diff line Loading @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. FROM python:3.9-slim FROM python:3.9-slim AS builder # Install dependencies RUN apt-get --yes --quiet --quiet update && \ Loading Loading @@ -75,4 +75,22 @@ COPY src/analytics/frontend/client/. analytics/frontend/client/ COPY src/automation/. automation/ # Start the service # ========================================== # Stage 2: Runner image # ========================================== FROM python:3.9-slim ENV PYTHONUNBUFFERED=0 # Copy health probe COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe # Copy installed libraries and binaries COPY --from=builder /usr/local /usr/local # Copy app code WORKDIR /var/teraflow COPY --from=builder /var/teraflow /var/teraflow ENTRYPOINT ["python", "-m", "automation.service"]
