Commit 4e8ac914 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Deploy scripts: 

- Added parallel build of components through docker buildx bake
- Added pre-deploy of services before workloads
parent ee1b4511

deploy/tfs.sh

+127 −13
Original line number Diff line number Diff line
@@ -212,6 +212,12 @@ if [[ $DOCKER_MAJOR_VERSION -ge 23 ]]; then 
    DOCKER_BUILD="docker buildx build"

fi



BUILDX_AVAILABLE=""

docker buildx version 1>/dev/null 2>/dev/null

if [[ $? -eq 0 ]]; then

    BUILDX_AVAILABLE="YES"

fi



LINKERD_STATUS="$(microk8s status -a linkerd)"

if [[ $linkerd_status =~ "enabled" ]]; then

    echo "LinkerD installed: workloads will be injected"
@@ -220,6 +226,45 @@ else 
fi

printf "\n"



normalize_registry() {

    echo "$1" | sed 's,//,/,g' | sed 's,http:/,,g' | sed 's,https:/,,g'

}



REGISTRY_PREFIX="$(normalize_registry "$TFS_REGISTRY_IMAGES")"

if [ -n "$REGISTRY_PREFIX" ]; then

    REGISTRY_PREFIX="${REGISTRY_PREFIX%/}/"

fi



image_url() {

    local image_name="$1"

    if [ -n "$REGISTRY_PREFIX" ]; then

        echo "${REGISTRY_PREFIX}${image_name}:${TFS_IMAGE_TAG}"

    else

        echo "${image_name}:${TFS_IMAGE_TAG}"

    fi

}



write_bake_target() {

    local bake_file="$1"

    local name="$2"

    local context="$3"

    local dockerfile="$4"

    local tag="$5"

    local target_stage="$6"



    {

        echo "target \"${name}\" {"

        echo "  context = \"${context}\""

        echo "  dockerfile = \"${dockerfile}\""

        echo "  tags = [\"${tag}\"]"

        if [ -n "$target_stage" ]; then

            echo "  target = \"${target_stage}\""

        fi

        echo "}"

        echo

    } >> "$bake_file"

}



if [[ "$TFS_COMPONENTS" == *"telemetry"* ]]; then

    echo "Configuring Telemetry Service..."
@@ -227,10 +272,63 @@ if [[ "$TFS_COMPONENTS" == *"telemetry"* ]]; then 
    sed -i "s|_LOAD_BALANCER_IP_|$LOAD_BALANCER_IP|g" manifests/telemetryservice.yaml

fi



if [ "$TFS_SKIP_BUILD" != "YES" ] && [ -n "$BUILDX_AVAILABLE" ]; then

    BAKE_FILE="${TMP_FOLDER}/${TFS_K8S_NAMESPACE}/bake.hcl"

    BAKE_LOG="$TMP_LOGS_FOLDER/buildx_bake.log"

    echo "Building Docker images in parallel with buildx bake..."



    BAKE_TARGETS=()

    for COMPONENT in $TFS_COMPONENTS; do

    echo "Processing '$COMPONENT' component..."

        if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then

            BAKE_TARGETS+=("${COMPONENT}-frontend" "${COMPONENT}-backend")

        elif [ "$COMPONENT" == "dlt" ]; then

            BAKE_TARGETS+=("dlt-connector" "dlt-gateway")

        else

            BAKE_TARGETS+=("${COMPONENT}")

        fi

    done



    {

        echo "group \"default\" {"

        echo "  targets = ["

        for TARGET in "${BAKE_TARGETS[@]}"; do

            echo "    \"${TARGET}\","

        done

        echo "  ]"

        echo "}"

        echo

    } > "$BAKE_FILE"



    if [ "$TFS_SKIP_BUILD" != "YES" ]; then

    for COMPONENT in $TFS_COMPONENTS; do

        if [ "$COMPONENT" == "ztp" ] || [ "$COMPONENT" == "policy" ]; then

            write_bake_target "$BAKE_FILE" "$COMPONENT" "./src/$COMPONENT" "./src/$COMPONENT/Dockerfile" "$(image_url "$COMPONENT")" ""

        elif [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then

            write_bake_target "$BAKE_FILE" "${COMPONENT}-frontend" "." "./src/$COMPONENT/frontend/Dockerfile" "$(image_url "${COMPONENT}-frontend")" ""

            write_bake_target "$BAKE_FILE" "${COMPONENT}-backend" "." "./src/$COMPONENT/backend/Dockerfile" "$(image_url "${COMPONENT}-backend")" ""

        elif [ "$COMPONENT" == "dlt" ]; then

            write_bake_target "$BAKE_FILE" "dlt-connector" "." "./src/dlt/connector/Dockerfile" "$(image_url "dlt-connector")" ""

            write_bake_target "$BAKE_FILE" "dlt-gateway" "." "./src/dlt/gateway/Dockerfile" "$(image_url "dlt-gateway")" ""

        else

            write_bake_target "$BAKE_FILE" "$COMPONENT" "." "./src/$COMPONENT/Dockerfile" "$(image_url "$COMPONENT")" ""

        fi

    done



    docker buildx bake -f "$BAKE_FILE" --push | tee "$BAKE_LOG"

    BAKE_EXIT_CODE=${PIPESTATUS[0]}

    if [[ $BAKE_EXIT_CODE != 0 ]]; then

        echo "  Buildx bake failed, exit code '${BAKE_EXIT_CODE}', exiting..."

        exit $BAKE_EXIT_CODE

    fi

    printf "\n"

elif [ "$TFS_SKIP_BUILD" != "YES" ]; then

    echo "Docker buildx bake is not available; falling back to sequential builds."

    printf "\n"

fi



for COMPONENT in $TFS_COMPONENTS; do

    echo "Preparing '$COMPONENT' component..."



    if [ "$TFS_SKIP_BUILD" != "YES" ] && [ -z "$BUILDX_AVAILABLE" ]; then

        echo "  Building Docker image..."

        BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
@@ -260,7 +358,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."



        if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ] ; then

            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

            IMAGE_URL=$(image_url "${COMPONENT}-frontend")



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"

            docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
@@ -268,7 +366,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"



            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

            IMAGE_URL=$(image_url "${COMPONENT}-backend")



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"

            docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
@@ -276,7 +374,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

        elif [ "$COMPONENT" == "dlt" ]; then

            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

            IMAGE_URL=$(image_url "${COMPONENT}-connector")



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"

            docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
@@ -284,7 +382,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"



            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

            IMAGE_URL=$(image_url "${COMPONENT}-gateway")



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"

            docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
@@ -292,7 +390,7 @@ for COMPONENT in $TFS_COMPONENTS; do 
            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

        else

            IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

            IMAGE_URL=$(image_url "$COMPONENT")



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"

            docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
@@ -311,32 +409,48 @@ for COMPONENT in $TFS_COMPONENTS; do 
    fi



    if [ "$COMPONENT" == "pathcomp" ] || [ "$COMPONENT" == "telemetry" ] || [ "$COMPONENT" == "analytics" ]; then

        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        IMAGE_URL=$(image_url "${COMPONENT}-frontend")

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4)

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"



        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        IMAGE_URL=$(image_url "${COMPONENT}-backend")

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f4)

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

    elif [ "$COMPONENT" == "dlt" ]; then

        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        IMAGE_URL=$(image_url "${COMPONENT}-connector")

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f4)

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"



        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        IMAGE_URL=$(image_url "${COMPONENT}-gateway")

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f4)

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

    else

        VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4)

        IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')

        IMAGE_URL=$(image_url "$COMPONENT")

        sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

    fi



    sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"



    # TODO: harmonize names of the monitoring component

    SERVICE_MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.services.yaml"

    awk 'BEGIN { RS="---"; ORS=""; } $0 ~ /kind:[[:space:]]*Service([[:space:]]|$)/ { print $0 "\n---\n" }' "$MANIFEST" > "$SERVICE_MANIFEST"



    printf "\n"

done



echo "Deploying services for selected components..."

for COMPONENT in $TFS_COMPONENTS; do

    SERVICE_MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.services.yaml"

    if [ -s "$SERVICE_MANIFEST" ]; then

        DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}_services.log"

        kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$SERVICE_MANIFEST" > "$DEPLOY_LOG"

    fi

done

printf "\n"



for COMPONENT in $TFS_COMPONENTS; do

    echo "  Deploying '$COMPONENT' component to Kubernetes..."

    MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"

    DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"

    kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"

    COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/g")