Commit d0b0bb86 authored by Muhammad Umair Khan's avatar Muhammad Umair Khan
Browse files

Add clusterrolebinding cleanup, sequential deployments, and custom priority... 

Add clusterrolebinding cleanup, sequential deployments, and custom priority sorting to meep-virt-engine

- Clean up orphaned ClusterRoleBindings prefixed with <sandboxName>: during sandbox creation and destruction to avoid deployment name collisions.
- Use Helm's --wait flag during chart installation to ensure pods start up sequentially (one-by-one) rather than concurrently. This prevents network congestion and CPU spikes.
- Implement custom priority sorting for scenario chart deployments. Ensure MQTT brokers and mosquitto services deploy first of all globally (score 10) to prevent connection timeouts in API containers, followed by MEC services of each MEP in logical numeric order, and finally non-MEP/MEC scenario pods.
parent 6be19fb7

go-apps/meep-virt-engine/helm/install.go

+4 −2
Original line number Diff line number Diff line
@@ -73,7 +73,8 @@ func install(chart Chart) error { 
			chart.Location, "--replace", "--disable-openapi-validation",

			"--set", "codecov.enabled="+codecovEnabled,

			"--set", "codecov.location="+codecovLocation,

			"--set", "image.env.MEEP_CODECOV="+codecovEnabled) /*,

			"--set", "image.env.MEEP_CODECOV="+codecovEnabled,

			"--wait") /*,

		"--set", "onboardedapp.enabled="+onboardedappEnabled,

		"--set", "onboardedapp.location="+onboardedappLocation)*/

	} else {
@@ -82,7 +83,8 @@ func install(chart Chart) error { 
			"--set", "nameOverride="+chart.Name,

			"--set", "fullnameOverride="+chart.Name,

			"-f", chart.ValuesFile,

			chart.Location, "--replace")

			chart.Location, "--replace",

			"--wait")

	}

	out, err := cmd.CombinedOutput()

	if err != nil {

go-apps/meep-virt-engine/server/chart-template.go

+49 −0
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ import ( 
	"errors"

	"os"

	"path/filepath"

	"sort"

	"strconv"

	"strings"

	"text/template"
@@ -455,6 +456,18 @@ func generateScenarioCharts(sandboxName string, procName string, model *mod.Mode 
		}

	}



	// Sort charts by priority score

	sort.Slice(charts, func(i, j int) bool {

		cI := charts[i]

		cJ := charts[j]

		scoreI := getChartPriorityScore(cI.Name)

		scoreJ := getChartPriorityScore(cJ.Name)

		if scoreI != scoreJ {

			return scoreI < scoreJ

		}

		return cI.Name < cJ.Name

	})



	return charts, nil

}
@@ -743,3 +756,39 @@ func getSanboxChartTemplates() (templates []string, err error) { 
	templates, _ = file.Readdirnames(0)

	return templates, nil

}



func getChartPriorityScore(name string) int {

	if strings.Contains(name, "mosquitto") || strings.Contains(name, "broker") {

		return 10

	}



	if strings.HasPrefix(name, "mep") {

		parts := strings.Split(name, "-")

		if len(parts) >= 2 {

			mepPart := parts[0]

			mepNum := 1

			if len(mepPart) > 3 {

				if num, err := strconv.Atoi(mepPart[3:]); err == nil {

					mepNum = num

				}

			}



			mecNum := -1

			for _, part := range parts {

				if strings.HasPrefix(part, "mec") && len(part) > 3 {

					if num, err := strconv.Atoi(part[3:]); err == nil {

						mecNum = num

						break

					}

				}

			}



			if mecNum != -1 {

				return mepNum*1000 + mecNum

			} else {

				return mepNum*1000 + 1

			}

		}

	}

	return 100000

}

go-apps/meep-virt-engine/server/virt-engine.go

+28 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ package server 
import (

	"errors"

	"os"

	"os/exec"

	"strconv"

	"strings"
@@ -386,6 +387,9 @@ func terminateScenario(sandboxName string) { 
func createSandbox(sandboxName string) {

	var err error



	// Clean up any leftover cluster role bindings first

	cleanUpClusterRoleBindings(sandboxName)



	// Create new Model instance

	modelCfg := mod.ModelCfg{

		Name:      moduleName,
@@ -415,6 +419,9 @@ func destroySandbox(sandboxName string) { 
	ve.activeScenarioNames[sandboxName] = ""

	ve.activeModels[sandboxName] = nil



	// Clean up any leftover cluster role bindings

	cleanUpClusterRoleBindings(sandboxName)



	// ticker := time.NewTicker(retryTimerDuration * time.Millisecond)



	// go func() {
@@ -485,3 +492,24 @@ func deleteReleases(sandboxName string, scenarioName string, procName string) (e 
	}

	return err, chartsToDelete

}



func cleanUpClusterRoleBindings(sandboxName string) {

	log.Info("Cleaning up ClusterRoleBindings for sandbox: ", sandboxName)

	cmd := exec.Command("kubectl", "get", "clusterrolebindings", "-o", "name")

	out, err := cmd.Output()

	if err != nil {

		log.Error("Failed to get clusterrolebindings: ", err.Error())

		return

	}



	lines := strings.Split(string(out), "\n")

	prefix := "clusterrolebinding.rbac.authorization.k8s.io/" + sandboxName + ":"

	for _, line := range lines {

		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, prefix) {

			log.Info("Deleting leftover clusterrolebinding: ", line)

			deleteCmd := exec.Command("kubectl", "delete", line)

			_ = deleteCmd.Run()

		}

	}

}