Commit da5d3289 authored by Kevin Di Lallo's avatar Kevin Di Lallo
Browse files

lifecycle bug fixes + increased prometheus retention duration to 30d

parent 6ff758f2

charts/kube-prometheus-stack/values.yaml

+1 −1
Original line number Diff line number Diff line
@@ -1835,7 +1835,7 @@ prometheus: 


    ## How long to retain metrics

    ##

    retention: 10d

    retention: 30d



    ## Maximum size of metrics

    ##

go-apps/meep-mg-manager/main.go

+1 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ import ( 
)



func init() {

	log.MeepJSONLogInit("meep-mg-manager")

	log.MeepTextLogInit("meep-mg-manager")

}



func main() {

go-apps/meep-mg-manager/server/mg-manager.go

+8 −2
Original line number Diff line number Diff line
@@ -135,6 +135,7 @@ type lbRulesStore struct { 


type MgManager struct {

	sandboxName  string

	scenarioName string

	baseKey      string

	mqLocal      *mq.MsgQueue

	handlerId    int
@@ -258,13 +259,18 @@ func msgHandler(msg *mq.Msg, userData interface{}) { 
}



func processActiveScenarioUpdate() {



	// Sync with active scenario store

	mgm.activeModel.UpdateScenario()



	_ = httpLog.ReInit(moduleName, mgm.sandboxName, mgm.activeModel.GetScenarioName(), redisAddr, influxAddr)

	scenarioName := mgm.activeModel.GetScenarioName()

	if mgm.scenarioName != scenarioName {

		mgm.scenarioName = scenarioName

		_ = httpLog.ReInit(moduleName, mgm.sandboxName, scenarioName, redisAddr, influxAddr)

	}



	// Handle empty/missing scenario

	if mgm.activeModel.GetScenarioName() == "" {

	if scenarioName == "" {

		clearScenario()

		return

	}

go-apps/meep-tc-engine/tc-engine.go

+10 −6
Original line number Diff line number Diff line
@@ -339,17 +339,19 @@ func processScenarioUpdate(eventType string) { 
		return

	}



	// Refresh NC rules

	refreshNcRules()



	// Trigger IP address refresh if necessary

	if eventType == mod.EventAddNode || eventType == mod.EventModifyNode || eventType == mod.EventRemoveNode {

		tce.ipManager.Refresh()

	}



	// Refresh NC rules

	refreshNcRules()



		// Refresh routing rules

		// NOTE: This operation is long in sidecars and should be avoided unless necessary.

		//       E.g. when ingress/egress rules or a group servicerules may have changed.

		tce.routingEngine.RefreshLbRules()

	}

}



func processScenarioTerminate() {

	// Sync with active scenario store
@@ -521,6 +523,7 @@ func processScenario(model *mod.Model) error { 
func ipAddrUpdated() {

	mutex.Lock()

	defer mutex.Unlock()

	log.Debug("ipAddrUpdated")



	// Update cached IP addresses

	updateIpAddresses()
@@ -629,6 +632,7 @@ func netCharUpdate(dstName string, srcName string, rate float64, latency float64 
func updateComplete() {

	mutex.Lock()

	defer mutex.Unlock()

	log.Debug("updateComplete")



	// Inform sidecars of NC rule updates

	publishNcRulesUpdate()

go-apps/meep-tc-sidecar/main.go

+8 −3
Original line number Diff line number Diff line
@@ -133,7 +133,7 @@ var metricsBaseKey string 
var nbAppliedOperations = 0



var opts Opts = Opts{

	timeout:         100000 * time.Millisecond,

	timeout:         2000 * time.Millisecond,

	interval:        1000 * time.Millisecond,

	trafficInterval: 100 * time.Millisecond,

	bind4:           "0.0.0.0",
@@ -333,13 +333,15 @@ func refreshNcRules() { 
	deleteUnusedIfbs()



	elapsed := time.Since(currentTime)

	log.Debug("RefreshNetCharRules execution time for ", nbAppliedOperations, " updates, elapsed time: ", elapsed)

	log.Debug("refreshNcRules: execution time for ", nbAppliedOperations, " updates, elapsed time: ", elapsed)



	// Refresh ping destinations

	refreshPingDests()

}



func refreshLbRules() {

	currentTime := time.Now()



	// Get currently installed chains in NAT table

	log.Debug("Fetching nat table chains")

	chains, err := ipTbl.ListChains("nat")
@@ -463,6 +465,9 @@ func refreshLbRules() { 
	if flushRequired {

		flushTrackedConnections()

	}



	elapsed := time.Since(currentTime)

	log.Debug("refreshLbRules: execution time: ", elapsed)

}



func flushTrackedConnections() {
@@ -800,7 +805,7 @@ func createFiltersHandler(key string, fields map[string]string, userData interfa 
		}

	} else {

		// Updated - only handle cases where IPs have changed

		if srcIp != prevIps.PodIp && srcSvcIp == prevIps.SvcIp {

		if srcIp != prevIps.PodIp || srcSvcIp != prevIps.SvcIp {

			if srcIp == ipAddrNone {

				// Filters can only exist if pod IP is valid

				_ = cmdDeleteFilter(filterNumber)