Mon Engine monitoring of expected core, dep & sandbox pods (96c013a0) · Commits · MEC - Multi-access Edge Computing / ETSI MEC Sandbox

charts/meep-mon-engine/templates/deployment.yaml

+5 −0

Original line number	Diff line number	Diff line
		@@ -39,6 +39,11 @@ spec:
		ports:
		- containerPort: {{ .Values.deployment.port }}
		protocol: {{ .Values.deployment.protocol }}
		env:
		{{- range $key, $value := .Values.image.env }}
		- name: {{ $key }}
		value: {{ $value }}
		{{- end }}
		{{- if .Values.codecov.enabled}}
		volumeMounts:
		- name: codecov-storage

charts/meep-mon-engine/values.yaml

+4 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,10 @@ image:
		repository: meep-mon-engine
		tag: latest
		pullPolicy: Always
		env:
		MEEP_DEPENDENCY_PODS: ""
		MEEP_CORE_PODS: ""
		MEEP_SANDBOX_PODS: "meep-loc-serv,meep-test"

		service:
		type: ClusterIP

go-apps/meep-mon-engine/go.mod

+4 −3

Original line number	Diff line number	Diff line
		@@ -5,8 +5,9 @@ go 1.12
		require (
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-data-key-mgr v0.0.0
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger v0.0.0
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model v0.0.0 // indirect
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mq v0.0.0
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis v0.0.0
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-sandbox-store v0.0.0
		github.com/gogo/protobuf v1.2.1 // indirect
		github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
		github.com/google/btree v1.0.0 // indirect
		@@ -35,8 +36,8 @@ require (

		replace (
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-data-key-mgr => ../../go-packages/meep-data-key-mgr
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-data-model => ../../go-packages/meep-data-model
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger => ../../go-packages/meep-logger
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model => ../../go-packages/meep-model
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mq => ../../go-packages/meep-mq
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis => ../../go-packages/meep-redis
		github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-sandbox-store => ../../go-packages/meep-sandbox-store
		)

go-apps/meep-mon-engine/go.sum

+0 −6

Original line number	Diff line number	Diff line
		@@ -2,10 +2,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
		github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db/go.mod h1:RU+6d0CNIRSp6yo1mXLIIrnFa/3LHhvcDVLVJyovptM=
		github.com/KromDaniel/rejonson v0.0.0-20180822072824-00b5bcf2b351 h1:1u1XrfCBnY+GijnyU6O1k4odp5TnqZQTsp5v7+n/E4Y=
		github.com/KromDaniel/rejonson v0.0.0-20180822072824-00b5bcf2b351/go.mod h1:HxwfbuElTuGf+/uKZfjJrCnv0BmmpkPJDI7gBwj1KkM=
		github.com/RyanCarrier/dijkstra v0.0.0-20190726134004-b51cadb5ae52/go.mod h1:DdR6ymcLl8+sN/XOVNjnYO1NDYfgHskGjreZUDuQCTY=
		github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30=
		github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw=
		github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
		github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
		github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
		github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
		@@ -37,7 +33,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
		github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
		github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
		github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
		github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA=
		github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
		github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
		github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
		@@ -88,7 +83,6 @@ k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93 h1:tT6oQBi0qwLbbZSfDkdIsb
		k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
		k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34=
		k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
		k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
		k8s.io/klog v0.0.0-20181108234604-8139d8cb77af h1:s6rm8OxBbyDNSRkpyAd5OL4icUdBICVw9+mFADa+t5E=
		k8s.io/klog v0.0.0-20181108234604-8139d8cb77af/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
		sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=

go-apps/meep-mon-engine/server/mon-engine.go

+351 −167

Original line number	Diff line number	Diff line
		@@ -20,12 +20,15 @@ import (
		"encoding/json"
		"fmt"
		"net/http"
		"os"
		"strings"

		dkm "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-data-key-mgr"
		log "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger"
		mq "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mq"
		redis "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis"
		ss "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-sandbox-store"
		v1 "k8s.io/api/core/v1"
		metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

		"k8s.io/apimachinery/pkg/fields"
		"k8s.io/client-go/kubernetes"
		@@ -33,24 +36,19 @@ import (
		"k8s.io/client-go/tools/cache"
		)

		const notFoundStr string = "na"
		const monEngineKey string = "mon-engine:"

		var baseKey string = dkm.GetKeyRootGlobal() + monEngineKey

		//index in array
		const EVENT_POD_ADDED = 0
		const EVENT_POD_MODIFIED = 1
		const EVENT_POD_DELETED = 2

		var pod_event_str = [3]string{"pod added", "pod modified", "pod deleted"}
		type UserData struct {
		AllPodsStatus PodsStatus
		ExpectedPods map[string]*PodStatus
		}

		type MonEngineInfo struct {
		PodType string
		PodName string
		Namespace string
		MeepApp string
		MeepOrigin string
		MeepScenario string
		Release string
		Phase string
		PodInitialized string
		PodReady string
		@@ -65,15 +63,89 @@ type MonEngineInfo struct {
		StartTime string
		}

		const moduleName = "meep-mon-engine"
		const moduleNamespace = "default"
		const notFoundStr = "na"
		const monEngineKey = "mon-engine:"

		// MQ payload fields
		const fieldSandboxName = "sandbox-name"

		// index in array
		const EVENT_POD_ADDED = 0
		const EVENT_POD_MODIFIED = 1
		const EVENT_POD_DELETED = 2

		var pod_event_str = [3]string{"pod added", "pod modified", "pod deleted"}
		var rc *redis.Connector
		var redisDBAddr = "meep-redis-master:6379"
		var redisAddr = "meep-redis-master:6379"
		var baseKey string = dkm.GetKeyRootGlobal() + monEngineKey
		var stopChan = make(chan struct{})
		var mqGlobal *mq.MsgQueue
		var handlerId int
		var sandboxStore *ss.SandboxStore

		var depPodsList []string
		var corePodsList []string
		var sboxPodsList []string

		var expectedDepPods map[string]*PodStatus
		var expectedCorePods map[string]*PodStatus
		var expectedSboxPods map[string]*PodStatus

		// Init - Mon Engine initialization
		func Init() (err error) {

		// Retrieve dependency pod list from environment variable
		depPodsStr := strings.TrimSpace(os.Getenv("MEEP_DEPENDENCY_PODS"))
		log.Info("MEEP_DEPENDENCY_PODS: ", depPodsStr)
		if depPodsStr != "" {
		depPodsList = strings.Split(depPodsStr, ",")
		expectedDepPods = make(map[string]*PodStatus)
		for _, pod := range depPodsList {
		podStatus := new(PodStatus)
		podStatus.PodType = "core"
		podStatus.Sandbox = "default"
		podStatus.Name = pod
		podStatus.LogicalState = "NotAvailable"
		expectedDepPods[pod] = podStatus
		}
		}

		// Retrieve core pod list from environment variable
		corePodsStr := strings.TrimSpace(os.Getenv("MEEP_CORE_PODS"))
		log.Info("MEEP_CORE_PODS: ", corePodsStr)
		if corePodsStr != "" {
		corePodsList = strings.Split(corePodsStr, ",")
		expectedCorePods = make(map[string]*PodStatus)
		for _, pod := range corePodsList {
		podStatus := new(PodStatus)
		podStatus.PodType = "core"
		podStatus.Sandbox = "default"
		podStatus.Name = pod
		podStatus.LogicalState = "NotAvailable"
		expectedCorePods[pod] = podStatus
		}
		}

		// Retrieve sandbox pod list from environment variable
		sboxPodsStr := strings.TrimSpace(os.Getenv("MEEP_SANDBOX_PODS"))
		log.Info("MEEP_SANDBOX_PODS: ", sboxPodsStr)
		if sboxPodsStr != "" {
		sboxPodsList = strings.Split(sboxPodsStr, ",")
		expectedSboxPods = make(map[string]*PodStatus)
		}

		// Create message queue
		mqGlobal, err = mq.NewMsgQueue(mq.GetGlobalName(), moduleName, moduleNamespace, redisAddr)
		if err != nil {
		log.Error("Failed to create Message Queue with error: ", err)
		return err
		}
		log.Info("Message Queue created")

		// Connect to Redis DB
		rc, err = redis.NewConnector(redisDBAddr, 0)
		rc, err = redis.NewConnector(redisAddr, 0)
		if err != nil {
		log.Error("Failed connection to Redis: ", err)
		return err
		@@ -83,12 +155,35 @@ func Init() (err error) {
		// Empty DB
		_ = rc.DBFlush(baseKey)

		// Connect to Sandbox Store
		sandboxStore, err = ss.NewSandboxStore(redisAddr)
		if err != nil {
		log.Error("Failed connection to Sandbox Store: ", err.Error())
		return err
		}
		log.Info("Connected to Sandbox Store")

		return nil
		}

		// Run - Mon Engine monitoring thread
		func Run() (err error) {

		// Initialize expected pods for existing sandboxes
		if sboxMap, err := sandboxStore.GetAll(); err == nil {
		for _, sbox := range sboxMap {
		addExpectedPods(sbox.Name)
		}
		}

		// Register Message Queue handler
		handler := mq.MsgHandler{Handler: msgHandler, UserData: nil}
		handlerId, err = mqGlobal.RegisterHandler(handler)
		if err != nil {
		log.Error("Failed to register MsgQueue handler: ", err.Error())
		return err
		}

		// Start thread to watch k8s pods
		err = k8sConnect()
		if err != nil {
		@@ -103,6 +198,20 @@ func Stop() {
		close(stopChan)
		}

		// Message Queue handler
		func msgHandler(msg *mq.Msg, userData interface{}) {
		switch msg.Message {
		case mq.MsgSandboxCreate:
		log.Debug("RX MSG: ", mq.PrintMsg(msg))
		addExpectedPods(msg.Payload[fieldSandboxName])
		case mq.MsgSandboxDestroy:
		log.Debug("RX MSG: ", mq.PrintMsg(msg))
		removeExpectedPods(msg.Payload[fieldSandboxName])
		default:
		log.Trace("Ignoring unsupported message: ", mq.PrintMsg(msg))
		}
		}

		func connectToAPISvr() (*kubernetes.Clientset, error) {

		// Create the in-cluster config
		@@ -123,11 +232,13 @@ func connectToAPISvr() (*kubernetes.Clientset, error) {
		func printfMonEngineInfo(monEngineInfo MonEngineInfo, reason int) {

		log.Debug("Monitoring Engine info * ", pod_event_str[reason], " * ",
		" pod name : ", monEngineInfo.PodName,
		" podType : ", monEngineInfo.PodType,
		" podName : ", monEngineInfo.PodName,
		" namespace : ", monEngineInfo.Namespace,
		" meepApp : ", monEngineInfo.MeepApp,
		" meepOrigin : ", monEngineInfo.MeepOrigin,
		" meepScenario : ", monEngineInfo.MeepScenario,
		" release : ", monEngineInfo.Release,
		" phase : ", monEngineInfo.Phase,
		" podInitialized : ", monEngineInfo.PodInitialized,
		" podUnschedulable : ", monEngineInfo.PodUnschedulable,
		@@ -143,7 +254,13 @@ func printfMonEngineInfo(monEngineInfo MonEngineInfo, reason int) {
		}

		func processEvent(obj interface{}, reason int) {
		if pod, ok := obj.(*v1.Pod); ok {
		var ok bool
		var pod *v1.Pod

		// Validate object type is Pod
		if pod, ok = obj.(*v1.Pod); !ok {
		return
		}

		var monEngineInfo MonEngineInfo

		@@ -177,15 +294,11 @@ func processEvent(obj interface{}, reason int) {
		for i := 0; i < nbContainers; i++ {
		if pod.Status.ContainerStatuses[i].Ready {
		okContainers++
		} else {
		if pod.Status.ContainerStatuses[i].State.Waiting != nil {
		} else if pod.Status.ContainerStatuses[i].State.Waiting != nil {
		reasonFailureStr = pod.Status.ContainerStatuses[i].State.Waiting.Reason
		} else if pod.Status.ContainerStatuses[i].State.Terminated != nil {
		if reasonFailureStr != "" {
		} else if pod.Status.ContainerStatuses[i].State.Terminated != nil && reasonFailureStr != "" {
		reasonFailureStr = pod.Status.ContainerStatuses[i].State.Terminated.Reason
		}
		}
		}
		//only update if the value is greater than 0, and we keep it
		if restartCount == 0 {
		restartCount = int(pod.Status.ContainerStatuses[i].RestartCount)
		@@ -208,59 +321,58 @@ func processEvent(obj interface{}, reason int) {
		monEngineInfo.PodName = pod.Name
		monEngineInfo.Namespace = pod.Namespace
		monEngineInfo.MeepApp = pod.Labels["meepApp"]
		monEngineInfo.MeepOrigin = pod.Labels["meepOrigin"]
		monEngineInfo.MeepScenario = pod.Labels["meepScenario"]
		if pod.Labels["meepApp"] != "" {
		monEngineInfo.MeepApp = pod.Labels["meepApp"]
		} else {
		if monEngineInfo.Release, ok = pod.Labels["release"]; !ok {
		monEngineInfo.Release = notFoundStr
		}
		if monEngineInfo.MeepApp, ok = pod.Labels["meepApp"]; !ok {
		monEngineInfo.MeepApp = notFoundStr
		}
		if pod.Labels["meepOrigin"] != "" {
		monEngineInfo.MeepOrigin = pod.Labels["meepOrigin"]
		} else {
		if monEngineInfo.MeepOrigin, ok = pod.Labels["meepOrigin"]; !ok {
		monEngineInfo.MeepOrigin = notFoundStr
		}
		if pod.Labels["meepScenario"] != "" {
		monEngineInfo.MeepScenario = pod.Labels["meepScenario"]
		} else {
		if monEngineInfo.MeepScenario, ok = pod.Labels["meepScenario"]; !ok {
		monEngineInfo.MeepScenario = notFoundStr
		}
		monEngineInfo.LogicalState = monEngineInfo.Phase
		monEngineInfo.PodType = getPodType(monEngineInfo.MeepOrigin, monEngineInfo.Release)

		//Phase is Running but might not really be because of some other attributes
		//start of override section of the LogicalState by specific conditions

		if pod.GetObjectMeta().GetDeletionTimestamp() != nil {
		monEngineInfo.LogicalState = "Terminating"
		} else {
		if monEngineInfo.PodReady != "True" {
		} else if monEngineInfo.PodReady != "True" {
		monEngineInfo.LogicalState = "Pending"
		} else {
		if monEngineInfo.NbOkContainers < monEngineInfo.NbTotalContainers {
		} else if monEngineInfo.NbOkContainers < monEngineInfo.NbTotalContainers {
		monEngineInfo.LogicalState = "Failed"
		}
		}
		}
		//end of override section

		printfMonEngineInfo(monEngineInfo, reason)

		// Add, update or remove entry in DB only if core or scenario pod
		if monEngineInfo.PodType != notFoundStr {
		if reason == EVENT_POD_DELETED {
		deleteEntryInDB(monEngineInfo)
		} else {
		addOrUpdateEntryInDB(monEngineInfo)
		}
		} else {
		log.Debug("Ignoring non-AdvantEDGE pod: ", monEngineInfo.PodName)
		}
		}

		func addOrUpdateEntryInDB(monEngineInfo MonEngineInfo) {
		// Populate rule fields
		fields := make(map[string]interface{})
		fields["type"] = monEngineInfo.PodType
		fields["name"] = monEngineInfo.PodName
		fields["namespace"] = monEngineInfo.Namespace
		fields["meepApp"] = monEngineInfo.MeepApp
		fields["meepOrigin"] = monEngineInfo.MeepOrigin
		fields["meepScenario"] = monEngineInfo.MeepScenario
		fields["release"] = monEngineInfo.Release
		fields["phase"] = monEngineInfo.Phase
		fields["initialised"] = monEngineInfo.PodInitialized
		fields["scheduled"] = monEngineInfo.PodScheduled
		@@ -274,7 +386,7 @@ func addOrUpdateEntryInDB(monEngineInfo MonEngineInfo) {
		fields["startTime"] = monEngineInfo.StartTime

		// Make unique key
		key := baseKey + monEngineInfo.MeepOrigin + ":" + monEngineInfo.Namespace + ":" + monEngineInfo.MeepScenario + ":" + monEngineInfo.MeepApp + ":" + monEngineInfo.PodName
		key := baseKey + monEngineInfo.Namespace + ":" + monEngineInfo.PodType + ":" + monEngineInfo.PodName

		// Set rule information in DB
		err := rc.SetEntry(key, fields)
		@@ -286,7 +398,7 @@ func addOrUpdateEntryInDB(monEngineInfo MonEngineInfo) {
		func deleteEntryInDB(monEngineInfo MonEngineInfo) {

		// Make unique key
		key := baseKey + monEngineInfo.MeepOrigin + ":" + monEngineInfo.Namespace + ":" + monEngineInfo.MeepScenario + ":" + monEngineInfo.MeepApp + ":" + monEngineInfo.PodName
		key := baseKey + monEngineInfo.Namespace + ":" + monEngineInfo.PodType + ":" + monEngineInfo.PodName

		// Set rule information in DB
		err := rc.DelEntry(key)
		@@ -304,23 +416,6 @@ func k8sConnect() (err error) {
		return err
		}

		meepOrigin := "core"

		// Retrieve pods from k8s api with scenario label
		pods, err := clientset.CoreV1().Pods("").List(
		metav1.ListOptions{LabelSelector: fmt.Sprintf("meepOrigin=%s", meepOrigin)})
		if err != nil {
		log.Error("Failed to retrieve services from k8s API Server. Error: ", err)
		return err
		}

		// Log currently installed core pods
		for _, pod := range pods.Items {
		podName := pod.ObjectMeta.Name
		podPhase := pod.Status.Phase
		log.Debug("podName: ", podName, " podPhase: ", podPhase)
		}

		watchlist := cache.NewListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", v1.NamespaceAll, fields.Everything())

		// also take a look at NewSharedIndexInformer
		@@ -348,8 +443,8 @@ func k8sConnect() (err error) {
		// Retrieve POD states
		// GET /states
		func meGetStates(w http.ResponseWriter, r *http.Request) {
		var allPodsStatus PodsStatus
		var filteredPodsStatus PodsStatus
		var err error
		var data UserData

		// Retrieve query parameters
		query := r.URL.Query()
		@@ -357,13 +452,50 @@ func meGetStates(w http.ResponseWriter, r *http.Request) {
		querySandbox := query.Get("sandbox")
		queryLong := query.Get("long")

		// Retrieve pod status information
		var err error
		keyName := baseKey + "*"
		// Get expected pods list
		data.ExpectedPods = make(map[string]*PodStatus)
		if queryType != "scenario" {
		if querySandbox == "" \|\| querySandbox == "all" {
		for k, v := range expectedCorePods {
		data.ExpectedPods[k] = v
		}
		for k, v := range expectedDepPods {
		data.ExpectedPods[k] = v
		}
		}
		if querySandbox != "" \|\| querySandbox == "all" {
		for k, v := range expectedSboxPods {
		if v.Sandbox == querySandbox \|\| querySandbox == "all" {
		data.ExpectedPods[k] = v
		}
		}
		}
		}

		// Create DB key using query filters
		sandboxKey := ""
		if querySandbox == "" {
		sandboxKey = "default:"
		} else if querySandbox == "all" {
		sandboxKey = "*:"
		} else {
		sandboxKey = querySandbox + ":"
		}

		typeKey := ""
		if queryType != "" {
		typeKey = queryType + ":"
		} else {
		typeKey = "*"
		}

		keyName := baseKey + sandboxKey + typeKey + "*"

		// Retrieve pod status information from DB
		if queryLong == "true" {
		err = rc.ForEachEntry(keyName, getPodDetails, &allPodsStatus)
		err = rc.ForEachEntry(keyName, getPodDetails, &data)
		} else {
		err = rc.ForEachEntry(keyName, getPodStatesOnly, &allPodsStatus)
		err = rc.ForEachEntry(keyName, getPodStatesOnly, &data)
		}
		if err != nil {
		log.Error(err.Error())
		@@ -371,29 +503,15 @@ func meGetStates(w http.ResponseWriter, r *http.Request) {
		return
		}

		// Filter results based on query parameters
		for _, podStatus := range allPodsStatus.PodStatus {

		// Filter on pod type
		if (podStatus.PodType == notFoundStr) \|\|
		(queryType == "core" && podStatus.PodType != "core") \|\|
		(queryType == "scenario" && podStatus.PodType != "scenario") {
		continue
		}

		// Filter on sandbox
		if (querySandbox == "" && podStatus.Sandbox != "default") \|\|
		(querySandbox != "" && querySandbox != "all" && querySandbox != podStatus.Sandbox) {
		continue
		}

		filteredPodsStatus.PodStatus = append(filteredPodsStatus.PodStatus, podStatus)
		// Add missing pods status
		for _, podStatus := range data.ExpectedPods {
		data.AllPodsStatus.PodStatus = append(data.AllPodsStatus.PodStatus, *podStatus)
		}

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")

		// Format response
		jsonResponse, err := json.Marshal(filteredPodsStatus)
		jsonResponse, err := json.Marshal(data.AllPodsStatus)
		if err != nil {
		log.Error(err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		@@ -406,15 +524,13 @@ func meGetStates(w http.ResponseWriter, r *http.Request) {
		}

		func getPodDetails(key string, fields map[string]string, userData interface{}) error {
		podsStatus := userData.(*PodsStatus)
		data := userData.(*UserData)

		// Append pod status
		var podStatus PodStatus
		podStatus.PodType = fields["meepOrigin"]
		podStatus.PodType = fields["type"]
		podStatus.Sandbox = fields["namespace"]
		if fields["meepApp"] != notFoundStr {
		podStatus.Name = fields["meepApp"]
		} else {
		podStatus.Name = fields["name"]
		}
		podStatus.Name = getPodName(fields["meepApp"], fields["name"])
		podStatus.Namespace = fields["namespace"]
		podStatus.MeepApp = fields["meepApp"]
		podStatus.MeepOrigin = fields["meepOrigin"]
		@@ -430,22 +546,90 @@ func getPodDetails(key string, fields map[string]string, userData interface{}) e
		podStatus.NbPodRestart = fields["nbPodRestart"]
		podStatus.LogicalState = fields["logicalState"]
		podStatus.StartTime = fields["startTime"]
		data.AllPodsStatus.PodStatus = append(data.AllPodsStatus.PodStatus, podStatus)

		// Remove from expected pods
		delete(data.ExpectedPods, fields["release"])

		podsStatus.PodStatus = append(podsStatus.PodStatus, podStatus)
		return nil
		}

		func getPodStatesOnly(key string, fields map[string]string, userData interface{}) error {
		podsStatus := userData.(*PodsStatus)
		data := userData.(*UserData)

		// Append pod status
		var podStatus PodStatus
		podStatus.PodType = fields["meepOrigin"]
		podStatus.PodType = fields["type"]
		podStatus.Sandbox = fields["namespace"]
		if fields["meepApp"] != notFoundStr {
		podStatus.Name = fields["meepApp"]
		} else {
		podStatus.Name = fields["name"]
		}
		podStatus.Name = getPodName(fields["meepApp"], fields["name"])
		podStatus.LogicalState = fields["logicalState"]
		podsStatus.PodStatus = append(podsStatus.PodStatus, podStatus)
		data.AllPodsStatus.PodStatus = append(data.AllPodsStatus.PodStatus, podStatus)

		// Remove from expected pods
		delete(data.ExpectedPods, fields["release"])

		return nil
		}

		func getPodType(origin string, release string) string {
		podType := notFoundStr
		if origin == "core" \|\| origin == "scenario" {
		podType = origin
		} else if release != notFoundStr {
		if _, ok := expectedDepPods[release]; ok {
		podType = "core"
		} else if _, ok := expectedCorePods[release]; ok {
		podType = "core"
		}
		}
		return podType
		}

		func getPodName(app string, name string) string {
		var podName string
		if app != notFoundStr {
		podName = app
		} else {
		podName = name
		}
		return podName
		}

		func addExpectedPods(sandboxName string) {
		for _, pod := range sboxPodsList {
		// Get sandbox-specific pod name
		var podName string
		prefix := "meep-"
		sandboxPrefix := prefix + sandboxName + "-"
		if strings.HasPrefix(pod, prefix) {
		podName = sandboxPrefix + pod[len(prefix):]
		} else {
		podName = sandboxPrefix + pod
		}

		// Add to expected sandbox pods list
		podStatus := new(PodStatus)
		podStatus.PodType = "core"
		podStatus.Sandbox = sandboxName
		podStatus.Name = podName
		podStatus.LogicalState = "NotAvailable"
		expectedSboxPods[podName] = podStatus
		}
		}

		func removeExpectedPods(sandboxName string) {
		for _, pod := range sboxPodsList {
		// Get sandbox-specific pod name
		var podName string
		prefix := "meep-"
		sandboxPrefix := prefix + sandboxName + "-"
		if strings.HasPrefix(pod, prefix) {
		podName = sandboxPrefix + pod[len(prefix):]
		} else {
		podName = sandboxPrefix + pod
		}

		// Delete from expected list
		delete(expectedSboxPods, podName)
		}
		}