Merge pull request #236 from dilallkx/kd_sp17_dev_perf (96202d39) · Commits · MEC - Multi-access Edge Computing / ETSI MEC Sandbox

charts/grafana/dashboards/platform-advantedge.json

+280 −15

Original line number	Diff line number	Diff line
		@@ -15,7 +15,8 @@
		"editable": true,
		"gnetId": null,
		"graphTooltip": 0,
		"iteration": 1613495656791,
		"id": 17,
		"iteration": 1618533539997,
		"links": [],
		"panels": [
		{
		@@ -27,6 +28,270 @@
		"x": 0,
		"y": 0
		},
		"id": 91,
		"panels": [],
		"title": "Sandbox Metrics",
		"type": "row"
		},
		{
		"aliasColors": {},
		"bars": false,
		"dashLength": 10,
		"dashes": false,
		"datasource": "meep-influxdb",
		"decimals": 2,
		"fieldConfig": {
		"defaults": {
		"custom": {},
		"displayName": "Sbox Creation Time",
		"unit": "s"
		},
		"overrides": []
		},
		"fill": 1,
		"fillGradient": 7,
		"gridPos": {
		"h": 7,
		"w": 10,
		"x": 0,
		"y": 1
		},
		"hiddenSeries": false,
		"id": 93,
		"interval": "",
		"legend": {
		"alignAsTable": true,
		"avg": true,
		"current": false,
		"hideEmpty": false,
		"hideZero": false,
		"max": true,
		"min": true,
		"rightSide": false,
		"show": true,
		"total": false,
		"values": true
		},
		"lines": false,
		"linewidth": 2,
		"nullPointMode": "null",
		"options": {
		"alertThreshold": true
		},
		"percentage": false,
		"pluginVersion": "7.3.5",
		"pointradius": 2,
		"points": true,
		"renderer": "flot",
		"seriesOverrides": [],
		"spaceLength": 10,
		"stack": false,
		"steppedLine": false,
		"targets": [
		{
		"alias": "",
		"groupBy": [
		{
		"params": [
		"$__interval"
		],
		"type": "time"
		},
		{
		"params": [
		"null"
		],
		"type": "fill"
		}
		],
		"hide": false,
		"orderByTime": "ASC",
		"policy": "default",
		"query": "SELECT createtime FROM global_sandbox_metrics.autogen.sbox WHERE $timeFilter",
		"rawQuery": true,
		"refId": "A",
		"resultFormat": "time_series",
		"select": [
		[
		{
		"params": [
		"value"
		],
		"type": "field"
		}
		]
		],
		"tags": []
		}
		],
		"thresholds": [],
		"timeFrom": null,
		"timeRegions": [],
		"timeShift": null,
		"title": "Sandbox Creation Time",
		"tooltip": {
		"shared": false,
		"sort": 0,
		"value_type": "individual"
		},
		"type": "graph",
		"xaxis": {
		"buckets": null,
		"mode": "time",
		"name": null,
		"show": true,
		"values": []
		},
		"yaxes": [
		{
		"decimals": 0,
		"format": "s",
		"label": null,
		"logBase": 1,
		"max": null,
		"min": "0",
		"show": true
		},
		{
		"format": "short",
		"label": null,
		"logBase": 1,
		"max": null,
		"min": null,
		"show": false
		}
		],
		"yaxis": {
		"align": false,
		"alignLevel": null
		}
		},
		{
		"datasource": "Prometheus",
		"fieldConfig": {
		"defaults": {
		"custom": {},
		"decimals": 0,
		"mappings": [],
		"noValue": "0",
		"thresholds": {
		"mode": "absolute",
		"steps": [
		{
		"color": "green",
		"value": null
		}
		]
		},
		"unit": "s"
		},
		"overrides": []
		},
		"gridPos": {
		"h": 7,
		"w": 4,
		"x": 10,
		"y": 1
		},
		"id": 95,
		"options": {
		"colorMode": "value",
		"graphMode": "area",
		"justifyMode": "auto",
		"orientation": "auto",
		"reduceOptions": {
		"calcs": [
		"lastNotNull"
		],
		"fields": "",
		"values": false
		},
		"textMode": "value"
		},
		"pluginVersion": "7.3.5",
		"targets": [
		{
		"expr": "sum(increase(mon_engine_sbox_create_duration_sum[$__range])) / sum(increase(mon_engine_sbox_create_duration_count[$__range]))",
		"instant": true,
		"interval": "",
		"intervalFactor": 1,
		"legendFormat": "",
		"refId": "A"
		}
		],
		"timeFrom": null,
		"timeShift": null,
		"title": "Average Sandbox Creation Time",
		"type": "stat"
		},
		{
		"datasource": "Prometheus",
		"fieldConfig": {
		"defaults": {
		"custom": {},
		"decimals": 0,
		"mappings": [],
		"thresholds": {
		"mode": "absolute",
		"steps": [
		{
		"color": "dark-green",
		"value": null
		},
		{
		"color": "light-green",
		"value": 0
		}
		]
		}
		},
		"overrides": []
		},
		"gridPos": {
		"h": 7,
		"w": 10,
		"x": 14,
		"y": 1
		},
		"id": 97,
		"options": {
		"displayMode": "gradient",
		"orientation": "auto",
		"reduceOptions": {
		"calcs": [
		"max"
		],
		"fields": "",
		"values": false
		},
		"showUnfilled": true
		},
		"pluginVersion": "7.3.5",
		"targets": [
		{
		"expr": "sum by (le) (increase(mon_engine_sbox_create_duration_bucket[$__range]))",
		"format": "heatmap",
		"instant": true,
		"interval": "",
		"legendFormat": "{{le}}",
		"refId": "A"
		}
		],
		"timeFrom": null,
		"timeShift": null,
		"title": "Sandbox Creation Time Distribution (seconds)",
		"transformations": [],
		"type": "bargauge"
		},
		{
		"collapsed": false,
		"datasource": null,
		"gridPos": {
		"h": 1,
		"w": 24,
		"x": 0,
		"y": 8
		},
		"id": 37,
		"panels": [],
		"title": "Platform API Requests & Notifications",
		@@ -44,7 +309,7 @@
		"h": 1,
		"w": 12,
		"x": 0,
		"y": 1
		"y": 9
		},
		"id": 78,
		"options": {
		@@ -69,7 +334,7 @@
		"h": 1,
		"w": 12,
		"x": 12,
		"y": 1
		"y": 9
		},
		"id": 79,
		"options": {
		@@ -106,7 +371,7 @@
		"h": 5,
		"w": 4,
		"x": 0,
		"y": 2
		"y": 10
		},
		"id": 55,
		"options": {
		@@ -167,7 +432,7 @@
		"h": 5,
		"w": 8,
		"x": 4,
		"y": 2
		"y": 10
		},
		"id": 77,
		"options": {
		@@ -242,7 +507,7 @@
		"h": 5,
		"w": 4,
		"x": 12,
		"y": 2
		"y": 10
		},
		"id": 85,
		"options": {
		@@ -303,7 +568,7 @@
		"h": 5,
		"w": 8,
		"x": 16,
		"y": 2
		"y": 10
		},
		"id": 86,
		"options": {
		@@ -385,7 +650,7 @@
		"h": 9,
		"w": 12,
		"x": 0,
		"y": 7
		"y": 15
		},
		"id": 30,
		"options": {
		@@ -473,7 +738,7 @@
		"h": 9,
		"w": 12,
		"x": 12,
		"y": 7
		"y": 15
		},
		"id": 87,
		"options": {
		@@ -532,7 +797,7 @@
		"h": 6,
		"w": 12,
		"x": 0,
		"y": 16
		"y": 24
		},
		"id": 33,
		"options": {
		@@ -591,7 +856,7 @@
		"h": 6,
		"w": 12,
		"x": 12,
		"y": 16
		"y": 24
		},
		"id": 88,
		"options": {
		@@ -655,7 +920,7 @@
		"h": 9,
		"w": 12,
		"x": 0,
		"y": 22
		"y": 30
		},
		"id": 38,
		"options": {
		@@ -719,7 +984,7 @@
		"h": 9,
		"w": 12,
		"x": 12,
		"y": 22
		"y": 30
		},
		"id": 89,
		"options": {
		@@ -813,5 +1078,5 @@
		"timezone": "",
		"title": "Platform (AdvantEDGE)",
		"uid": "platform-advantedge",
		"version": 1
		"version": 2
		}

charts/grafana/values.yaml

+2 −0

Original line number	Diff line number	Diff line
		@@ -526,6 +526,8 @@ grafana.ini:
		org_role: Viewer
		security:
		allow_embedding: true
		dashboards:
		min_refresh_interval: 1s
		# https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana
		# auth.github:
		# enabled: false

go-apps/meep-gis-engine/server/gis-automation.go

+3 −0

Original line number	Diff line number	Diff line
		@@ -60,6 +60,9 @@ func resetAutomation() {
		}

		func setAutomation(automationType string, state bool) (err error) {
		ge.mutex.Lock()
		defer ge.mutex.Unlock()

		// Validate automation type
		if _, found := ge.automation[automationType]; !found {
		return errors.New("Automation type not found")

go-apps/meep-mon-engine/server/mon-engine.go

+114 −4

Original line number	Diff line number	Diff line
		@@ -22,14 +22,19 @@ import (
		"net/http"
		"os"
		"strings"
		"sync"
		"time"

		dkm "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-data-key-mgr"
		log "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger"
		met "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-metrics"
		mq "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mq"
		redis "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis"
		sbs "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-sandbox-store"
		v1 "k8s.io/api/core/v1"

		"github.com/prometheus/client_golang/prometheus"
		"github.com/prometheus/client_golang/prometheus/promauto"
		"k8s.io/apimachinery/pkg/fields"
		"k8s.io/client-go/kubernetes"
		"k8s.io/client-go/rest"
		@@ -63,6 +68,12 @@ type MonEngineInfo struct {
		StartTime string
		}

		type Sandbox struct {
		Releases map[string]bool
		StartTime time.Time
		Running bool
		}

		const serviceName = "Monitoring Engine"
		const moduleName = "meep-mon-engine"
		const moduleNamespace = "default"
		@@ -77,6 +88,15 @@ const EVENT_POD_ADDED = 0
		const EVENT_POD_MODIFIED = 1
		const EVENT_POD_DELETED = 2

		// Metrics
		var (
		metricSboxCreateDuration = promauto.NewHistogram(prometheus.HistogramOpts{
		Name: "mon_engine_sbox_create_duration",
		Help: "A histogram of sandbox creation durations",
		Buckets: prometheus.LinearBuckets(10, 5, 5),
		})
		)

		var pod_event_str = [3]string{"pod added", "pod modified", "pod deleted"}
		var rc *redis.Connector
		var redisAddr = "meep-redis-master:6379"
		@@ -85,6 +105,11 @@ var stopChan = make(chan struct{})
		var mqGlobal *mq.MsgQueue
		var handlerId int
		var sandboxStore *sbs.SandboxStore
		var metricStore *met.MetricStore
		var influxDBAddr string = "http://meep-influxdb.default.svc.cluster.local:8086"
		var mutex sync.Mutex

		var sandboxes map[string]*Sandbox

		var depPodsList []string
		var corePodsList []string
		@@ -164,6 +189,16 @@ func Init() (err error) {
		}
		log.Info("Connected to Sandbox Store")

		// Connect to Metric Store
		metricStore, err = met.NewMetricStore("sandbox-metrics", "global", influxDBAddr, met.MetricsDbDisabled)
		if err != nil {
		log.Error("Failed connection to Metric Store: ", err)
		return err
		}

		// Initialize sandbox map
		sandboxes = make(map[string]*Sandbox)

		return nil
		}

		@@ -174,6 +209,7 @@ func Run() (err error) {
		if sboxMap, err := sandboxStore.GetAll(); err == nil {
		for _, sbox := range sboxMap {
		addExpectedPods(sbox.Name)
		createSandbox(sbox.Name)
		}
		}

		@@ -205,9 +241,11 @@ func msgHandler(msg *mq.Msg, userData interface{}) {
		case mq.MsgSandboxCreate:
		log.Debug("RX MSG: ", mq.PrintMsg(msg))
		addExpectedPods(msg.Payload[fieldSandboxName])
		createSandbox(msg.Payload[fieldSandboxName])
		case mq.MsgSandboxDestroy:
		log.Debug("RX MSG: ", mq.PrintMsg(msg))
		removeExpectedPods(msg.Payload[fieldSandboxName])
		deleteSandbox(msg.Payload[fieldSandboxName])
		default:
		log.Trace("Ignoring unsupported message: ", mq.PrintMsg(msg))
		}
		@@ -355,16 +393,17 @@ func processEvent(obj interface{}, reason int) {
		// Add, update or remove entry in DB only if core or scenario pod
		if monEngineInfo.PodType != notFoundStr {
		if reason == EVENT_POD_DELETED {
		deleteEntryInDB(monEngineInfo)
		deleteEntryInDB(&monEngineInfo)
		} else {
		addOrUpdateEntryInDB(monEngineInfo)
		addOrUpdateEntryInDB(&monEngineInfo)
		monitorSboxCreation(&monEngineInfo)
		}
		} else {
		log.Debug("Ignoring non-AdvantEDGE pod: ", monEngineInfo.PodName)
		}
		}

		func addOrUpdateEntryInDB(monEngineInfo MonEngineInfo) {
		func addOrUpdateEntryInDB(monEngineInfo *MonEngineInfo) {
		// Populate rule fields
		fields := make(map[string]interface{})
		fields["type"] = monEngineInfo.PodType
		@@ -396,7 +435,7 @@ func addOrUpdateEntryInDB(monEngineInfo MonEngineInfo) {
		}
		}

		func deleteEntryInDB(monEngineInfo MonEngineInfo) {
		func deleteEntryInDB(monEngineInfo *MonEngineInfo) {

		// Make unique key
		key := baseKey + monEngineInfo.Namespace + ":" + monEngineInfo.PodType + ":" + monEngineInfo.PodName
		@@ -441,6 +480,48 @@ func k8sConnect() (err error) {
		return nil
		}

		func monitorSboxCreation(monEngineInfo *MonEngineInfo) {
		mutex.Lock()
		defer mutex.Unlock()

		// Find matching sandbox entry
		sboxName := monEngineInfo.Namespace
		if sbox, found := sandboxes[sboxName]; found {

		// Ignore if sbox already running
		if !sbox.Running {

		// Set release running state
		if _, exists := sbox.Releases[monEngineInfo.Release]; exists {

		sbox.Releases[monEngineInfo.Release] = (monEngineInfo.LogicalState == "Running")

		// Check if sandbox is running
		sboxRunning := true
		for _, running := range sbox.Releases {
		if !running {
		sboxRunning = false
		break
		}
		}

		// If all releases are running, log sandbox creation time metric
		if sboxRunning {
		sbox.Running = true
		creationTime := float64(time.Since(sbox.StartTime).Milliseconds()) / 1000.0
		log.Info("Sbox: ", sboxName, " creationTime: ", creationTime)

		var metric met.SandboxMetric
		metric.Name = sboxName
		metric.CreateTime = creationTime
		_ = metricStore.SetSandboxMetric(met.SboxMetCreateTime, metric)
		metricSboxCreateDuration.Observe(creationTime)
		}
		}
		}
		}
		}

		// Retrieve POD states
		// GET /states
		func meGetStates(w http.ResponseWriter, r *http.Request) {
		@@ -636,3 +717,32 @@ func removeExpectedPods(sandboxName string) {
		delete(expectedSboxPods, podName)
		}
		}

		// Create new sandbox to monitor
		func createSandbox(sandboxName string) {
		mutex.Lock()
		defer mutex.Unlock()

		if _, exists := sandboxes[sandboxName]; !exists {
		sbox := new(Sandbox)
		sbox.Running = false
		sbox.Releases = make(map[string]bool)
		sbox.StartTime = time.Now()
		for _, pod := range sboxPodsList {
		sbox.Releases[pod] = false
		}
		sandboxes[sandboxName] = sbox
		log.Info("Created new sandbox to monitor: ", sandboxName)
		}
		}

		// Delete monitored sandbox
		func deleteSandbox(sandboxName string) {
		mutex.Lock()
		defer mutex.Unlock()

		if _, exists := sandboxes[sandboxName]; exists {
		delete(sandboxes, sandboxName)
		log.Info("Removed sandbox to monitor: ", sandboxName)
		}
		}

go-packages/meep-gis-asset-mgr/asset-mgr.go

+132 −67

File changed.

Preview size limit exceeded, changes collapsed.