Loading go-apps/meep-tc-engine/go.mod +0 −3 Original line number Diff line number Diff line Loading @@ -7,13 +7,11 @@ require ( github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-ctrl-engine-model v0.0.0 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger v0.0.0 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mg-manager-model v0.0.0 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model v0.0.0-00010101000000-000000000000 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis v0.0.0 github.com/gogo/protobuf v1.2.1 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/gofuzz v1.0.0 // indirect github.com/googleapis/gnostic v0.2.0 // indirect github.com/gorilla/handlers v1.4.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/json-iterator/go v1.1.6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect Loading @@ -37,6 +35,5 @@ replace ( github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-ctrl-engine-model => ../../go-packages/meep-ctrl-engine-model github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger => ../../go-packages/meep-logger github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mg-manager-model => ../../go-packages/meep-mg-manager-model github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model => ../../go-packages/meep-model github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis => ../../go-packages/meep-redis ) go-apps/meep-tc-engine/go.sum +10 −13 Original line number Diff line number Diff line cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/InterDigitalInc/AdvantEDGE v1.2.0 h1:46Mr4OqKz/6WU/1CziSacw62Z3pc52dgSOLppB5N/Bc= github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db h1:Zkf5kwhxdW0xV7WM/crqIcOP5LCFGnAmumWSFAewJ74= github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db/go.mod h1:RU+6d0CNIRSp6yo1mXLIIrnFa/3LHhvcDVLVJyovptM= github.com/KromDaniel/rejonson v0.0.0-20180822072824-00b5bcf2b351 h1:1u1XrfCBnY+GijnyU6O1k4odp5TnqZQTsp5v7+n/E4Y= github.com/KromDaniel/rejonson v0.0.0-20180822072824-00b5bcf2b351/go.mod h1:HxwfbuElTuGf+/uKZfjJrCnv0BmmpkPJDI7gBwj1KkM= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/flimzy/kivik v1.8.1/go.mod h1:S2aPycbG0eDFll4wgXt9uacSNkXISPufutnc9sv+mdA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-kivik/couchdb v1.8.1/go.mod h1:5XJRkAMpBlEVA4q0ktIZjUPYBjoBmRoiWvwUBzP3BOQ= github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= Loading @@ -20,16 +19,10 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= Loading @@ -41,17 +34,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= Loading @@ -65,6 +58,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= Loading @@ -77,10 +71,13 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= Loading go-apps/meep-tc-engine/tc-engine.go +56 −40 Original line number Diff line number Diff line Loading @@ -28,7 +28,6 @@ import ( ceModel "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-ctrl-engine-model" log "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger" mgModel "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mg-manager-model" mod "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model" redis "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" Loading Loading @@ -63,7 +62,8 @@ const channelTcLb string = moduleTcEngine + "-" + typeLb const MAX_THROUGHPUT = 9999999999 //easy value to spot in the array const COMMON_CORRELATION = 50 const COMMON_PACKET_LOSS = 10 // 1000 -> 10.00% // const COMMON_PACKET_LOSS = 10 // 1000 -> 10.00% const THROUGHPUT_UNIT = 1000000 //convert from Mbps to bps const DEFAULT_THROUGHPUT_LINK = 1000 //1000 mbps) const DEFAULT_THROUGHPUT_APP = 1000 //1000mbps) Loading @@ -79,9 +79,9 @@ const ( stateReady = 2 ) const DEFAULT_SCENARIO_DB = 0 const DEFAULT_NET_CHAR_DB = 0 const DEFAULT_LB_RULES_DB = 0 const DEFAULT_METRICS_DB = 0 const redisAddr string = "meep-redis-master:6379" type NetChar struct { Loading Loading @@ -170,21 +170,20 @@ type PodInfo struct { EgressSvcMapList map[string]*EgressSvcMap } type NetCharStore struct { type ScenarioStore struct { rc *redis.Connector } type LbRulesStore struct { type NetCharStore struct { rc *redis.Connector } type MetricsStore struct { type LbRulesStore struct { rc *redis.Connector } type TcEngine struct { scenarioModel *mod.Model scenarioStore *ScenarioStore netCharStore *NetCharStore lbRulesStore *LbRulesStore metricsStore *MetricsStore bwSharing *bws.BwSharing // Flag & Counters used to indicate when TC Engine is ready to Loading Loading @@ -240,12 +239,14 @@ func Init() (err error) { tce.svcCount = 0 tce.nextTransactionId = 1 // Create new Scenario Model instance (NOTE: opens Active Scenario Store) tce.scenarioModel, err = mod.NewModel(mod.DbAddress, "meep-tc-engine", "activeScenario") // Open Scenario Store tce.scenarioStore = new(ScenarioStore) tce.scenarioStore.rc, err = redis.NewConnector(redisAddr, DEFAULT_SCENARIO_DB) if err != nil { log.Error("Failed to create model: ", err.Error()) log.Error("Failed connection to Scenario Store Redis DB. Error: ", err) return err } log.Info("Connected to Scenario Store redis DB") // Open Network Characteristics Store tce.netCharStore = new(NetCharStore) Loading @@ -259,8 +260,6 @@ func Init() (err error) { // Flush any remaining TC Engine rules tce.netCharStore.rc.DBFlush(moduleTcEngine) bwSharing, err = bws.NewBwSharing("default", redisAddr, updateOneFilterRule, applyOneFilterRule) // Open Load Balancing Rules Store tce.lbRulesStore = new(LbRulesStore) tce.lbRulesStore.rc, err = redis.NewConnector(redisAddr, DEFAULT_LB_RULES_DB) Loading @@ -270,20 +269,8 @@ func Init() (err error) { } log.Info("Connected to LB Rules Store redis DB") // Open Metrics Store tce.metricsStore = new(MetricsStore) tce.metricsStore.rc, err = redis.NewConnector(redisAddr, DEFAULT_METRICS_DB) if err != nil { log.Error("Failed connection to Metrics Store Redis DB. Error: ", err) return err } log.Info("Connected to Metrics Store redis DB") // Flush existing values tce.metricsStore.rc.DBFlush(moduleMetrics) // Create new Bandwidth Sharing instance tce.bwSharing, err = bws.NewBwSharing("default", updateOneFilterRule, applyOneFilterRule) tce.bwSharing, err = bws.NewBwSharing("default", redisAddr, updateOneFilterRule, applyOneFilterRule) if err != nil { log.Error("Failed to create a bwSharing object. Error: ", err) return err Loading @@ -291,7 +278,11 @@ func Init() (err error) { // Configure & Start BW Sharing tce.bwSharing.UpdateControls() _ = tce.bwSharing.Start() err = tce.bwSharing.Start() if err != nil { log.Error("Failed to start BW Sharing. Error: ", err) return err } // Initialize TC Engine with current active scenario & LB rules processActiveScenarioUpdate() Loading @@ -303,16 +294,19 @@ func Init() (err error) { // Run - MEEP TC Engine execution func Run() error { // Listen for model updates err := tce.scenarioModel.Listen(eventHandler) // Listen for Active Scenario updates go func() { err := tce.scenarioStore.rc.Subscribe(channelCtrlActive) if err != nil { log.Error("Unable to listen to model updates: ", err.Error()) return err log.Error("Failed to subscribe to Pub/Sub events. Error: ", err) return } _ = tce.scenarioStore.rc.Listen(eventHandler) }() // Listen for LB Rules updates go func() { err = tce.lbRulesStore.rc.Subscribe(channelMgManagerLb) err := tce.lbRulesStore.rc.Subscribe(channelMgManagerLb) if err != nil { log.Error("Failed to subscribe to Pub/Sub events. Error: ", err) return Loading @@ -324,6 +318,8 @@ func Run() error { } func eventHandler(channel string, payload string) { mutex.Lock() // Handle Message according to Rx Channel switch channel { Loading @@ -339,12 +335,27 @@ func eventHandler(channel string, payload string) { default: log.Warn("Unsupported channel") } mutex.Unlock() } func processActiveScenarioUpdate() { // Retrieve active scenario from DB jsonScenario, err := tce.scenarioStore.rc.JSONGetEntry(moduleCtrlEngine+":"+typeActive, ".") if err != nil { log.Error(err.Error()) stopScenario() return } // Unmarshal Active scenario var scenario ceModel.Scenario err = json.Unmarshal([]byte(jsonScenario), &scenario) if err != nil { log.Error(err.Error()) stopScenario() return } // Parse scenario parseScenario(scenario) Loading @@ -359,7 +370,6 @@ func processActiveScenarioUpdate() { case stateReady: // Update Network Characteristic matrix table mutex.Lock() refreshNetCharTable() //debug for the tables Loading @@ -370,7 +380,6 @@ func processActiveScenarioUpdate() { // Apply network characteristic rules applyNetCharRules() mutex.Unlock() //Update the Db for state information (only transactionId for now) updateDbState(tce.nextTransactionId) Loading Loading @@ -508,7 +517,6 @@ func stopScenario() { scenarioName = "" tce.netCharStore.rc.DBFlush(moduleTcEngine) tce.metricsStore.rc.DBFlush(moduleMetrics) _ = tce.netCharStore.rc.Publish(channelTcNet, "delAll") _ = tce.netCharStore.rc.Publish(channelTcLb, "delAll") Loading Loading @@ -1078,6 +1086,8 @@ func updateOneFilterRule(dstName string, srcName string, rate float64) { } func applyOneFilterRule() { mutex.Lock() //Update the Db for state information (only transactionId for now) updateDbState(tce.nextTransactionId) Loading @@ -1085,6 +1095,8 @@ func applyOneFilterRule() { transactionIdStr := strconv.Itoa(tce.nextTransactionId) _ = tce.netCharStore.rc.Publish(channelTcNet, transactionIdStr) tce.nextTransactionId++ mutex.Unlock() } func applyNetCharRules() { Loading Loading @@ -1188,11 +1200,11 @@ func applyNetCharRules() { if needCreate { //follows +2 convention since one odd and even number reserved for the same rule (applied and updated one) dstElementPtr.NextUniqueNumber += 2 _ = updateFilterRule(&filterInfo, !bwSharing.IsRunning()) _ = updateFilterRule(&filterInfo, !tce.bwSharing.IsRunning()) } else if needUpdateFilter { _ = updateFilterRule(&filterInfo, !bwSharing.IsRunning()) _ = updateFilterRule(&filterInfo, !tce.bwSharing.IsRunning()) } else if needUpdateNetChar { _ = updateNetCharRule(&filterInfo, !bwSharing.IsRunning()) _ = updateNetCharRule(&filterInfo, !tce.bwSharing.IsRunning()) } indexToNetElemMap[j] = *dstElementPtr curNetCharList[j] = *dstElementPtr Loading Loading @@ -1499,11 +1511,15 @@ func getPlatformInfo() { log.Info("TC Engine scenario data retrieved. Moving to Ready state.") tce.tcEngineState = stateReady mutex.Lock() // Refresh & apply network characteristics rules processActiveScenarioUpdate() // Refresh & apply LB rules processMgSvcMapUpdate() mutex.Unlock() } else { log.Warn("TC Engine thread completed while not in Initializing state") } Loading go-packages/meep-redis/db.go +12 −1 Original line number Diff line number Diff line Loading @@ -27,6 +27,8 @@ import ( "github.com/go-redis/redis" ) const dbMaxRetryCount = 2 // Connector - Implements a Redis connector type Connector struct { addr string Loading @@ -41,11 +43,20 @@ type Connector struct { // NewConnector - Creates and initialize a Redis connector func NewConnector(addr string, table int) (rc *Connector, err error) { rc = new(Connector) // Connect to Redis DB for retry := 0; !rc.connected && retry <= dbMaxRetryCount; retry++ { err = rc.connectDB(addr, table) if err != nil { log.Warn("Failed to connect to DB. Retrying... Error: ", err) continue } } if err != nil { return nil, err } log.Info("Successfully connected to DB") return rc, nil } Loading Loading
go-apps/meep-tc-engine/go.mod +0 −3 Original line number Diff line number Diff line Loading @@ -7,13 +7,11 @@ require ( github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-ctrl-engine-model v0.0.0 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger v0.0.0 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mg-manager-model v0.0.0 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model v0.0.0-00010101000000-000000000000 github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis v0.0.0 github.com/gogo/protobuf v1.2.1 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/gofuzz v1.0.0 // indirect github.com/googleapis/gnostic v0.2.0 // indirect github.com/gorilla/handlers v1.4.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/json-iterator/go v1.1.6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect Loading @@ -37,6 +35,5 @@ replace ( github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-ctrl-engine-model => ../../go-packages/meep-ctrl-engine-model github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger => ../../go-packages/meep-logger github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mg-manager-model => ../../go-packages/meep-mg-manager-model github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model => ../../go-packages/meep-model github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis => ../../go-packages/meep-redis )
go-apps/meep-tc-engine/go.sum +10 −13 Original line number Diff line number Diff line cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/InterDigitalInc/AdvantEDGE v1.2.0 h1:46Mr4OqKz/6WU/1CziSacw62Z3pc52dgSOLppB5N/Bc= github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db h1:Zkf5kwhxdW0xV7WM/crqIcOP5LCFGnAmumWSFAewJ74= github.com/KromDaniel/jonson v0.0.0-20180630143114-d2f9c3c389db/go.mod h1:RU+6d0CNIRSp6yo1mXLIIrnFa/3LHhvcDVLVJyovptM= github.com/KromDaniel/rejonson v0.0.0-20180822072824-00b5bcf2b351 h1:1u1XrfCBnY+GijnyU6O1k4odp5TnqZQTsp5v7+n/E4Y= github.com/KromDaniel/rejonson v0.0.0-20180822072824-00b5bcf2b351/go.mod h1:HxwfbuElTuGf+/uKZfjJrCnv0BmmpkPJDI7gBwj1KkM= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/flimzy/kivik v1.8.1/go.mod h1:S2aPycbG0eDFll4wgXt9uacSNkXISPufutnc9sv+mdA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-kivik/couchdb v1.8.1/go.mod h1:5XJRkAMpBlEVA4q0ktIZjUPYBjoBmRoiWvwUBzP3BOQ= github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= Loading @@ -20,16 +19,10 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= Loading @@ -41,17 +34,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= Loading @@ -65,6 +58,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= Loading @@ -77,10 +71,13 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= Loading
go-apps/meep-tc-engine/tc-engine.go +56 −40 Original line number Diff line number Diff line Loading @@ -28,7 +28,6 @@ import ( ceModel "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-ctrl-engine-model" log "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-logger" mgModel "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-mg-manager-model" mod "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-model" redis "github.com/InterDigitalInc/AdvantEDGE/go-packages/meep-redis" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" Loading Loading @@ -63,7 +62,8 @@ const channelTcLb string = moduleTcEngine + "-" + typeLb const MAX_THROUGHPUT = 9999999999 //easy value to spot in the array const COMMON_CORRELATION = 50 const COMMON_PACKET_LOSS = 10 // 1000 -> 10.00% // const COMMON_PACKET_LOSS = 10 // 1000 -> 10.00% const THROUGHPUT_UNIT = 1000000 //convert from Mbps to bps const DEFAULT_THROUGHPUT_LINK = 1000 //1000 mbps) const DEFAULT_THROUGHPUT_APP = 1000 //1000mbps) Loading @@ -79,9 +79,9 @@ const ( stateReady = 2 ) const DEFAULT_SCENARIO_DB = 0 const DEFAULT_NET_CHAR_DB = 0 const DEFAULT_LB_RULES_DB = 0 const DEFAULT_METRICS_DB = 0 const redisAddr string = "meep-redis-master:6379" type NetChar struct { Loading Loading @@ -170,21 +170,20 @@ type PodInfo struct { EgressSvcMapList map[string]*EgressSvcMap } type NetCharStore struct { type ScenarioStore struct { rc *redis.Connector } type LbRulesStore struct { type NetCharStore struct { rc *redis.Connector } type MetricsStore struct { type LbRulesStore struct { rc *redis.Connector } type TcEngine struct { scenarioModel *mod.Model scenarioStore *ScenarioStore netCharStore *NetCharStore lbRulesStore *LbRulesStore metricsStore *MetricsStore bwSharing *bws.BwSharing // Flag & Counters used to indicate when TC Engine is ready to Loading Loading @@ -240,12 +239,14 @@ func Init() (err error) { tce.svcCount = 0 tce.nextTransactionId = 1 // Create new Scenario Model instance (NOTE: opens Active Scenario Store) tce.scenarioModel, err = mod.NewModel(mod.DbAddress, "meep-tc-engine", "activeScenario") // Open Scenario Store tce.scenarioStore = new(ScenarioStore) tce.scenarioStore.rc, err = redis.NewConnector(redisAddr, DEFAULT_SCENARIO_DB) if err != nil { log.Error("Failed to create model: ", err.Error()) log.Error("Failed connection to Scenario Store Redis DB. Error: ", err) return err } log.Info("Connected to Scenario Store redis DB") // Open Network Characteristics Store tce.netCharStore = new(NetCharStore) Loading @@ -259,8 +260,6 @@ func Init() (err error) { // Flush any remaining TC Engine rules tce.netCharStore.rc.DBFlush(moduleTcEngine) bwSharing, err = bws.NewBwSharing("default", redisAddr, updateOneFilterRule, applyOneFilterRule) // Open Load Balancing Rules Store tce.lbRulesStore = new(LbRulesStore) tce.lbRulesStore.rc, err = redis.NewConnector(redisAddr, DEFAULT_LB_RULES_DB) Loading @@ -270,20 +269,8 @@ func Init() (err error) { } log.Info("Connected to LB Rules Store redis DB") // Open Metrics Store tce.metricsStore = new(MetricsStore) tce.metricsStore.rc, err = redis.NewConnector(redisAddr, DEFAULT_METRICS_DB) if err != nil { log.Error("Failed connection to Metrics Store Redis DB. Error: ", err) return err } log.Info("Connected to Metrics Store redis DB") // Flush existing values tce.metricsStore.rc.DBFlush(moduleMetrics) // Create new Bandwidth Sharing instance tce.bwSharing, err = bws.NewBwSharing("default", updateOneFilterRule, applyOneFilterRule) tce.bwSharing, err = bws.NewBwSharing("default", redisAddr, updateOneFilterRule, applyOneFilterRule) if err != nil { log.Error("Failed to create a bwSharing object. Error: ", err) return err Loading @@ -291,7 +278,11 @@ func Init() (err error) { // Configure & Start BW Sharing tce.bwSharing.UpdateControls() _ = tce.bwSharing.Start() err = tce.bwSharing.Start() if err != nil { log.Error("Failed to start BW Sharing. Error: ", err) return err } // Initialize TC Engine with current active scenario & LB rules processActiveScenarioUpdate() Loading @@ -303,16 +294,19 @@ func Init() (err error) { // Run - MEEP TC Engine execution func Run() error { // Listen for model updates err := tce.scenarioModel.Listen(eventHandler) // Listen for Active Scenario updates go func() { err := tce.scenarioStore.rc.Subscribe(channelCtrlActive) if err != nil { log.Error("Unable to listen to model updates: ", err.Error()) return err log.Error("Failed to subscribe to Pub/Sub events. Error: ", err) return } _ = tce.scenarioStore.rc.Listen(eventHandler) }() // Listen for LB Rules updates go func() { err = tce.lbRulesStore.rc.Subscribe(channelMgManagerLb) err := tce.lbRulesStore.rc.Subscribe(channelMgManagerLb) if err != nil { log.Error("Failed to subscribe to Pub/Sub events. Error: ", err) return Loading @@ -324,6 +318,8 @@ func Run() error { } func eventHandler(channel string, payload string) { mutex.Lock() // Handle Message according to Rx Channel switch channel { Loading @@ -339,12 +335,27 @@ func eventHandler(channel string, payload string) { default: log.Warn("Unsupported channel") } mutex.Unlock() } func processActiveScenarioUpdate() { // Retrieve active scenario from DB jsonScenario, err := tce.scenarioStore.rc.JSONGetEntry(moduleCtrlEngine+":"+typeActive, ".") if err != nil { log.Error(err.Error()) stopScenario() return } // Unmarshal Active scenario var scenario ceModel.Scenario err = json.Unmarshal([]byte(jsonScenario), &scenario) if err != nil { log.Error(err.Error()) stopScenario() return } // Parse scenario parseScenario(scenario) Loading @@ -359,7 +370,6 @@ func processActiveScenarioUpdate() { case stateReady: // Update Network Characteristic matrix table mutex.Lock() refreshNetCharTable() //debug for the tables Loading @@ -370,7 +380,6 @@ func processActiveScenarioUpdate() { // Apply network characteristic rules applyNetCharRules() mutex.Unlock() //Update the Db for state information (only transactionId for now) updateDbState(tce.nextTransactionId) Loading Loading @@ -508,7 +517,6 @@ func stopScenario() { scenarioName = "" tce.netCharStore.rc.DBFlush(moduleTcEngine) tce.metricsStore.rc.DBFlush(moduleMetrics) _ = tce.netCharStore.rc.Publish(channelTcNet, "delAll") _ = tce.netCharStore.rc.Publish(channelTcLb, "delAll") Loading Loading @@ -1078,6 +1086,8 @@ func updateOneFilterRule(dstName string, srcName string, rate float64) { } func applyOneFilterRule() { mutex.Lock() //Update the Db for state information (only transactionId for now) updateDbState(tce.nextTransactionId) Loading @@ -1085,6 +1095,8 @@ func applyOneFilterRule() { transactionIdStr := strconv.Itoa(tce.nextTransactionId) _ = tce.netCharStore.rc.Publish(channelTcNet, transactionIdStr) tce.nextTransactionId++ mutex.Unlock() } func applyNetCharRules() { Loading Loading @@ -1188,11 +1200,11 @@ func applyNetCharRules() { if needCreate { //follows +2 convention since one odd and even number reserved for the same rule (applied and updated one) dstElementPtr.NextUniqueNumber += 2 _ = updateFilterRule(&filterInfo, !bwSharing.IsRunning()) _ = updateFilterRule(&filterInfo, !tce.bwSharing.IsRunning()) } else if needUpdateFilter { _ = updateFilterRule(&filterInfo, !bwSharing.IsRunning()) _ = updateFilterRule(&filterInfo, !tce.bwSharing.IsRunning()) } else if needUpdateNetChar { _ = updateNetCharRule(&filterInfo, !bwSharing.IsRunning()) _ = updateNetCharRule(&filterInfo, !tce.bwSharing.IsRunning()) } indexToNetElemMap[j] = *dstElementPtr curNetCharList[j] = *dstElementPtr Loading Loading @@ -1499,11 +1511,15 @@ func getPlatformInfo() { log.Info("TC Engine scenario data retrieved. Moving to Ready state.") tce.tcEngineState = stateReady mutex.Lock() // Refresh & apply network characteristics rules processActiveScenarioUpdate() // Refresh & apply LB rules processMgSvcMapUpdate() mutex.Unlock() } else { log.Warn("TC Engine thread completed while not in Initializing state") } Loading
go-packages/meep-redis/db.go +12 −1 Original line number Diff line number Diff line Loading @@ -27,6 +27,8 @@ import ( "github.com/go-redis/redis" ) const dbMaxRetryCount = 2 // Connector - Implements a Redis connector type Connector struct { addr string Loading @@ -41,11 +43,20 @@ type Connector struct { // NewConnector - Creates and initialize a Redis connector func NewConnector(addr string, table int) (rc *Connector, err error) { rc = new(Connector) // Connect to Redis DB for retry := 0; !rc.connected && retry <= dbMaxRetryCount; retry++ { err = rc.connectDB(addr, table) if err != nil { log.Warn("Failed to connect to DB. Retrying... Error: ", err) continue } } if err != nil { return nil, err } log.Info("Successfully connected to DB") return rc, nil } Loading