Perf Eval:

- added results

src/common/method_wrappers/results-perf-eval/MW/generate.sh

+#!/bin/bash
+
+python generate_plot.py "DEVICE_DRIVER_MW"
+python generate_plot.py "SERVICE_HANDLER_MW"

src/common/method_wrappers/results-perf-eval/MW/generate_plot.py

+import enum, sys
+import numpy as np
+import matplotlib.pyplot as plt
+
+class PlotName(enum.Enum):
+    DEVICE_DRIVER_MW = 'dev-drv-mw'
+    SERVICE_HANDLER_MW = 'srv-hlr-mw'
+
+plot_name = PlotName.__members__.get(sys.argv[1])
+if plot_name is None: raise Exception('Unsupported plot: {:s}'.format(str(plot_name)))
+
+PLOTS = {
+    PlotName.DEVICE_DRIVER_MW: (
+        #'Device Driver - MicroWave', '0.0001-100', [
+        #    ('GetConfig',    [0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,10,172,0,1,0,0,0,0,0,0]),
+        #    ('SetConfig',    [89,1,0,0,0,0,0,0,0,0,0,0,0,0,0,6,34,50,1,0,0,0,0,0,0,0]),
+        #    ('DeleteConfig', [90,1,0,0,0,0,0,0,0,0,0,0,0,0,2,3,0,4,72,12,0,0,0,0,0,0]),
+        #]),
+        'Device Driver - MicroWave', '0.1-10', [
+            ('GetConfig',    [0,1,0,10,172,0,1,0]),
+            ('SetConfig',    [0,0,6,34,50,1,0,0]),
+            ('DeleteConfig', [0,2,3,0,4,72,12,0]),
+        ]),
+    PlotName.SERVICE_HANDLER_MW: (
+        'Service Handler - L2NM MicroWave', '1-100', [
+            ('SetEndpoint',    [0,1,0,1,5,75,6,0]),
+            ('DeleteEndpoint', [0,0,0,0,1,77,17,0]),
+        ]),
+}
+
+BINS_RANGES = {
+    '0.0001-100'    : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10,
+                        25, 50, 75, 100, 200],
+    '0.1-10'    : [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10],
+    '0.0001-1'      : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1],
+    '0.0001-0.25'   : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25],
+    '1-100'     : [1, 2.5, 5, 7.5, 10, 25, 50, 75, 100],
+    '0.001-100'     : [0, 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075,
+                        0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 200],
+    '0.001-7.5'     : [0, 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075,
+                        0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10],
+    '0.01-5'        : [0, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5],
+}
+
+# plot the cumulative histogram
+fig, ax = plt.subplots(figsize=(8, 8))
+
+bins = PLOTS[plot_name][1]
+if isinstance(bins, str): bins = BINS_RANGES[PLOTS[plot_name][1]]
+bins = np.array(bins).astype(float)
+
+for label, counts in PLOTS[plot_name][2]:
+    counts = np.array(counts).astype(float)
+    assert len(bins) == len(counts) + 1
+    centroids = (bins[1:] + bins[:-1]) / 2
+    ax.hist(centroids, bins=bins, weights=counts, range=(min(bins), max(bins)), density=True,
+            histtype='step', cumulative=True, label=label)
+
+ax.grid(True)
+ax.legend(loc='upper left')
+ax.set_title(PLOTS[plot_name][0])
+ax.set_xlabel('seconds')
+ax.set_ylabel('Likelihood of occurrence')
+plt.xscale('log')
+plt.savefig('{:s}.png'.format(plot_name.value), dpi = (600)) 
+plt.show()

src/common/method_wrappers/results-perf-eval/OpenConfig/generate.sh

+#!/bin/bash
+
+python generate_plot.py "DEVICE_DRIVER_OPENCONFIG"
+python generate_plot.py "SERVICE_HANDLER_OPENCONFIG_L2NM"
+python generate_plot.py "SERVICE_HANDLER_OPENCONFIG_L3NM"

src/common/method_wrappers/results-perf-eval/OpenConfig/generate_plot.py

+import enum, sys
+import numpy as np
+import matplotlib.pyplot as plt
+
+class PlotName(enum.Enum):
+    DEVICE_DRIVER_OPENCONFIG = 'dev-drv-openconfig'
+    SERVICE_HANDLER_OPENCONFIG_L2NM = 'srv-hlr-openconfig-l2nm'
+    SERVICE_HANDLER_OPENCONFIG_L3NM = 'srv-hlr-openconfig-l3nm'
+
+plot_name = PlotName.__members__.get(sys.argv[1])
+if plot_name is None: raise Exception('Unsupported plot: {:s}'.format(str(plot_name)))
+
+PLOTS = {
+    PlotName.DEVICE_DRIVER_OPENCONFIG: (
+        'Device Driver - OpenConfig', '0.0001-100', [
+            #('GetConfig',    [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,121,0,0,0,0]),
+            #('SetConfig',    [127,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,43,19,0,0,0,0,0,0]),
+            #('DeleteConfig', [92,1,0,0,0,0,0,0,0,0,0,0,0,0,5,2,8,71,14,0,0,0,0,0,0,0]),
+
+            ('GetConfig',    [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,121,0,0,0,0]),
+            ('SetConfig',    [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,43,19,0,0,0,0,0,0]),
+            ('DeleteConfig', [0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,2,8,71,14,0,0,0,0,0,0,0]),
+        ]),
+    PlotName.SERVICE_HANDLER_OPENCONFIG_L2NM: (
+        'Service Handler - L2NM OpenConfig', '0.001-100', [
+            ('SetEndpoint',    [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,10]),
+            ('DeleteEndpoint', [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14]),
+        ]),
+    PlotName.SERVICE_HANDLER_OPENCONFIG_L3NM: (
+        'Service Handler - L3NM OpenConfig', '0.001-100', [
+            ('SetEndpoint',    [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,17]),
+            ('DeleteEndpoint', [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20]),
+        ]),
+}
+
+BINS_RANGES = {
+    '0.0001-100'    : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10,
+                        25, 50, 75, 100, 200],
+    '0.1-10'    : [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10],
+    '0.0001-1'      : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1],
+    '0.0001-0.25'   : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25],
+    '1-100'     : [1, 2.5, 5, 7.5, 10, 25, 50, 75, 100],
+    '0.001-100'     : [0, 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075,
+                        0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 200],
+    '0.001-7.5'     : [0, 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075,
+                        0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10],
+    '0.01-5'        : [0, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5],
+}
+
+# plot the cumulative histogram
+fig, ax = plt.subplots(figsize=(8, 8))
+
+bins = PLOTS[plot_name][1]
+if isinstance(bins, str): bins = BINS_RANGES[PLOTS[plot_name][1]]
+bins = np.array(bins).astype(float)
+
+for label, counts in PLOTS[plot_name][2]:
+    counts = np.array(counts).astype(float)
+    assert len(bins) == len(counts) + 1
+    centroids = (bins[1:] + bins[:-1]) / 2
+    ax.hist(centroids, bins=bins, weights=counts, range=(min(bins), max(bins)), density=True,
+            histtype='step', cumulative=True, label=label)
+
+ax.grid(True)
+ax.legend(loc='upper left')
+ax.set_title(PLOTS[plot_name][0])
+ax.set_xlabel('seconds')
+ax.set_ylabel('Likelihood of occurrence')
+plt.xscale('log')
+plt.savefig('{:s}.png'.format(plot_name.value), dpi = (600)) 
+plt.show()

src/common/method_wrappers/results-perf-eval/TE/te-cdf.py

+import matplotlib.pyplot as plt
+
+flow_creation_us = [
+    3.007065,3.007783,3.010780,3.007374,3.006519,3.006668,3.006303,3.006463,3.006758,3.007992,3.012198,3.001413,
+    3.007289,3.006241,3.007523,3.007569,3.006643,3.006255,3.007058,3.006111,3.006918,3.007972,3.006829,3.007378,
+    3.007666,3.003071,3.006774,3.006060,3.006731,3.005812
+]
+
+flow_update_us = [
+    3.005123,3.004228,3.003897,3.006692,3.003767,3.003749,3.004626,3.004333,3.004449,3.003895,3.004092,3.003979,
+    3.005099,3.213206,3.004625,3.004707,3.004187,3.004609,3.003885,3.004064,3.004308,3.004280,3.004423,3.211980,
+    3.004138,3.004394,3.004018,3.004747,3.005719,3.003656
+]
+
+n_bins = 10
+fig, ax = plt.subplots(figsize=(8, 8))
+
+# plot the cumulative histograms
+n, bins, _ = ax.hist(flow_creation_us, n_bins, density=True, histtype='step', cumulative=True, label='FlowCreate')
+print(n, bins)
+n, bins, _ = ax.hist(flow_update_us, n_bins, density=True, histtype='step', cumulative=True, label='FlowUpdate')
+print(n, bins)
+
+ax.grid(True)
+ax.legend(loc='lower center')
+ax.set_title('TE Flow Management Delay')
+ax.set_xlabel('seconds')
+ax.set_ylabel('Likelihood of occurrence')
+plt.savefig('te-perf-eval.png', dpi = (600)) 
+plt.show()

src/common/method_wrappers/results-perf-eval/XR/generate.sh

+#!/bin/bash
+
+python generate_plot.py "DEVICE_DRIVER_XR"

src/common/method_wrappers/results-perf-eval/XR/generate_plot.py

+import enum, sys
+import numpy as np
+import matplotlib.pyplot as plt
+
+class PlotName(enum.Enum):
+    DEVICE_DRIVER_XR = 'dev-drv-xr'
+
+plot_name = PlotName.__members__.get(sys.argv[1])
+if plot_name is None: raise Exception('Unsupported plot: {:s}'.format(str(plot_name)))
+
+PLOTS = {
+    PlotName.DEVICE_DRIVER_XR: (
+        #'Device Driver - XR', '0.0001-0.25', [
+        #    ('GetConfig',    [0,0,0,0,0,0,0,0,0,77,1,1,0,0]),
+        #    ('SetConfig',    [0,15,17,7,0,0,0,0,0,0,34,3,2,0]),
+        #    ('DeleteConfig', [23,16,0,0,0,0,0,0,0,1,32,5,1,0]),
+        #]),
+        'Device Driver - XR', '0.0001-0.25', [
+            ('GetConfig',    [0,0,0,0,0,0,0,0,0,77,1,1,0,0]),
+            ('SetConfig',    [0,0,0,0,0,0,0,0,0,0,34,3,2,0]),
+            ('DeleteConfig', [0,0,0,0,0,0,0,0,0,1,32,5,1,0]),
+        ]),
+}
+
+BINS_RANGES = {
+    '0.0001-100'    : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10,
+                        25, 50, 75, 100, 200],
+    '0.0001-1'      : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1],
+    '0.0001-0.25'   : [0, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0025, 0.005, 0.0075,
+                        0.01, 0.025, 0.05, 0.075, 0.1, 0.25],
+    '0.001-100'     : [0, 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075,
+                        0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 200],
+    '0.001-7.5'     : [0, 0.001, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.075,
+                        0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10],
+    '0.01-5'        : [0, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5],
+}
+
+# plot the cumulative histogram
+fig, ax = plt.subplots(figsize=(8, 8))
+
+bins = PLOTS[plot_name][1]
+if isinstance(bins, str): bins = BINS_RANGES[PLOTS[plot_name][1]]
+bins = np.array(bins).astype(float)
+
+for label, counts in PLOTS[plot_name][2]:
+    counts = np.array(counts).astype(float)
+    assert len(bins) == len(counts) + 1
+    centroids = (bins[1:] + bins[:-1]) / 2
+    ax.hist(centroids, bins=bins, weights=counts, range=(min(bins), max(bins)), density=True,
+            histtype='step', cumulative=True, label=label)
+
+ax.grid(True)
+ax.legend(loc='upper left')
+ax.set_title(PLOTS[plot_name][0])
+ax.set_xlabel('seconds')
+ax.set_ylabel('Likelihood of occurrence')
+plt.xscale('log')
+plt.savefig('{:s}.png'.format(plot_name.value), dpi = (600)) 
+plt.show()