Improved Service metering:

- Added methods to automate metric class generation
- Added decorator for servicer RPC methods to add metering and logging features automatically
- Added unit tests
- Minor imptovements in context unit testing

report_coverage_context.sh

 #!/bin/bash
 
-./report_coverage_all.sh | grep --color -E -i "^context/.*$|$"
+./report_coverage_all.sh | grep -v -E "^(cent|com|devi|moni|serv|test)" | grep --color -E -i "^context/.*$|$"

run_local_tests.sh

@@ -15,30 +15,32 @@ cat $PROJECTDIR/coverage/.coveragerc.template | sed s+~/teraflow/controller+$PRO
 rm -f $COVERAGEFILE
 
 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    common/database/tests/test_unitary.py \
-    common/database/tests/test_engine_inmemory.py
+    common/metrics/tests/test_unitary.py \
+    common/orm/tests/test_unitary.py
 
-coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    centralizedcybersecurity/tests/test_unitary.py
-
-coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    context/tests/test_unitary.py
-
-coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    device/tests/test_unitary_driverapi.py \
-    device/tests/test_unitary_service.py
-
-coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    service/tests/test_unitary.py
-
-coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    compute/tests/test_unitary.py
+#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+#    centralizedcybersecurity/tests/test_unitary.py
 
-# Run integration tests and analyze coverage of code at same time
-export DB_ENGINE='redis'
-export REDIS_SERVICE_HOST='10.1.7.194'
-export REDIS_SERVICE_PORT='31789'
-export REDIS_DATABASE_ID='0'
 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
-    common/database/tests/test_engine_redis.py \
-    tester_integration/test_context_device_service.py
+    context/tests/test_unitary_orm_context_inmemory.py \
+#    context/tests/test_unitary.py
+
+#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+#    device/tests/test_unitary_driverapi.py \
+#    device/tests/test_unitary_service.py
+
+#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+#    service/tests/test_unitary.py
+
+#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+#    compute/tests/test_unitary.py
+
+## Run integration tests and analyze coverage of code at same time
+#export DB_ENGINE='redis'
+#export REDIS_SERVICE_HOST='10.1.7.194'
+## Find exposed port for Redis service port 6379
+#export REDIS_SERVICE_PORT=$(kubectl --namespace gitlab-ci get service redis-public -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')
+#export REDIS_DATABASE_ID='0'
+#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \
+#    context/tests/test_database_engine_redis.py \
+#    tester_integration/test_context_device_service.py

src/common/metrics/Metrics.py

+import grpc, logging
+from enum import Enum
+from typing import Dict, List
+from prometheus_client import Counter, Histogram
+from prometheus_client.metrics import MetricWrapperBase
+from common.exceptions.ServiceException import ServiceException
+
+class RequestConditionEnum(Enum):
+    STARTED   = 'started'
+    COMPLETED = 'completed'
+    FAILED    = 'failed'
+
+def get_counter_requests(method_name : str, request_condition : RequestConditionEnum) -> Counter:
+    str_request_condition = request_condition.value
+    name = '{:s}_counter_requests_{:s}'.format(method_name.replace(':', '_'), str_request_condition)
+    description = '{:s} counter of requests {:s}'.format(method_name, str_request_condition)
+    return Counter(name, description)
+
+def get_histogram_duration(method_name : str) -> Histogram:
+    name = '{:s}_histogram_duration'.format(method_name.replace(':', '_'))
+    description = '{:s} histogram of request duration'.format(method_name)
+    return Histogram(name, description)
+
+METRIC_TEMPLATES = {
+    '{:s}_COUNTER_STARTED'   : lambda method_name: get_counter_requests  (method_name, RequestConditionEnum.STARTED),
+    '{:s}_COUNTER_COMPLETED' : lambda method_name: get_counter_requests  (method_name, RequestConditionEnum.COMPLETED),
+    '{:s}_COUNTER_FAILED'    : lambda method_name: get_counter_requests  (method_name, RequestConditionEnum.FAILED),
+    '{:s}_HISTOGRAM_DURATION': lambda method_name: get_histogram_duration(method_name),
+}
+
+def create_metrics(service_name : str, method_names : List[str]) -> Dict[str, MetricWrapperBase]:
+    metrics = {}
+    for method_name in method_names:
+        for template_name, template_generator_function in METRIC_TEMPLATES.items():
+            metric_name = template_name.format(method_name).upper()
+            metrics[metric_name] = template_generator_function('{:s}:{:s}'.format(service_name, method_name))
+    return metrics
+
+def safe_and_metered_rpc_method(metrics : Dict[str, MetricWrapperBase], logger : logging.Logger):
+    def outer_wrapper(func):
+        function_name = func.__name__
+        print('function_name', function_name)
+        HISTOGRAM_DURATION : Histogram = metrics.get('{:s}_HISTOGRAM_DURATION'.format(function_name).upper())
+        COUNTER_STARTED    : Counter   = metrics.get('{:s}_COUNTER_STARTED'   .format(function_name).upper())
+        COUNTER_COMPLETED  : Counter   = metrics.get('{:s}_COUNTER_COMPLETED' .format(function_name).upper())
+        COUNTER_FAILED     : Counter   = metrics.get('{:s}_COUNTER_FAILED'    .format(function_name).upper())
+
+        @HISTOGRAM_DURATION.time()
+        def inner_wrapper(self, request, grpc_context : grpc.ServicerContext):
+            COUNTER_STARTED.inc()
+            try:
+                logger.debug('{:s} request: {:s}'.format(function_name, str(request)))
+                reply = func(self, request, grpc_context)
+                logger.debug('{:s} reply: {:s}'.format(function_name, str(reply)))
+                COUNTER_COMPLETED.inc()
+                return reply
+            except ServiceException as e:                               # pragma: no cover (ServiceException not thrown)
+                logger.exception('{:s} exception'.format(function_name))
+                COUNTER_FAILED.inc()
+                grpc_context.abort(e.code, e.details)
+            except Exception as e:                                      # pragma: no cover
+                logger.exception('{:s} exception'.format(function_name))
+                COUNTER_FAILED.inc()
+                grpc_context.abort(grpc.StatusCode.INTERNAL, str(e))
+        return inner_wrapper
+    return outer_wrapper

src/common/metrics/tests/test_unitary.py

+import grpc, logging, pytest, time
+from common.metrics.Metrics import create_metrics, safe_and_metered_rpc_method
+
+logging.basicConfig(level=logging.DEBUG)
+LOGGER = logging.getLogger(__name__)
+
+def test_database_instantiation():
+    SERVICE_NAME = 'Context'
+    METHOD_NAMES = [
+        'ListContextIds',  'ListContexts',   'GetContext',  'SetContext',  'RemoveContext',  'GetContextEvents',
+        'ListTopologyIds', 'ListTopologies', 'GetTopology', 'SetTopology', 'RemoveTopology', 'GetTopologyEvents',
+        'ListDeviceIds',   'ListDevices',    'GetDevice',   'SetDevice',   'RemoveDevice',   'GetDeviceEvents',
+        'ListLinkIds',     'ListLinks',      'GetLink',     'SetLink',     'RemoveLink',     'GetLinkEvents',
+        'ListServiceIds',  'ListServices',   'GetService',  'SetService',  'RemoveService',  'GetServiceEvents',
+    ]
+    METRICS = create_metrics(SERVICE_NAME, METHOD_NAMES)
+
+    class TestServiceServicerImpl:
+        @safe_and_metered_rpc_method(METRICS, LOGGER)
+        def GetTopology(self, request, grpc_context : grpc.ServicerContext):
+            print('doing funny things')
+            time.sleep(0.1)
+            return 'done'
+
+    tssi = TestServiceServicerImpl()
+    tssi.GetTopology(1, 2)
+
+    for metric_name,metric in METRICS.items():
+        if 'GETTOPOLOGY_' not in metric_name: continue
+        print(metric_name, metric._child_samples())