Commit 2d5fc449 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Improved Service metering: 

- Added methods to automate metric class generation
- Added decorator for servicer RPC methods to add metering and logging features automatically
- Added unit tests
- Minor imptovements in context unit testing
parent 10e25f75

report_coverage_context.sh

+1 −1
Original line number Original line Diff line number Diff line 
#!/bin/bash

#!/bin/bash





./report_coverage_all.sh | grep --color -E -i "^context/.*$|$"
 
./report_coverage_all.sh | grep -v -E "^(cent|com|devi|moni|serv|test)" | grep --color -E -i "^context/.*$|$"

run_local_tests.sh

+26 −24
Original line number Original line Diff line number Diff line
@@ -15,30 +15,32 @@ cat $PROJECTDIR/coverage/.coveragerc.template | sed s+~/teraflow/controller+$PRO 
rm -f $COVERAGEFILE

rm -f $COVERAGEFILE





coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    common/database/tests/test_unitary.py \

    common/metrics/tests/test_unitary.py \

    common/database/tests/test_engine_inmemory.py

    common/orm/tests/test_unitary.py





coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    centralizedcybersecurity/tests/test_unitary.py

#    centralizedcybersecurity/tests/test_unitary.py



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    context/tests/test_unitary.py



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    device/tests/test_unitary_driverapi.py \

    device/tests/test_unitary_service.py



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    service/tests/test_unitary.py



coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    compute/tests/test_unitary.py





# Run integration tests and analyze coverage of code at same time

export DB_ENGINE='redis'

export REDIS_SERVICE_HOST='10.1.7.194'

export REDIS_SERVICE_PORT='31789'

export REDIS_DATABASE_ID='0'

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

    common/database/tests/test_engine_redis.py \

    context/tests/test_unitary_orm_context_inmemory.py \

    tester_integration/test_context_device_service.py
 
#    context/tests/test_unitary.py



#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#    device/tests/test_unitary_driverapi.py \

#    device/tests/test_unitary_service.py



#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#    service/tests/test_unitary.py



#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#    compute/tests/test_unitary.py



## Run integration tests and analyze coverage of code at same time

#export DB_ENGINE='redis'

#export REDIS_SERVICE_HOST='10.1.7.194'

## Find exposed port for Redis service port 6379

#export REDIS_SERVICE_PORT=$(kubectl --namespace gitlab-ci get service redis-public -o 'jsonpath={.spec.ports[?(@.port==6379)].nodePort}')

#export REDIS_DATABASE_ID='0'

#coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#    context/tests/test_database_engine_redis.py \

#    tester_integration/test_context_device_service.py

src/common/metrics/Metrics.py

0 → 100644
+66 −0
Original line number Original line Diff line number Diff line 
import grpc, logging

from enum import Enum

from typing import Dict, List

from prometheus_client import Counter, Histogram

from prometheus_client.metrics import MetricWrapperBase

from common.exceptions.ServiceException import ServiceException



class RequestConditionEnum(Enum):

    STARTED   = 'started'

    COMPLETED = 'completed'

    FAILED    = 'failed'



def get_counter_requests(method_name : str, request_condition : RequestConditionEnum) -> Counter:

    str_request_condition = request_condition.value

    name = '{:s}_counter_requests_{:s}'.format(method_name.replace(':', '_'), str_request_condition)

    description = '{:s} counter of requests {:s}'.format(method_name, str_request_condition)

    return Counter(name, description)



def get_histogram_duration(method_name : str) -> Histogram:

    name = '{:s}_histogram_duration'.format(method_name.replace(':', '_'))

    description = '{:s} histogram of request duration'.format(method_name)

    return Histogram(name, description)



METRIC_TEMPLATES = {

    '{:s}_COUNTER_STARTED'   : lambda method_name: get_counter_requests  (method_name, RequestConditionEnum.STARTED),

    '{:s}_COUNTER_COMPLETED' : lambda method_name: get_counter_requests  (method_name, RequestConditionEnum.COMPLETED),

    '{:s}_COUNTER_FAILED'    : lambda method_name: get_counter_requests  (method_name, RequestConditionEnum.FAILED),

    '{:s}_HISTOGRAM_DURATION': lambda method_name: get_histogram_duration(method_name),

}



def create_metrics(service_name : str, method_names : List[str]) -> Dict[str, MetricWrapperBase]:

    metrics = {}

    for method_name in method_names:

        for template_name, template_generator_function in METRIC_TEMPLATES.items():

            metric_name = template_name.format(method_name).upper()

            metrics[metric_name] = template_generator_function('{:s}:{:s}'.format(service_name, method_name))

    return metrics



def safe_and_metered_rpc_method(metrics : Dict[str, MetricWrapperBase], logger : logging.Logger):

    def outer_wrapper(func):

        function_name = func.__name__

        print('function_name', function_name)

        HISTOGRAM_DURATION : Histogram = metrics.get('{:s}_HISTOGRAM_DURATION'.format(function_name).upper())

        COUNTER_STARTED    : Counter   = metrics.get('{:s}_COUNTER_STARTED'   .format(function_name).upper())

        COUNTER_COMPLETED  : Counter   = metrics.get('{:s}_COUNTER_COMPLETED' .format(function_name).upper())

        COUNTER_FAILED     : Counter   = metrics.get('{:s}_COUNTER_FAILED'    .format(function_name).upper())



        @HISTOGRAM_DURATION.time()

        def inner_wrapper(self, request, grpc_context : grpc.ServicerContext):

            COUNTER_STARTED.inc()

            try:

                logger.debug('{:s} request: {:s}'.format(function_name, str(request)))

                reply = func(self, request, grpc_context)

                logger.debug('{:s} reply: {:s}'.format(function_name, str(reply)))

                COUNTER_COMPLETED.inc()

                return reply

            except ServiceException as e:                               # pragma: no cover (ServiceException not thrown)

                logger.exception('{:s} exception'.format(function_name))

                COUNTER_FAILED.inc()

                grpc_context.abort(e.code, e.details)

            except Exception as e:                                      # pragma: no cover

                logger.exception('{:s} exception'.format(function_name))

                COUNTER_FAILED.inc()

                grpc_context.abort(grpc.StatusCode.INTERNAL, str(e))

        return inner_wrapper

    return outer_wrapper

src/common/metrics/__init__.py

0 → 100644
+0 −0

Empty file added.

src/common/metrics/tests/__init__.py

0 → 100644
+0 −0

Empty file added.