Commit 131f870b authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Monitoring component: 

- updated and migrated unitary test
parent f173762e

src/monitoring/tests/test_unitary.py

+83 −139
Original line number Diff line number Diff line
@@ -12,58 +12,48 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import copy, os, pytest

import threading

import time

from queue import Queue

import copy, os, pytest #, threading, time

import logging

#from queue import Queue

from random import random

from time import sleep

from typing import Tuple



from typing import Union #, Tuple

from apscheduler.executors.pool import ProcessPoolExecutor

from apscheduler.schedulers.background import BackgroundScheduler

from apscheduler.schedulers.base import STATE_STOPPED

from grpc._channel import _MultiThreadedRendezvous



from common.Constants import ServiceNameEnum

from common.Settings import (

    ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc)

from common.logger import getJSONLogger

from common.orm.Database import Database

from common.orm.Factory import get_database_backend, BackendEnum as DatabaseBackendEnum

from common.message_broker.Factory import get_messagebroker_backend, BackendEnum as MessageBrokerBackendEnum

from common.message_broker.MessageBroker import MessageBroker

#from common.logger import getJSONLogger

from common.proto import monitoring_pb2

from common.proto.context_pb2 import EventTypeEnum, DeviceEvent, Device, Empty

from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server

from common.proto.kpi_sample_types_pb2 import KpiSampleType

from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, KpiList, SubsDescriptor, SubsList, AlarmID, \

    AlarmDescriptor, AlarmList, Kpi, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable

from common.tools.timestamp.Converters import timestamp_utcnow_to_float, timestamp_string_to_float



from common.proto.monitoring_pb2 import KpiId, KpiDescriptor, SubsDescriptor, SubsList, AlarmID, \

    AlarmDescriptor, AlarmList, KpiDescriptorList, SubsResponse, AlarmResponse, RawKpiTable #, Kpi, KpiList

from common.tests.MockServicerImpl_Context import MockServicerImpl_Context

from common.tools.service.GenericGrpcService import GenericGrpcService

from common.tools.timestamp.Converters import timestamp_utcnow_to_float #, timestamp_string_to_float

from context.client.ContextClient import ContextClient

from context.service.grpc_server.ContextService import ContextService

from common.proto.context_pb2 import EventTypeEnum, DeviceEvent, Device, Empty



from context.service.ContextService import ContextService

from device.client.DeviceClient import DeviceClient

from device.service.DeviceService import DeviceService

from device.service.driver_api.DriverFactory import DriverFactory

from device.service.driver_api.DriverInstanceCache import DriverInstanceCache

from monitoring.service.AlarmManager import AlarmManager

from monitoring.service.MetricsDBTools import MetricsDB

from monitoring.service.SubscriptionManager import SubscriptionManager



os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE'

from device.service.drivers import DRIVERS  # pylint: disable=wrong-import-position



from monitoring.client.MonitoringClient import MonitoringClient

from monitoring.service import ManagementDBTools, MetricsDBTools

from monitoring.service.MonitoringService import MonitoringService

#from monitoring.service.AlarmManager import AlarmManager

from monitoring.service.EventTools import EventsDeviceCollector

from monitoring.service.MetricsDBTools import MetricsDB

from monitoring.service.MonitoringService import MonitoringService

#from monitoring.service.SubscriptionManager import SubscriptionManager

from monitoring.tests.Messages import create_kpi_request, include_kpi_request, monitor_kpi_request, \

    create_kpi_request_b, create_kpi_request_c, kpi_query, subs_descriptor, alarm_descriptor, \

    alarm_subscription

    create_kpi_request_c, kpi_query, subs_descriptor, alarm_descriptor, alarm_subscription #, create_kpi_request_b

from monitoring.tests.Objects import DEVICE_DEV1, DEVICE_DEV1_CONNECT_RULES, DEVICE_DEV1_UUID



from monitoring.service.MonitoringServiceServicerImpl import LOGGER

os.environ['DEVICE_EMULATED_ONLY'] = 'TRUE'

from device.service.drivers import DRIVERS  # pylint: disable=wrong-import-position,ungrouped-imports





###########################
@@ -71,49 +61,54 @@ from monitoring.service.MonitoringServiceServicerImpl import LOGGER 
###########################



LOCAL_HOST = '127.0.0.1'

MOCKSERVICE_PORT = 10000



CONTEXT_SERVICE_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.CONTEXT) # avoid privileged ports

os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST     )] = str(LOCAL_HOST)

os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(CONTEXT_SERVICE_PORT)



DEVICE_SERVICE_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.DEVICE) # avoid privileged ports

DEVICE_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.DEVICE) # avoid privileged ports

os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST     )] = str(LOCAL_HOST)

os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DEVICE_SERVICE_PORT)



MONITORING_SERVICE_PORT = 10000 + get_service_port_grpc(ServiceNameEnum.MONITORING) # avoid privileged ports

MONITORING_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.MONITORING) # avoid privileged ports

os.environ[get_env_var_name(ServiceNameEnum.MONITORING, ENVVAR_SUFIX_SERVICE_HOST     )] = str(LOCAL_HOST)

os.environ[get_env_var_name(ServiceNameEnum.MONITORING, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(MONITORING_SERVICE_PORT)



METRICSDB_HOSTNAME = os.environ.get("METRICSDB_HOSTNAME")

METRICSDB_ILP_PORT = os.environ.get("METRICSDB_ILP_PORT")

METRICSDB_REST_PORT = os.environ.get("METRICSDB_REST_PORT")

METRICSDB_TABLE = os.environ.get("METRICSDB_TABLE")

METRICSDB_HOSTNAME = os.environ.get('METRICSDB_HOSTNAME')

METRICSDB_ILP_PORT = os.environ.get('METRICSDB_ILP_PORT')

METRICSDB_REST_PORT = os.environ.get('METRICSDB_REST_PORT')

METRICSDB_TABLE = os.environ.get('METRICSDB_TABLE')



LOGGER = logging.getLogger(__name__)



@pytest.fixture(scope='session')

def context_db_mb() -> Tuple[Database, MessageBroker]:

    _database = Database(get_database_backend(backend=DatabaseBackendEnum.INMEMORY))

    _message_broker = MessageBroker(get_messagebroker_backend(backend=MessageBrokerBackendEnum.INMEMORY))

    yield _database, _message_broker

    _message_broker.terminate()

class MockService_Dependencies(GenericGrpcService):

    # Mock Service implementing Context and Device to simplify unitary tests of Monitoring



    def __init__(self, bind_port: Union[str, int]) -> None:

        super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService')



    # pylint: disable=attribute-defined-outside-init

    def install_servicers(self):

        self.context_servicer = MockServicerImpl_Context()

        add_ContextServiceServicer_to_server(self.context_servicer, self.server)



    def configure_env_vars(self):

        os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST     )] = str(self.bind_address)

        os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(self.bind_port)



@pytest.fixture(scope='session')

def context_service(context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name

    database, message_broker = context_db_mb

    database.clear_all()

    _service = ContextService(database, message_broker)

def context_service():

    _service = MockService_Dependencies(MOCKSERVICE_PORT)

    _service.configure_env_vars()

    _service.start()

    yield _service

    _service.stop()



@pytest.fixture(scope='session')

def context_client(context_service : ContextService): # pylint: disable=redefined-outer-name

def context_client(context_service : ContextService): # pylint: disable=redefined-outer-name,unused-argument

    _client = ContextClient()

    yield _client

    _client.close()



@pytest.fixture(scope='session')

def device_service(context_service : ContextService): # pylint: disable=redefined-outer-name

def device_service(context_service : ContextService): # pylint: disable=redefined-outer-name,unused-argument

    LOGGER.info('Initializing DeviceService...')

    driver_factory = DriverFactory(DRIVERS)

    driver_instance_cache = DriverInstanceCache(driver_factory)
@@ -128,7 +123,7 @@ def device_service(context_service : ContextService): # pylint: disable=redefine 
    _service.stop()



@pytest.fixture(scope='session')

def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name

def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name,unused-argument

    _client = DeviceClient()

    yield _client

    _client.close()
@@ -136,8 +131,8 @@ def device_client(device_service : DeviceService): # pylint: disable=redefined-o 
# This fixture will be requested by test cases and last during testing session

@pytest.fixture(scope='session')

def monitoring_service(

        context_service : ContextService,  # pylint: disable=redefined-outer-name

        device_service : DeviceService     # pylint: disable=redefined-outer-name

        context_service : ContextService,  # pylint: disable=redefined-outer-name,unused-argument

        device_service : DeviceService     # pylint: disable=redefined-outer-name,unused-argument

    ):

    LOGGER.info('Initializing MonitoringService...')

    _service = MonitoringService()
@@ -153,7 +148,7 @@ def monitoring_service( 
# This fixture will be requested by test cases and last during testing session.

# The client requires the server, so client fixture has the server as dependency.

@pytest.fixture(scope='session')

def monitoring_client(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name

def monitoring_client(monitoring_service : MonitoringService): # pylint: disable=redefined-outer-name,unused-argument

    LOGGER.info('Initializing MonitoringClient...')

    _client = MonitoringClient()
@@ -183,10 +178,13 @@ def subs_scheduler(): 
    return _scheduler



def ingestion_data(kpi_id_int):

    metrics_db = MetricsDB("localhost", "9009", "9000", "monitoring")

    # pylint: disable=redefined-outer-name,unused-argument

    metrics_db = MetricsDB('localhost', '9009', '9000', 'monitoring')



    for i in range(50):

        kpiSampleType   = KpiSampleType.Name(KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED).upper().replace('KPISAMPLETYPE_', '')

    kpiSampleType = KpiSampleType.KPISAMPLETYPE_PACKETS_RECEIVED

    kpiSampleType_name = KpiSampleType.Name(kpiSampleType).upper().replace('KPISAMPLETYPE_', '')

    for _ in range(50):

        kpiSampleType   = kpiSampleType_name

        kpiId           = kpi_id_int

        deviceId        = 'DEV'+ str(kpi_id_int)

        endpointId      = 'END' + str(kpi_id_int)
@@ -250,26 +248,12 @@ def test_include_kpi(monitoring_client): # pylint: disable=redefined-outer-name 


# Test case that makes use of client fixture to test server's MonitorKpi method

def test_monitor_kpi(

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name,unused-argument

        device_client : DeviceClient,                   # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name

        context_db_mb : Tuple[Database, MessageBroker]  # pylint: disable=redefined-outer-name

    ):

    LOGGER.info('test_monitor_kpi begin')



    context_database = context_db_mb[0]



    # ----- Clean the database -----------------------------------------------------------------------------------------

    context_database.clear_all()



    # ----- Dump state of database before create the object ------------------------------------------------------------

    db_entries = context_database.dump()

    LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries)))

    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 0



    # ----- Update the object ------------------------------------------------------------------------------------------

    LOGGER.info('Adding Device {:s}'.format(DEVICE_DEV1_UUID))

    device_with_connect_rules = copy.deepcopy(DEVICE_DEV1)
@@ -313,7 +297,7 @@ def test_set_kpi_subscription(monitoring_client,subs_scheduler): # pylint: disab 
        subs_scheduler.shutdown()



# Test case that makes use of client fixture to test server's GetSubsDescriptor method

def test_get_subs_descriptor(monitoring_client):

def test_get_subs_descriptor(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_get_subs_descriptor')

    kpi_id = monitoring_client.SetKpi(create_kpi_request_c())

    monitoring_client.IncludeKpi(include_kpi_request(kpi_id))
@@ -324,14 +308,14 @@ def test_get_subs_descriptor(monitoring_client): 
        assert isinstance(response, SubsDescriptor)



# Test case that makes use of client fixture to test server's GetSubscriptions method

def test_get_subscriptions(monitoring_client):

def test_get_subscriptions(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_get_subscriptions')

    response = monitoring_client.GetSubscriptions(Empty())

    LOGGER.debug(response)

    assert isinstance(response, SubsList)



# Test case that makes use of client fixture to test server's DeleteSubscription method

def test_delete_subscription(monitoring_client):

def test_delete_subscription(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_delete_subscription')

    kpi_id = monitoring_client.SetKpi(create_kpi_request_c())

    monitoring_client.IncludeKpi(include_kpi_request(kpi_id))
@@ -341,7 +325,7 @@ def test_delete_subscription(monitoring_client): 
        assert isinstance(response, Empty)



# Test case that makes use of client fixture to test server's SetKpiAlarm method

def test_set_kpi_alarm(monitoring_client):

def test_set_kpi_alarm(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_set_kpi_alarm')

    kpi_id = monitoring_client.SetKpi(create_kpi_request_c())

    response = monitoring_client.SetKpiAlarm(alarm_descriptor(kpi_id))
@@ -349,14 +333,14 @@ def test_set_kpi_alarm(monitoring_client): 
    assert isinstance(response, AlarmID)



# Test case that makes use of client fixture to test server's GetAlarms method

def test_get_alarms(monitoring_client):

def test_get_alarms(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_get_alarms')

    response = monitoring_client.GetAlarms(Empty())

    LOGGER.debug(response)

    assert isinstance(response, AlarmList)



# Test case that makes use of client fixture to test server's GetAlarmDescriptor method

def test_get_alarm_descriptor(monitoring_client):

def test_get_alarm_descriptor(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_get_alarm_descriptor')

    _kpi_id = monitoring_client.SetKpi(create_kpi_request_c())

    _alarm_id = monitoring_client.SetKpiAlarm(alarm_descriptor(_kpi_id))
@@ -365,7 +349,7 @@ def test_get_alarm_descriptor(monitoring_client): 
    assert isinstance(_response, AlarmDescriptor)



# Test case that makes use of client fixture to test server's GetAlarmResponseStream method

def test_get_alarm_response_stream(monitoring_client,subs_scheduler):

def test_get_alarm_response_stream(monitoring_client,subs_scheduler): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_get_alarm_descriptor')

    _kpi_id = monitoring_client.SetKpi(create_kpi_request('3'))

    _alarm_id = monitoring_client.SetKpiAlarm(alarm_descriptor(_kpi_id))
@@ -380,7 +364,7 @@ def test_get_alarm_response_stream(monitoring_client,subs_scheduler): 
        subs_scheduler.shutdown()



# Test case that makes use of client fixture to test server's DeleteAlarm method

def test_delete_alarm(monitoring_client):

def test_delete_alarm(monitoring_client): # pylint: disable=redefined-outer-name

    LOGGER.warning('test_delete_alarm')

    _kpi_id = monitoring_client.SetKpi(create_kpi_request_c())

    _alarm_id = monitoring_client.SetKpiAlarm(alarm_descriptor(_kpi_id))
@@ -413,10 +397,12 @@ def test_managementdb_tools_kpis(management_db): # pylint: disable=redefined-out 
    kpi_device_id      = _create_kpi_request.device_id.device_uuid.uuid         # pylint: disable=maybe-no-member

    kpi_endpoint_id    = _create_kpi_request.endpoint_id.endpoint_uuid.uuid     # pylint: disable=maybe-no-member

    kpi_service_id     = _create_kpi_request.service_id.service_uuid.uuid       # pylint: disable=maybe-no-member

    kpi_slice_id = _create_kpi_request.slice_id.slice_uuid.uuid

    kpi_connection_id  = _create_kpi_request.connection_id.connection_uuid.uuid

    kpi_slice_id       = _create_kpi_request.slice_id.slice_uuid.uuid           # pylint: disable=maybe-no-member

    kpi_connection_id  = _create_kpi_request.connection_id.connection_uuid.uuid # pylint: disable=maybe-no-member



    _kpi_id = management_db.insert_KPI(kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, kpi_service_id,kpi_slice_id,kpi_connection_id)

    _kpi_id = management_db.insert_KPI(

        kpi_description, kpi_sample_type, kpi_device_id, kpi_endpoint_id, kpi_service_id,

        kpi_slice_id, kpi_connection_id)

    assert isinstance(_kpi_id, int)



    response = management_db.get_KPI(_kpi_id)
@@ -517,30 +503,16 @@ def test_managementdb_tools_insert_alarm(management_db): 
#     assert total_points != 0



def test_events_tools(

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name,unused-argument

        device_client : DeviceClient,                   # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name

        context_db_mb : Tuple[Database, MessageBroker]  # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name,unused-argument

    ):

    LOGGER.warning('test_get_device_events begin')



    context_database = context_db_mb[0]



    # ----- Clean the database -----------------------------------------------------------------------------------------

    context_database.clear_all()



    # ----- Initialize the EventsCollector -----------------------------------------------------------------------------

    events_collector = EventsDeviceCollector()

    events_collector.start()



    # ----- Dump state of database before create the object ------------------------------------------------------------

    db_entries = context_database.dump()

    LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries)))

    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 0



    # ----- Update the object ------------------------------------------------------------------------------------------

    LOGGER.info('Adding Device {:s}'.format(DEVICE_DEV1_UUID))

    device_with_connect_rules = copy.deepcopy(DEVICE_DEV1)
@@ -552,31 +524,17 @@ def test_events_tools( 




def test_get_device_events(

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name,unused-argument

        device_client : DeviceClient,                   # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name

        context_db_mb : Tuple[Database, MessageBroker]  # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name,unused-argument

    ):



    LOGGER.warning('test_get_device_events begin')



    context_database = context_db_mb[0]



    # ----- Clean the database -----------------------------------------------------------------------------------------

    context_database.clear_all()



    # ----- Initialize the EventsCollector -----------------------------------------------------------------------------

    events_collector = EventsDeviceCollector()

    events_collector.start()



    # ----- Dump state of database before create the object ------------------------------------------------------------

    db_entries = context_database.dump()

    LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries)))

    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 0



    # ----- Check create event -----------------------------------------------------------------------------------------

    LOGGER.info('Adding Device {:s}'.format(DEVICE_DEV1_UUID))

    device_with_connect_rules = copy.deepcopy(DEVICE_DEV1)
@@ -592,31 +550,17 @@ def test_get_device_events( 
    events_collector.stop()



def test_listen_events(

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name

        context_client : ContextClient,                 # pylint: disable=redefined-outer-name,unused-argument

        device_client : DeviceClient,                   # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name

        context_db_mb : Tuple[Database, MessageBroker]  # pylint: disable=redefined-outer-name

        monitoring_client : MonitoringClient,           # pylint: disable=redefined-outer-name,unused-argument

    ):



    LOGGER.warning('test_listen_events begin')



    context_database = context_db_mb[0]



    # ----- Clean the database -----------------------------------------------------------------------------------------

    context_database.clear_all()



    # ----- Initialize the EventsCollector -----------------------------------------------------------------------------

    events_collector = EventsDeviceCollector()

    events_collector.start()



    # ----- Dump state of database before create the object ------------------------------------------------------------

    db_entries = context_database.dump()

    LOGGER.info('----- Database Dump [{:3d} entries] -------------------------'.format(len(db_entries)))

    for db_entry in db_entries:

        LOGGER.info('  [{:>4s}] {:40s} :: {:s}'.format(*db_entry)) # pragma: no cover

    LOGGER.info('-----------------------------------------------------------')

    assert len(db_entries) == 0



    LOGGER.info('Adding Device {:s}'.format(DEVICE_DEV1_UUID))

    device_with_connect_rules = copy.deepcopy(DEVICE_DEV1)

    device_with_connect_rules['device_config']['config_rules'].extend(DEVICE_DEV1_CONNECT_RULES)