test_unitary_microwave.py

# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import calendar, copy, dateutil.parser, grpc, json, logging, operator, os, pytest, queue, time
from datetime import datetime, timezone
from typing import Tuple
from common.orm.Database import Database
from common.orm.Factory import get_database_backend, BackendEnum as DatabaseBackendEnum
from common.message_broker.Factory import get_messagebroker_backend, BackendEnum as MessageBrokerBackendEnum
from common.message_broker.MessageBroker import MessageBroker
from common.tools.grpc.Tools import grpc_message_to_json_string
from common.tools.object_factory.EndPoint import json_endpoint, json_endpoint_id
from context.Config import (
    GRPC_SERVICE_PORT as CONTEXT_GRPC_SERVICE_PORT, GRPC_MAX_WORKERS as CONTEXT_GRPC_MAX_WORKERS,
    GRPC_GRACE_PERIOD as CONTEXT_GRPC_GRACE_PERIOD)
from context.client.ContextClient import ContextClient
from context.proto.context_pb2 import DeviceId, DeviceOperationalStatusEnum
from context.service.grpc_server.ContextService import ContextService
from device.Config import (
    GRPC_SERVICE_PORT as DEVICE_GRPC_SERVICE_PORT, GRPC_MAX_WORKERS as DEVICE_GRPC_MAX_WORKERS,
    GRPC_GRACE_PERIOD as DEVICE_GRPC_GRACE_PERIOD)
from device.client.DeviceClient import DeviceClient
from device.proto.context_pb2 import ConfigActionEnum, Context, Device, Topology
from device.proto.device_pb2 import MonitoringSettings
from device.proto.kpi_sample_types_pb2 import KpiSampleType
from device.service.DeviceService import DeviceService
from device.service.driver_api._Driver import _Driver
from device.service.driver_api.DriverFactory import DriverFactory
from device.service.driver_api.DriverInstanceCache import DriverInstanceCache
from device.service.drivers import DRIVERS
from device.tests.MockMonitoringService import MockMonitoringService
from monitoring.Config import (
    GRPC_SERVICE_PORT as MONITORING_GRPC_SERVICE_PORT, GRPC_MAX_WORKERS as MONITORING_GRPC_MAX_WORKERS,
    GRPC_GRACE_PERIOD as MONITORING_GRPC_GRACE_PERIOD)
from monitoring.client.monitoring_client import MonitoringClient
from .CommonObjects import CONTEXT, TOPOLOGY

from .Device_Emulated import (
    DEVICE_EMU, DEVICE_EMU_CONFIG_ADDRESSES, DEVICE_EMU_CONFIG_ENDPOINTS, DEVICE_EMU_CONNECT_RULES,
    DEVICE_EMU_DECONFIG_ADDRESSES, DEVICE_EMU_DECONFIG_ENDPOINTS, DEVICE_EMU_EP_DESCS, DEVICE_EMU_ENDPOINTS_COOKED,
    DEVICE_EMU_ID, DEVICE_EMU_RECONFIG_ADDRESSES, DEVICE_EMU_UUID)
ENABLE_EMULATED = True

try:
    from .Device_OpenConfig_Infinera1 import(
    #from .Device_OpenConfig_Infinera2 import(
        DEVICE_OC, DEVICE_OC_CONFIG_RULES, DEVICE_OC_DECONFIG_RULES, DEVICE_OC_CONNECT_RULES, DEVICE_OC_ID,
        DEVICE_OC_UUID)
    ENABLE_OPENCONFIG = True
except ImportError:
    ENABLE_OPENCONFIG = False

try:
    from .Device_Transport_Api_CTTC import (
        DEVICE_TAPI, DEVICE_TAPI_CONNECT_RULES, DEVICE_TAPI_UUID, DEVICE_TAPI_ID, DEVICE_TAPI_CONFIG_RULES,
        DEVICE_TAPI_DECONFIG_RULES)
    ENABLE_TAPI = True
except ImportError:
    ENABLE_TAPI = False

try:
    from .Device_Microwave_Template import (
        DEVICE_MICROWAVE, DEVICE_MICROWAVE_CONNECT_RULES, DEVICE_MICROWAVE_UUID, DEVICE_MICROWAVE_ID, DEVICE_MICROWAVE_CONFIG_RULES,
        DEVICE_MICROWAVE_DECONFIG_RULES)
    ENABLE_MICROWAVE = True
except ImportError as error:
    ENABLE_MICROWAVE = False
    print(error.__class__.__name__ + ": " + error.message)

from .mock_p4runtime_service import MockP4RuntimeService
try:
    from .device_p4 import(
        DEVICE_P4, DEVICE_P4_ID, DEVICE_P4_UUID, DEVICE_P4_NAME, DEVICE_P4_ADDRESS, DEVICE_P4_PORT, DEVICE_P4_WORKERS,
        DEVICE_P4_GRACE_PERIOD, DEVICE_P4_CONNECT_RULES, DEVICE_P4_CONFIG_RULES)
    ENABLE_P4 = True
except ImportError:
    ENABLE_P4 = False

#ENABLE_EMULATED   = False # set to False to disable tests of Emulated devices
#ENABLE_OPENCONFIG = False # set to False to disable tests of OpenConfig devices
#ENABLE_TAPI       = False # set to False to disable tests of TAPI devices
#ENABLE_P4         = False # set to False to disable tests of P4 devices

ENABLE_OPENCONFIG_CONFIGURE   = True
ENABLE_OPENCONFIG_MONITOR     = True
ENABLE_OPENCONFIG_DECONFIGURE = True


logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING)
logging.getLogger('apscheduler.scheduler').setLevel(logging.WARNING)
logging.getLogger('monitoring-client').setLevel(logging.WARNING)

LOGGER = logging.getLogger(__name__)
LOGGER.setLevel(logging.DEBUG)

CONTEXT_GRPC_SERVICE_PORT = 10000 + CONTEXT_GRPC_SERVICE_PORT # avoid privileged ports
DEVICE_GRPC_SERVICE_PORT = 10000 + DEVICE_GRPC_SERVICE_PORT # avoid privileged ports
MONITORING_GRPC_SERVICE_PORT = 10000 + MONITORING_GRPC_SERVICE_PORT # avoid privileged ports

DEFAULT_REDIS_SERVICE_HOST = '127.0.0.1'
DEFAULT_REDIS_SERVICE_PORT = 6379
DEFAULT_REDIS_DATABASE_ID  = 0

REDIS_CONFIG = {
    'REDIS_SERVICE_HOST': os.environ.get('REDIS_SERVICE_HOST', DEFAULT_REDIS_SERVICE_HOST),
    'REDIS_SERVICE_PORT': os.environ.get('REDIS_SERVICE_PORT', DEFAULT_REDIS_SERVICE_PORT),
    'REDIS_DATABASE_ID' : os.environ.get('REDIS_DATABASE_ID',  DEFAULT_REDIS_DATABASE_ID ),
}

SCENARIOS = [
    ('all_inmemory', DatabaseBackendEnum.INMEMORY, {},           MessageBrokerBackendEnum.INMEMORY, {}          ),
    #('all_redis',    DatabaseBackendEnum.REDIS,    REDIS_CONFIG, MessageBrokerBackendEnum.REDIS,    REDIS_CONFIG),
]

@pytest.fixture(scope='session', ids=[str(scenario[0]) for scenario in SCENARIOS], params=SCENARIOS)
def context_db_mb(request) -> Tuple[Database, MessageBroker]:
    name,db_backend,db_settings,mb_backend,mb_settings = request.param
    msg = 'Running scenario {:s} db_backend={:s}, db_settings={:s}, mb_backend={:s}, mb_settings={:s}...'
    LOGGER.info(msg.format(str(name), str(db_backend.value), str(db_settings), str(mb_backend.value), str(mb_settings)))
    _database = Database(get_database_backend(backend=db_backend, **db_settings))
    _message_broker = MessageBroker(get_messagebroker_backend(backend=mb_backend, **mb_settings))
    yield _database, _message_broker
    _message_broker.terminate()

@pytest.fixture(scope='session')
def context_service(context_db_mb : Tuple[Database, MessageBroker]): # pylint: disable=redefined-outer-name
    _service = ContextService(
        context_db_mb[0], context_db_mb[1], port=CONTEXT_GRPC_SERVICE_PORT, max_workers=CONTEXT_GRPC_MAX_WORKERS,
        grace_period=CONTEXT_GRPC_GRACE_PERIOD)
    _service.start()
    yield _service
    _service.stop()

@pytest.fixture(scope='session')
def context_client(context_service : ContextService): # pylint: disable=redefined-outer-name
    _client = ContextClient(address='127.0.0.1', port=CONTEXT_GRPC_SERVICE_PORT)
    yield _client
    _client.close()

@pytest.fixture(scope='session')
def monitoring_service():
    _service = MockMonitoringService(port=MONITORING_GRPC_SERVICE_PORT, max_workers=MONITORING_GRPC_MAX_WORKERS,
        grace_period=MONITORING_GRPC_GRACE_PERIOD)
    _service.start()
    yield _service
    _service.stop()

@pytest.fixture(scope='session')
def monitoring_client(monitoring_service : MockMonitoringService): # pylint: disable=redefined-outer-name
    _client = MonitoringClient(server='127.0.0.1', port=MONITORING_GRPC_SERVICE_PORT)
    #yield _client
    #_client.close()
    return _client

@pytest.fixture(scope='session')
def device_service(
    context_client : ContextClient,         # pylint: disable=redefined-outer-name
    monitoring_client : MonitoringClient):  # pylint: disable=redefined-outer-name

    _driver_factory = DriverFactory(DRIVERS)
    _driver_instance_cache = DriverInstanceCache(_driver_factory)
    _service = DeviceService(
        context_client, monitoring_client, _driver_instance_cache, port=DEVICE_GRPC_SERVICE_PORT,
        max_workers=DEVICE_GRPC_MAX_WORKERS, grace_period=DEVICE_GRPC_GRACE_PERIOD)
    _service.start()
    yield _service
    _service.stop()

@pytest.fixture(scope='session')
def device_client(device_service : DeviceService): # pylint: disable=redefined-outer-name
    _client = DeviceClient(address='127.0.0.1', port=DEVICE_GRPC_SERVICE_PORT)
    yield _client
    _client.close()

@pytest.fixture(scope='session')
def p4runtime_service():
    _service = MockP4RuntimeService(
        address=DEVICE_P4_ADDRESS, port=DEVICE_P4_PORT,
        max_workers=DEVICE_P4_WORKERS,
        grace_period=DEVICE_P4_GRACE_PERIOD)
    _service.start()
    yield _service
    _service.stop()


def test_prepare_environment(
    context_client : ContextClient,     # pylint: disable=redefined-outer-name
    device_client : DeviceClient,       # pylint: disable=redefined-outer-name
    device_service : DeviceService):    # pylint: disable=redefined-outer-name

    context_client.SetContext(Context(**CONTEXT))
    context_client.SetTopology(Topology(**TOPOLOGY))




# ----- Test Device Driver Microwave ------------------------------------------------

def test_device_microwave_add_correct(
    device_client: DeviceClient,        # pylint: disable=redefined-outer-name
    device_service: DeviceService):     # pylint: disable=redefined-outer-name

    if not ENABLE_MICROWAVE: pytest.skip('Skipping test: No MICROWAVE device has been configured')
    
    DEVICE_MICROWAVE_WITH_CONNECT_RULES = copy.deepcopy(DEVICE_MICROWAVE)
    DEVICE_MICROWAVE_WITH_CONNECT_RULES['device_config']['config_rules'].extend(DEVICE_MICROWAVE_CONNECT_RULES)
    device_client.AddDevice(Device(**DEVICE_MICROWAVE_WITH_CONNECT_RULES))
    driver: _Driver = device_service.driver_instance_cache.get(DEVICE_MICROWAVE_UUID)
    assert driver is not None


def test_device_microwave_get(
    context_client: ContextClient,      # pylint: disable=redefined-outer-name
    device_client: DeviceClient):       # pylint: disable=redefined-outer-name

    if not ENABLE_MICROWAVE: pytest.skip('Skipping test: No MICROWAVE device has been configured')

    initial_config = device_client.GetInitialConfig(DeviceId(**DEVICE_MICROWAVE_ID))
    LOGGER.info('initial_config = {:s}'.format(grpc_message_to_json_string(initial_config)))

    device_data = context_client.GetDevice(DeviceId(**DEVICE_MICROWAVE_ID))
    LOGGER.info('device_data = {:s}'.format(grpc_message_to_json_string(device_data)))


def test_device_microwave_configure(
    context_client: ContextClient,      # pylint: disable=redefined-outer-name
    device_client: DeviceClient,        # pylint: disable=redefined-outer-name
    device_service: DeviceService):     # pylint: disable=redefined-outer-name

    if not ENABLE_MICROWAVE: pytest.skip('Skipping test: No MICROWAVE device has been configured')

    driver : _Driver = device_service.driver_instance_cache.get(DEVICE_MICROWAVE_UUID)
    assert driver is not None

    # Requires to retrieve data from device; might be slow. Uncomment only when needed and test does not pass directly.
    #driver_config = sorted(driver.GetConfig(), key=operator.itemgetter(0))
    #LOGGER.info('driver_config = {:s}'.format(str(driver_config)))

    DEVICE_MICROWAVE_WITH_CONFIG_RULES = copy.deepcopy(DEVICE_MICROWAVE)
    DEVICE_MICROWAVE_WITH_CONFIG_RULES['device_config']['config_rules'].extend(DEVICE_MICROWAVE_CONFIG_RULES)
    device_client.ConfigureDevice(Device(**DEVICE_MICROWAVE_WITH_CONFIG_RULES))

    # Requires to retrieve data from device; might be slow. Uncomment only when needed and test does not pass directly.
    #driver_config = sorted(driver.GetConfig(), key=operator.itemgetter(0))
    #LOGGER.info('driver_config = {:s}'.format(str(driver_config)))

    device_data = context_client.GetDevice(DeviceId(**DEVICE_MICROWAVE_ID))
    config_rules = [
        (ConfigActionEnum.Name(config_rule.action), config_rule.resource_key, config_rule.resource_value)
        for config_rule in device_data.device_config.config_rules
    ]
    LOGGER.info('device_data.device_config.config_rules = \n{:s}'.format(
        '\n'.join(['{:s} {:s} = {:s}'.format(*config_rule) for config_rule in config_rules])))
    for config_rule in DEVICE_MICROWAVE_CONFIG_RULES:
        #import pdb;
        #pdb. set_trace()
        config_rule = (
            ConfigActionEnum.Name(config_rule['action']), config_rule['resource_key'], config_rule['resource_value'])
        assert config_rule in config_rules


def test_device_microwave_deconfigure(
    context_client: ContextClient,      # pylint: disable=redefined-outer-name
    device_client: DeviceClient,        # pylint: disable=redefined-outer-name
    device_service: DeviceService):     # pylint: disable=redefined-outer-name

    if not ENABLE_MICROWAVE: pytest.skip('Skipping test: No MICROWAVE device has been configured')

    driver: _Driver = device_service.driver_instance_cache.get(DEVICE_MICROWAVE_UUID)
    assert driver is not None

    # Requires to retrieve data from device; might be slow. Uncomment only when needed and test does not pass directly.
    #driver_config = sorted(driver.GetConfig(), key=operator.itemgetter(0))
    #LOGGER.info('driver_config = {:s}'.format(str(driver_config)))

    DEVICE_MICROWAVE_WITH_DECONFIG_RULES = copy.deepcopy(DEVICE_MICROWAVE)
    DEVICE_MICROWAVE_WITH_DECONFIG_RULES['device_config']['config_rules'].extend(DEVICE_MICROWAVE_DECONFIG_RULES)
    device_client.ConfigureDevice(Device(**DEVICE_MICROWAVE_WITH_DECONFIG_RULES))

    # Requires to retrieve data from device; might be slow. Uncomment only when needed and test does not pass directly.
    #driver_config = sorted(driver.GetConfig(), key=operator.itemgetter(0))
    #LOGGER.info('driver_config = {:s}'.format(str(driver_config)))

    device_data = context_client.GetDevice(DeviceId(**DEVICE_MICROWAVE_ID))
    config_rules = [
        (ConfigActionEnum.Name(config_rule.action), config_rule.resource_key, config_rule.resource_value)
        for config_rule in device_data.device_config.config_rules
    ]
    LOGGER.info('device_data.device_config.config_rules = \n{:s}'.format(
        '\n'.join(['{:s} {:s} = {:s}'.format(*config_rule) for config_rule in config_rules])))
    for config_rule in DEVICE_MICROWAVE_DECONFIG_RULES:
        action_set = ConfigActionEnum.Name(ConfigActionEnum.CONFIGACTION_SET)
        config_rule = (action_set, config_rule['resource_key'], config_rule['resource_value'])
        assert config_rule not in config_rules


def test_device_microwave_delete(
    device_client : DeviceClient,       # pylint: disable=redefined-outer-name
    device_service : DeviceService):    # pylint: disable=redefined-outer-name

    if not ENABLE_MICROWAVE: pytest.skip('Skipping test: No MICROWAVE device has been configured')

    device_client.DeleteDevice(DeviceId(**DEVICE_MICROWAVE_ID))
    driver : _Driver = device_service.driver_instance_cache.get(DEVICE_MICROWAVE_UUID, {})
    assert driver is None