Commit 2a29d661 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

OFC25 test: 

- Upgraded fixtures to support different profiles per TFS instance
- Upgraded bootstrap/cleanup scripts to support multiple profiles
- Parametrized tests
- Adapted test scripts
parent f2504d33

src/tests/ofc25/Dockerfile

+33 −16
Original line number Diff line number Diff line
@@ -91,35 +91,52 @@ set -e 
export PYTHONPATH=/var/teraflow



echo "Initialize optical layer controller"

source /var/teraflow/tfs_runtime_env_vars_opt.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_bootstrap_opt.py --junitxml=/opt/results/report_bootstrap_opt.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_bootstrap.py \

    --tfs-runtime-script=tfs_runtime_env_vars_opt.sh \

    --tfs-profile=opt \

    --tfs-topology-descriptor=topology_opt.json \

    --junitxml=/opt/results/report_bootstrap_opt.xml



echo "Initialize IP/packet layer controller"

source /var/teraflow/tfs_runtime_env_vars_ip.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_bootstrap_ip.py --junitxml=/opt/results/report_bootstrap_ip.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_bootstrap.py \

    --tfs-runtime-script=tfs_runtime_env_vars_ip.sh \

    --tfs-profile=ip \

    --tfs-topology-descriptor=topology_ip.json \

    --junitxml=/opt/results/report_bootstrap_ip.xml



echo "Initialize E2E layer orchestrator"

source /var/teraflow/tfs_runtime_env_vars_e2e.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_bootstrap_e2e.py --junitxml=/opt/results/report_bootstrap_e2e.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_bootstrap.py \

    --tfs-runtime-script=tfs_runtime_env_vars_e2e.sh \

    --tfs-profile=e2e \

    --tfs-topology-descriptor=topology_e2e.json \

    --junitxml=/opt/results/report_bootstrap_e2e.xml



echo "Create IP virtual links"

source /var/teraflow/tfs_runtime_env_vars_ip.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_create_service.py --junitxml=/opt/results/report_create_service.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_create_vlinks.py --junitxml=/opt/results/report_create_service.xml



echo "Delete IP virtual links"

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_delete_service.py --junitxml=/opt/results/report_delete_service.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_delete_vlinks.py --junitxml=/opt/results/report_delete_service.xml



echo "Cleanup E2E layer orchestrator"

source /var/teraflow/tfs_runtime_env_vars_e2e.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_cleanup_e2e.py --junitxml=/opt/results/report_cleanup_e2e.xml

echo "Cleanup E2E layer orchestrator first"

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_cleanup.py \

    --tfs-runtime-script=tfs_runtime_env_vars_e2e.sh \

    --tfs-profile=e2e \

    --tfs-topology-descriptor=topology_e2e.json \

    --junitxml=/opt/results/report_cleanup_e2e.xml



echo "Cleanup IP/packet layer controller"

source /var/teraflow/tfs_runtime_env_vars_ip.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_cleanup_ip.py --junitxml=/opt/results/report_cleanup_ip.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_cleanup.py \

    --tfs-runtime-script=tfs_runtime_env_vars_ip.sh \

    --tfs-profile=ip \

    --tfs-topology-descriptor=topology_ip.json \

    --junitxml=/opt/results/report_cleanup_ip.xml



echo "Cleanup optical layer controller"

source /var/teraflow/tfs_runtime_env_vars_opt.sh

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_cleanup_opt.py --junitxml=/opt/results/report_cleanup_opt.xml

pytest --verbose --log-level=INFO /var/teraflow/tests/ofc25/tests/test_functional_cleanup.py \

    --tfs-runtime-script=tfs_runtime_env_vars_opt.sh \

    --tfs-profile=opt \

    --tfs-topology-descriptor=topology_opt.json \

    --junitxml=/opt/results/report_cleanup_opt.xml



EOF

RUN chmod ug+x ./run_tests.sh

src/tests/ofc25/run_test.sh

+55 −23
Original line number Diff line number Diff line
@@ -26,62 +26,94 @@ case "$TEST_SUITE" in 
        echo "=== Running Full Test Suite ==="



        echo "--- Running Optical Layer Initialization ---"

        source $PROJECTDIR/tfs_runtime_env_vars_opt.sh

        pytest --verbose tests/ofc25/tests/test_functional_cleanup_opt.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap_opt.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap.py \

            --tfs-runtime-script=tfs_runtime_env_vars_opt.sh \

            --tfs-profile=opt \

            --tfs-topology-descriptor=topology_opt.json

        echo "Waiting 5 seconds for initialization ..."

        sleep 5



        echo "--- Running IP/Packet Layer Initialization ---"

        source $PROJECTDIR/tfs_runtime_env_vars_ip.sh

        pytest --verbose tests/ofc25/tests/test_functional_cleanup_ip.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap_ip.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap.py \

            --tfs-runtime-script=tfs_runtime_env_vars_ip.sh \

            --tfs-profile=ip \

            --tfs-topology-descriptor=topology_ip.json

        echo "Waiting 5 seconds for initialization ..."

        sleep 5



        echo "--- Running E2E Layer Initialization ---"

        source $PROJECTDIR/tfs_runtime_env_vars_e2e.sh

        pytest --verbose --log-cli-level=DEBUG tests/ofc25/tests/test_functional_cleanup_e2e.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap_e2e.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap.py \

            --tfs-runtime-script=tfs_runtime_env_vars_e2e.sh \

            --tfs-profile=e2e \

            --tfs-topology-descriptor=topology_e2e.json

        echo "Waiting 5 seconds for initialization ..."

        sleep 5



        echo "--- Running Service Creation/Deletion ---"

        pytest --verbose tests/ofc25/tests/test_functional_create_service.py

        pytest --verbose tests/ofc25/tests/test_functional_create_vlinks.py

        sleep 5

        pytest --verbose tests/ofc25/tests/test_functional_delete_service.py

        pytest --verbose tests/ofc25/tests/test_functional_delete_vlinks.py



        echo "--- Running Cleanup In Reverse Order ---"

        pytest --verbose tests/ofc25/tests/test_functional_cleanup.py \

            --tfs-runtime-script=tfs_runtime_env_vars_e2e.sh \

            --tfs-profile=e2e \

            --tfs-topology-descriptor=topology_e2e.json

        pytest --verbose tests/ofc25/tests/test_functional_cleanup.py \

            --tfs-runtime-script=tfs_runtime_env_vars_ip.sh \

            --tfs-profile=ip \

            --tfs-topology-descriptor=topology_ip.json

        pytest --verbose tests/ofc25/tests/test_functional_cleanup.py \

            --tfs-runtime-script=tfs_runtime_env_vars_opt.sh \

            --tfs-profile=opt \

            --tfs-topology-descriptor=topology_opt.json

        ;;



    "init_opt")

        echo "=== Running Optical Layer Initialization ==="

        source $PROJECTDIR/tfs_runtime_env_vars_opt.sh

        pytest --verbose tests/ofc25/tests/test_functional_cleanup_opt.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap_opt.py

        pytest --verbose tests/ofc25/tests/test_functional_cleanup.py \

            --tfs-runtime-script=tfs_runtime_env_vars_opt.sh \

            --tfs-profile=opt \

            --tfs-topology-descriptor=topology_opt.json

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap.py \

            --tfs-runtime-script=tfs_runtime_env_vars_opt.sh \

            --tfs-profile=opt \

            --tfs-topology-descriptor=topology_opt.json

        ;;



    "init_ip")

        echo "=== Running IP/Packet Layer Initialization ==="

        source $PROJECTDIR/tfs_runtime_env_vars_ip.sh

        pytest --verbose tests/ofc25/tests/test_functional_cleanup_ip.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap_ip.py

        pytest --verbose tests/ofc25/tests/test_functional_cleanup.py \

            --tfs-runtime-script=tfs_runtime_env_vars_ip.sh \

            --tfs-profile=ip \

            --tfs-topology-descriptor=topology_ip.json

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap.py \

            --tfs-runtime-script=tfs_runtime_env_vars_ip.sh \

            --tfs-profile=ip \

            --tfs-topology-descriptor=topology_ip.json

        ;;



    "init_e2e")

        echo "=== Running E2E Layer Initialization ==="

        source $PROJECTDIR/tfs_runtime_env_vars_e2e.sh

        pytest --verbose --log-cli-level=DEBUG tests/ofc25/tests/test_functional_cleanup_e2e.py

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap_e2e.py

        pytest --verbose tests/ofc25/tests/test_functional_cleanup.py \

            --tfs-runtime-script=tfs_runtime_env_vars_e2e.sh \

            --tfs-profile=e2e \

            --tfs-topology-descriptor=topology_e2e.json

        pytest --verbose tests/ofc25/tests/test_functional_bootstrap.py \

            --tfs-runtime-script=tfs_runtime_env_vars_e2e.sh \

            --tfs-profile=e2e \

            --tfs-topology-descriptor=topology_e2e.json

        ;;



    "service")

        echo "=== Running Service Creation/Deletion ==="

        pytest --verbose tests/ofc25/tests/test_functional_create_service.py

        pytest --verbose tests/ofc25/tests/test_functional_create_vlinks.py

        sleep 5

        pytest --verbose tests/ofc25/tests/test_functional_delete_service.py

        pytest --verbose tests/ofc25/tests/test_functional_delete_vlinks.py

        ;;



    *)

        echo "Usage: $0 [opt|ip|e2e|service|all]"

        echo "Usage: $0 [init_opt|init_ip|init_e2e|service|all]"

        echo "  init_opt - Run optical layer initialization only"

        echo "  init_ip  - Run IP/packet layer initialization only"

        echo "  init_e2e - Run E2E orchestration layer initialization only"

src/tests/ofc25/tests/Fixtures.py

+112 −13
Original line number Diff line number Diff line
@@ -12,26 +12,125 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import re

from dataclasses import dataclass

from pathlib import Path

from typing import Dict, Mapping



import pytest



from context.client.ContextClient import ContextClient

from device.client.DeviceClient import DeviceClient

from service.client.ServiceClient import ServiceClient



PROFILE_OPT = 'opt'

PROFILE_IP = 'ip'

PROFILE_E2E = 'e2e'



RUNTIME_ENV_DIR = Path('/var/teraflow')

PROFILE_FILENAMES = {

    PROFILE_OPT: 'tfs_runtime_env_vars_opt.sh',

    PROFILE_IP: 'tfs_runtime_env_vars_ip.sh',

    PROFILE_E2E: 'tfs_runtime_env_vars_e2e.sh',

}

EXPORT_REGEX = re.compile(r'^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$')





@dataclass(frozen=True)

class ServiceEndpoint:

    host: str

    port: int





@dataclass(frozen=True)

class TfsProfile:

    name: str

    env_vars: Mapping[str, str]

    context: ServiceEndpoint

    device: ServiceEndpoint

    service: ServiceEndpoint





@dataclass(frozen=True)

class TfsClientBundle:

    context: ContextClient

    device: DeviceClient

    service: ServiceClient

    env_vars: Mapping[str, str]





def _parse_runtime_env_file(filepath: Path) -> Dict[str, str]:

    env_vars: Dict[str, str] = {}

    for raw_line in filepath.read_text(encoding='utf-8').splitlines():

        line = raw_line.strip()

        if not line or line.startswith('#'):

            continue

        match = EXPORT_REGEX.match(line)

        if match is None:

            continue

        key, value = match.groups()

        value = value.strip()

        if (value.startswith('"') and value.endswith('"')) or \

           (value.startswith("'") and value.endswith("'")):

            value = value[1:-1]

        env_vars[key] = value

    return env_vars





def _read_service_endpoint(env_vars: Mapping[str, str], service_name: str) -> ServiceEndpoint:

    host_key = '{:s}_SERVICE_HOST'.format(service_name)

    port_key = '{:s}_SERVICE_PORT_GRPC'.format(service_name)



    if host_key not in env_vars:

        raise KeyError('Missing key "{:s}" in runtime env vars'.format(host_key))

    if port_key not in env_vars:

        raise KeyError('Missing key "{:s}" in runtime env vars'.format(port_key))



    return ServiceEndpoint(host=env_vars[host_key], port=int(env_vars[port_key]))





def _load_tfs_profile(profile_name: str) -> TfsProfile:

    filepath = RUNTIME_ENV_DIR / PROFILE_FILENAMES[profile_name]

    if not filepath.exists():

        raise FileNotFoundError('Runtime env file not found: {:s}'.format(str(filepath)))



    env_vars = _parse_runtime_env_file(filepath)

    return TfsProfile(

        name=profile_name,

        env_vars=env_vars,

        context=_read_service_endpoint(env_vars, 'CONTEXTSERVICE'),

        device=_read_service_endpoint(env_vars, 'DEVICESERVICE'),

        service=_read_service_endpoint(env_vars, 'SERVICESERVICE'),

    )



@pytest.fixture(scope='session')

def context_client() -> ContextClient:

    _client = ContextClient()

    yield _client

    _client.close()



@pytest.fixture(scope='session')

def device_client() -> DeviceClient:

    _client = DeviceClient()

    yield _client

    _client.close()

def tfs_profiles() -> Dict[str, TfsProfile]:

    profiles: Dict[str, TfsProfile] = {}

    for profile_name in [PROFILE_OPT, PROFILE_IP, PROFILE_E2E]:

        filepath = RUNTIME_ENV_DIR / PROFILE_FILENAMES[profile_name]

        if not filepath.exists():

            continue

        profiles[profile_name] = _load_tfs_profile(profile_name)

    if len(profiles) == 0:

        raise FileNotFoundError('No runtime env files found in {:s}'.format(str(RUNTIME_ENV_DIR)))

    return profiles





@pytest.fixture(scope='session')

def service_client() -> ServiceClient:

    _client = ServiceClient()

    yield _client

    _client.close()
 
def tfs_clients(tfs_profiles: Dict[str, TfsProfile]) -> Dict[str, TfsClientBundle]:

    clients: Dict[str, TfsClientBundle] = {}

    for profile_name, profile in tfs_profiles.items():

        clients[profile_name] = TfsClientBundle(

            context=ContextClient(profile.context.host, profile.context.port),

            device=DeviceClient(profile.device.host, profile.device.port),

            service=ServiceClient(profile.service.host, profile.service.port),

            env_vars=profile.env_vars,

        )



    yield clients



    for bundle in clients.values():

        bundle.context.close()

        bundle.device.close()

        bundle.service.close()

src/tests/ofc25/tests/conftest.py

0 → 100644
+88 −0
Original line number Diff line number Diff line 
# Copyright 2022-2026 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



from pathlib import Path



import pytest



from .Fixtures import PROFILE_FILENAMES, PROFILE_E2E, PROFILE_IP, PROFILE_OPT, RUNTIME_ENV_DIR, tfs_clients





def pytest_addoption(parser):

    parser.addoption(

        '--tfs-profile',

        action='store',

        choices=[PROFILE_OPT, PROFILE_IP, PROFILE_E2E],

        default=None,

        help='TFS deployment profile to use (opt|ip|e2e).',

    )

    parser.addoption(

        '--tfs-runtime-script',

        action='store',

        default=None,

        help='Runtime env script filename under /var/teraflow.',

    )

    parser.addoption(

        '--tfs-topology-descriptor',

        action='store',

        default=None,

        help='Topology descriptor filename (or absolute path).',

    )





def _require_option(request: pytest.FixtureRequest, option_name: str) -> str:

    value = request.config.getoption(option_name)

    if value is None:

        raise ValueError('Missing required pytest option: --{:s}'.format(option_name.replace('_', '-')))

    return value





@pytest.fixture(scope='session')

def selected_tfs_profile(request: pytest.FixtureRequest) -> str:

    return _require_option(request, 'tfs_profile')





@pytest.fixture(scope='session')

def selected_runtime_script(request: pytest.FixtureRequest, selected_tfs_profile: str) -> str:

    runtime_script = _require_option(request, 'tfs_runtime_script')

    expected_script = PROFILE_FILENAMES[selected_tfs_profile]

    if runtime_script != expected_script:

        msg = 'Runtime script "{:s}" does not match profile "{:s}" (expected "{:s}")'

        raise ValueError(msg.format(runtime_script, selected_tfs_profile, expected_script))

    runtime_file = RUNTIME_ENV_DIR / runtime_script

    if not runtime_file.exists():

        raise FileNotFoundError('Runtime env file not found: {:s}'.format(str(runtime_file)))

    return runtime_script





@pytest.fixture(scope='session')

def selected_topology_descriptor(request: pytest.FixtureRequest) -> str:

    descriptor = _require_option(request, 'tfs_topology_descriptor')

    descriptor_path = Path(descriptor)

    if not descriptor_path.is_absolute():

        descriptor_path = Path(__file__).resolve().parent.parent / 'descriptors' / descriptor

    if not descriptor_path.exists():

        raise FileNotFoundError('Topology descriptor not found: {:s}'.format(str(descriptor_path)))

    return str(descriptor_path)





@pytest.fixture(scope='session')

def selected_tfs_client_bundle(

    tfs_clients,

    selected_tfs_profile: str,

    selected_runtime_script: str,  # pylint: disable=unused-argument

):

    if selected_tfs_profile not in tfs_clients:

        raise KeyError('Profile "{:s}" not loaded in tfs_clients'.format(selected_tfs_profile))

    return tfs_clients[selected_tfs_profile]

src/tests/ofc25/tests/test_functional_bootstrap_e2e.py→src/tests/ofc25/tests/test_functional_bootstrap.py

+50 −44
Original line number Diff line number Diff line 
# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

# Copyright 2022-2026 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.
@@ -12,72 +12,78 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import logging, os, time

import logging

import time



from common.Constants import DEFAULT_CONTEXT_NAME

from common.proto.context_pb2 import (

    ContextId, DeviceOperationalStatusEnum, Empty,

)

from common.tools.descriptor.Loader import (

    DescriptorLoader, check_descriptor_load_results, validate_empty_scenario,

)

from common.tools.grpc.Tools import (

    grpc_message_to_json, grpc_message_to_json_string,

)

from common.proto.context_pb2 import ContextId, DeviceOperationalStatusEnum, Empty

from common.tools.descriptor.Loader import DescriptorLoader, check_descriptor_load_results, validate_empty_scenario

from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string

from common.tools.object_factory.Context import json_context_id

from context.client.ContextClient import ContextClient

from device.client.DeviceClient import DeviceClient



from .Fixtures import (

    context_client, device_client,

)  # pylint: disable=unused-import

from .conftest import selected_tfs_client_bundle, selected_tfs_profile, selected_topology_descriptor  # pylint: disable=unused-import



LOGGER = logging.getLogger(__name__)

LOGGER.setLevel(logging.DEBUG)



DESCRIPTOR_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'descriptors', 'topology_e2e.json')

ADMIN_CONTEXT_ID = ContextId(**json_context_id(DEFAULT_CONTEXT_NAME))





def _check_devices_enabled_or_raise(context_client, profile_name: str, max_retry: int = 10, wait_seconds: float = 1.0) -> None:

    op_status_enabled = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED



    num_devices = -1

    num_devices_enabled = 0

    num_retry = 0

    disabled_devices = list()



    while (num_retry < max_retry) and (num_devices != num_devices_enabled):

        time.sleep(wait_seconds)

        response = context_client.ListDevices(Empty())

        num_devices = len(response.devices)

        num_devices_enabled = 0

        disabled_devices = list()

        for device in response.devices:

            if device.device_operational_status == op_status_enabled:

                num_devices_enabled += 1

            else:

                disabled_devices.append(grpc_message_to_json(device))

        LOGGER.info('[%s] Num Devices enabled: %d/%d', profile_name, num_devices_enabled, num_devices)

        num_retry += 1



    if num_devices_enabled != num_devices:

        msg = '[{:s}] Devices enabled timeout after {:d} retries: {:d}/{:d}; disabled={:s}'

        raise Exception(msg.format(profile_name, max_retry, num_devices_enabled, num_devices, str(disabled_devices)))



    LOGGER.info('[%s] Devices: %s', profile_name, grpc_message_to_json_string(response))





def test_scenario_bootstrap(

    context_client : ContextClient, # pylint: disable=redefined-outer-name

    device_client : DeviceClient,   # pylint: disable=redefined-outer-name

    selected_tfs_client_bundle,

    selected_topology_descriptor: str,

) -> None:

    context_client = selected_tfs_client_bundle.context

    device_client = selected_tfs_client_bundle.device



    validate_empty_scenario(context_client)



    descriptor_loader = DescriptorLoader(

        descriptors_file=DESCRIPTOR_FILE,

        context_client=context_client, device_client=device_client

        descriptors_file=selected_topology_descriptor,

        context_client=context_client,

        device_client=device_client,

    )

    results = descriptor_loader.process()

    check_descriptor_load_results(results, descriptor_loader)

    descriptor_loader.validate()



    # Verify the scenario has no services/slices

    response = context_client.GetContext(ADMIN_CONTEXT_ID)

    assert len(response.service_ids) == 0

    assert len(response.slice_ids) == 0





def test_scenario_devices_enabled(

    context_client : ContextClient,         # pylint: disable=redefined-outer-name

    selected_tfs_client_bundle,

    selected_tfs_profile: str,

) -> None:

    DEVICE_OP_STATUS_ENABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_ENABLED



    num_devices = -1

    num_devices_enabled, num_retry = 0, 0

    while (num_devices != num_devices_enabled) and (num_retry < 10):

        time.sleep(1.0)

        response = context_client.ListDevices(Empty())

        num_devices = len(response.devices)

        num_devices_enabled = 0

        disabled_devices = list()

        for device in response.devices:

            if device.device_operational_status == DEVICE_OP_STATUS_ENABLED:

                num_devices_enabled += 1

            else:

                disabled_devices.append(grpc_message_to_json(device))

        LOGGER.info('Num Devices enabled: {:d}/{:d}'.format(num_devices_enabled, num_devices))

        num_retry += 1

    if num_devices_enabled != num_devices:

        LOGGER.info('Disabled Devices: {:s}'.format(str(disabled_devices)))

    LOGGER.info('Devices: {:s}'.format(grpc_message_to_json_string(response)))

    assert num_devices_enabled == num_devices
 
    context_client = selected_tfs_client_bundle.context

    _check_devices_enabled_or_raise(context_client, selected_tfs_profile)