Commit 6b9c15f9 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

OFC25 test: 

- Enhanced logic to track progress and detect stuck tests
parent a3094313

src/tests/ofc25/tests/Helper.py

+61 −13
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ from typing import List, Set, Tuple 


from common.Constants import DEFAULT_CONTEXT_NAME

from common.proto.context_pb2 import ContextId, Device, Empty, Link, LinkTypeEnum, ServiceStatusEnum, ServiceTypeEnum

from common.tools.grpc.Tools import grpc_message_list_to_json_string, grpc_message_to_json_string

from common.tools.grpc.Tools import grpc_message_list_to_json_string, grpc_message_to_json, grpc_message_to_json_string

from common.tools.object_factory.Context import json_context_id



LOGGER = logging.getLogger(__name__)
@@ -181,8 +181,10 @@ def list_active_optical_services(context_client) -> List: 


    active_optical_services = []

    for service in response.services:

        assert service.service_type == ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY

        assert service.service_status.service_status == ServiceStatusEnum.SERVICESTATUS_ACTIVE

        if service.service_type != ServiceTypeEnum.SERVICETYPE_OPTICAL_CONNECTIVITY:

            continue

        if service.service_status.service_status != ServiceStatusEnum.SERVICESTATUS_ACTIVE:

            continue

        active_optical_services.append(service)

    return active_optical_services
@@ -198,6 +200,22 @@ def count_service_connections(context_client, service) -> int: 
    return len(response.connections)





def describe_services(context_client, profile_name: str) -> str:

    response = context_client.ListServices(ADMIN_CONTEXT_ID)

    services = []

    for service in response.services:

        service_json = grpc_message_to_json(service)

        status_value = service.service_status.service_status

        service_json['service_status_name'] = ServiceStatusEnum.Name(status_value)

        try:

            service_json['num_connections'] = count_service_connections(context_client, service)

        except Exception as exc:  # pylint: disable=broad-except

            service_json['num_connections_error'] = str(exc)

        services.append(service_json)

    LOGGER.info('[%s] Service snapshot: %s', profile_name, str(services))

    return str(services)





def get_virtual_link_ids(context_client) -> Set[str]:

    response = context_client.ListLinks(Empty())

    virtual_link_ids = {
@@ -209,6 +227,24 @@ def get_virtual_link_ids(context_client) -> Set[str]: 
    return virtual_link_ids





def describe_links(context_client, profile_name: str) -> str:

    response = context_client.ListLinks(Empty())

    links = []

    for link in response.links:

        link_json = grpc_message_to_json(link)

        link_json['link_type_name'] = LinkTypeEnum.Name(link.link_type)

        links.append(link_json)

    LOGGER.info('[%s] Link snapshot: %s', profile_name, str(links))

    return str(links)





def log_global_state(ip_context_client, e2e_context_client, opt_context_client) -> None:

    describe_links(ip_context_client, 'ip')

    describe_services(ip_context_client, 'ip')

    describe_services(e2e_context_client, 'e2e')

    describe_services(opt_context_client, 'opt')





def assert_global_state(

    ip_context_client,

    e2e_context_client,
@@ -225,7 +261,8 @@ def assert_global_state( 


    e2e_services = list_active_optical_services(e2e_context_client)

    if expected_e2e_services == 0:

        assert len(e2e_services) == 0

        response = e2e_context_client.ListServices(ADMIN_CONTEXT_ID)

        assert len(response.services) == 0

    else:

        assert len(e2e_services) == expected_e2e_services

        for service in e2e_services:
@@ -233,7 +270,8 @@ def assert_global_state( 


    opt_services = list_active_optical_services(opt_context_client)

    if expected_opt_connections == 0:

        assert len(opt_services) == 0

        response = opt_context_client.ListServices(ADMIN_CONTEXT_ID)

        assert len(response.services) == 0

    else:

        assert len(opt_services) == 1

        assert count_service_connections(opt_context_client, opt_services[0]) == expected_opt_connections
@@ -246,12 +284,16 @@ def wait_for_state_or_raise( 
    expected_virtual_link_ids: Set[str],

    expected_e2e_services: int,

    expected_opt_connections: int,

    max_retry: int = 12,

    wait_seconds: float = 1.0,

    max_retry: int = 30,

    wait_seconds: float = 2.0,

) -> None:

    last_error: Exception = Exception('state not reached')

    for _ in range(max_retry):

    for attempt in range(1, max_retry + 1):

        try:

            LOGGER.info(

                'Checking expected state attempt %d/%d: virtual_links=%s e2e_services=%d opt_connections=%d',

                attempt, max_retry, str(sorted(expected_virtual_link_ids)), expected_e2e_services, expected_opt_connections

            )

            assert_global_state(

                ip_context_client=ip_context_client,

                e2e_context_client=e2e_context_client,
@@ -263,11 +305,17 @@ def wait_for_state_or_raise( 
            return

        except Exception as error:  # pylint: disable=broad-except

            last_error = error

            LOGGER.warning(

                'Expected state not reached on attempt %d/%d: %s',

                attempt, max_retry, str(error)

            )

            log_global_state(ip_context_client, e2e_context_client, opt_context_client)

            time.sleep(wait_seconds)



    MSG = 'Timed out waiting expected state: virtual_links={:s} e2e_services={:d} opt_connections={:d}; error={:s}'

    raise Exception(MSG.format(

    LOGGER.error(

        'Timed out waiting expected state: virtual_links=%s e2e_services=%d opt_connections=%d',

        str(sorted(expected_virtual_link_ids)),

        expected_e2e_services, expected_opt_connections,

        str(last_error)

    ))
 
        expected_e2e_services,

        expected_opt_connections,

    )

    raise last_error

src/tests/ofc25/tests/test_functional_create_vlinks.py

+9 −0
Original line number Diff line number Diff line
@@ -49,6 +49,10 @@ def test_create_virtual_link( 
    expected_virtual_link_ids: Set[str] = set()

    for index, (descriptor_name, virtual_link_id) in enumerate(VIRTUAL_LINK_DESCRIPTORS, start=1):

        descriptor_file = os.path.join(DESCRIPTORS_DIR, descriptor_name)

        LOGGER.info(

            'Creating virtual link step %d/%d from descriptor %s',

            index, len(VIRTUAL_LINK_DESCRIPTORS), descriptor_file

        )

        descriptor_loader = DescriptorLoader(

            descriptors_file=descriptor_file,

            context_client=ip_context_client,
@@ -57,8 +61,13 @@ def test_create_virtual_link( 
        )

        results = descriptor_loader.process()

        check_descriptor_load_results(results, descriptor_loader)

        LOGGER.info('Virtual link request submitted successfully for %s', virtual_link_id)



        expected_virtual_link_ids.add(virtual_link_id)

        LOGGER.info(

            'Waiting for propagated state after creating %s: expected_virtual_links=%s expected_e2e_services=%d',

            virtual_link_id, str(sorted(expected_virtual_link_ids)), index

        )

        wait_for_state_or_raise(

            ip_context_client=ip_context_client,

            e2e_context_client=e2e_context_client,

src/tests/ofc25/tests/test_functional_delete_vlinks.py

+13 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



import logging

import os



from common.tools.descriptor.Loader import DescriptorLoader
@@ -24,6 +25,9 @@ from .Helper import ( 
    wait_for_state_or_raise,

)



LOGGER = logging.getLogger(__name__)

LOGGER.setLevel(logging.DEBUG)





def test_delete_virtual_links(

    tfs_clients,
@@ -50,6 +54,10 @@ def test_delete_virtual_links( 
        [2, 1, 0], reversed(VIRTUAL_LINK_DESCRIPTORS)

    ):

        descriptor_file = os.path.join(DESCRIPTORS_DIR, descriptor_name)

        LOGGER.info(

            'Deleting virtual link from descriptor %s; expecting %d remaining E2E services afterwards',

            descriptor_file, remaining

        )

        descriptor_loader = DescriptorLoader(

            descriptors_file=descriptor_file,

            context_client=ip_context_client,
@@ -57,8 +65,13 @@ def test_delete_virtual_links( 
            vntm_client=ip_vnt_manager_client,

        )

        descriptor_loader.unload()

        LOGGER.info('Virtual link removal submitted successfully for %s', virtual_link_id)



        expected_virtual_link_ids.remove(virtual_link_id)

        LOGGER.info(

            'Waiting for propagated state after deleting %s: expected_virtual_links=%s expected_e2e_services=%d',

            virtual_link_id, str(sorted(expected_virtual_link_ids)), remaining

        )

        wait_for_state_or_raise(

            ip_context_client=ip_context_client,

            e2e_context_client=e2e_context_client,