diff --git a/src/context/service/database/Link.py b/src/context/service/database/Link.py index deef3769cb64e34ae6c300f562c31983a8807286..673947f0306d28e1c7d98b6993dc7f8164df2bab 100644 --- a/src/context/service/database/Link.py +++ b/src/context/service/database/Link.py @@ -28,6 +28,7 @@ from .models.TopologyModel import TopologyLinkModel, TopologyModel from .uuids.EndPoint import endpoint_get_uuid from .uuids.Link import link_get_uuid from .Events import notify_event_context, notify_event_link, notify_event_topology +from .models.enums.LinkType import grpc_to_enum__link_type_enum LOGGER = logging.getLogger(__name__) @@ -67,8 +68,8 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) raw_link_name = request.name link_name = raw_link_uuid if len(raw_link_name) == 0 else raw_link_name link_uuid = link_get_uuid(request.link_id, link_name=link_name, allow_random=True) - - now = datetime.datetime.utcnow() + link_type = grpc_to_enum__link_type_enum(request.link_type) + now = datetime.datetime.now(datetime.timezone.utc) topology_uuids : Set[str] = set() related_topologies : List[Dict] = list() @@ -117,6 +118,7 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) link_data = [{ 'link_uuid' : link_uuid, 'link_name' : link_name, + 'link_type' : link_type, 'total_capacity_gbps' : total_capacity_gbps, 'used_capacity_gbps' : used_capacity_gbps, 'created_at' : now, @@ -129,6 +131,7 @@ def link_set(db_engine : Engine, messagebroker : MessageBroker, request : Link) index_elements=[LinkModel.link_uuid], set_=dict( link_name = stmt.excluded.link_name, + link_type = stmt.excluded.link_type, total_capacity_gbps = stmt.excluded.total_capacity_gbps, used_capacity_gbps = stmt.excluded.used_capacity_gbps, updated_at = stmt.excluded.updated_at, diff --git a/src/context/service/database/models/LinkModel.py b/src/context/service/database/models/LinkModel.py index 423e39832201cc19d98a106b136fb545f4e24b7d..2de279a6e6386de1fe072e78944438bfc612059d 100644 --- a/src/context/service/database/models/LinkModel.py +++ b/src/context/service/database/models/LinkModel.py @@ -13,17 +13,20 @@ # limitations under the License. import operator -from sqlalchemy import CheckConstraint, Column, DateTime, Float, ForeignKey, Integer, String +from sqlalchemy import CheckConstraint, Column, DateTime, Enum, Float, ForeignKey, Integer, String from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship from typing import Dict from ._Base import _Base +from common.proto.context_pb2 import LinkTypeEnum +from .enums.LinkType import ORM_LinkTypeEnum class LinkModel(_Base): __tablename__ = 'link' link_uuid = Column(UUID(as_uuid=False), primary_key=True) link_name = Column(String, nullable=False) + link_type = Column(Enum(ORM_LinkTypeEnum), nullable=False) total_capacity_gbps = Column(Float, nullable=True) used_capacity_gbps = Column(Float, nullable=True) created_at = Column(DateTime, nullable=False) @@ -44,11 +47,14 @@ class LinkModel(_Base): result = { 'link_id' : self.dump_id(), 'name' : self.link_name, + 'link_type' : self.link_type.value, 'link_endpoint_ids': [ link_endpoint.endpoint.dump_id() for link_endpoint in sorted(self.link_endpoints, key=operator.attrgetter('position')) ], } + if self.link_type is None: + self.link_type = LinkTypeEnum.LINKTYPE_UNKNOWN if self.total_capacity_gbps is not None: attributes : Dict = result.setdefault('attributes', dict()) attributes.setdefault('total_capacity_gbps', self.total_capacity_gbps) diff --git a/src/context/service/database/models/enums/LinkType.py b/src/context/service/database/models/enums/LinkType.py new file mode 100644 index 0000000000000000000000000000000000000000..1ac1a547fe085c722a97bbdabeae663fdcffdc37 --- /dev/null +++ b/src/context/service/database/models/enums/LinkType.py @@ -0,0 +1,32 @@ +# Copyright 2022-2024 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum, functools +from common.proto.context_pb2 import LinkTypeEnum +from ._GrpcToEnum import grpc_to_enum + +# IMPORTANT: Entries of enum class ORM_DeviceDriverEnum should be named as in +# the proto files removing the prefixes. For example, proto item +# DeviceDriverEnum.DEVICEDRIVER_OPENCONFIG should be included as +# OPENCONFIG. If item name does not match, automatic mapping of +# proto enums to database enums will fail. +class ORM_LinkTypeEnum(enum.Enum): + UNKNOWN = LinkTypeEnum.LINKTYPE_UNKNOWN + COPPER = LinkTypeEnum.LINKTYPE_COPPER + VIRTUAL_COPPER = LinkTypeEnum.LINKTYPE_VIRTUAL_COPPER + OPTICAL = LinkTypeEnum.LINKTYPE_OPTICAL + VIRTUAL_OPTICAL = LinkTypeEnum.LINKTYPE_VIRTUAL_OPTICAL + +grpc_to_enum__link_type_enum = functools.partial( + grpc_to_enum, LinkTypeEnum, ORM_LinkTypeEnum) diff --git a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py index ae9bfc7d2d8dcab172ef8da5b9600d2c20763415..c9681f7087ff2b80f775fce88225e32a2965f6ca 100644 --- a/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py +++ b/src/e2e_orchestrator/service/E2EOrchestratorServiceServicerImpl.py @@ -80,7 +80,6 @@ class SubscriptionServer(Thread): LOGGER.debug("Received message from WebSocket: {}".format(message)) except Exception as ex: LOGGER.error('Exception receiving from WebSocket: {}'.format(ex)) - self._events_server() @@ -99,12 +98,13 @@ class SubscriptionServer(Thread): def _event_received(self, connection): + LOGGER.debug('Event received') for message in connection: message_json = json.loads(message) - # LOGGER.info("message_json: {}".format(message_json)) # Link creation if 'link_id' in message_json: + LOGGER.debug('Link creation') link = Link(**message_json) service = Service() @@ -114,12 +114,12 @@ class SubscriptionServer(Thread): service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED service_client.CreateService(service) - links = context_client.ListLinks(Empty()).links a_device_uuid = device_get_uuid(link.link_endpoint_ids[0].device_id) a_endpoint_uuid = endpoint_get_uuid(link.link_endpoint_ids[0])[2] z_device_uuid = device_get_uuid(link.link_endpoint_ids[1].device_id) z_endpoint_uuid = endpoint_get_uuid(link.link_endpoint_ids[1])[2] + links = context_client.ListLinks(Empty()).links for _link in links: for _endpoint_id in _link.link_endpoint_ids: if _endpoint_id.device_id.device_uuid.uuid == a_device_uuid and \ @@ -130,7 +130,9 @@ class SubscriptionServer(Thread): z_ep_id = _endpoint_id if (not 'a_ep_id' in locals()) or (not 'z_ep_id' in locals()): - error_msg = 'Could not get VNT link endpoints' + error_msg = f'Could not get VNT link endpoints\ + \n\ta_endpoint_uuid= {a_endpoint_uuid}\ + \n\tz_endpoint_uuid= {z_device_uuid}' LOGGER.error(error_msg) connection.send(error_msg) return @@ -138,20 +140,22 @@ class SubscriptionServer(Thread): service.service_endpoint_ids.append(copy.deepcopy(a_ep_id)) service.service_endpoint_ids.append(copy.deepcopy(z_ep_id)) - # service_client.UpdateService(service) + service_client.UpdateService(service) + re_svc = context_client.GetService(service.service_id) connection.send(grpc_message_to_json_string(link)) - # Link removal + context_client.SetLink(link) elif 'link_uuid' in message_json: + LOGGER.debug('Link removal') link_id = LinkId(**message_json) service_id = ServiceId() service_id.service_uuid.uuid = link_id.link_uuid.uuid service_id.context_id.context_uuid.uuid = DEFAULT_CONTEXT_NAME - # service_client.DeleteService(service_id) + service_client.DeleteService(service_id) connection.send(grpc_message_to_json_string(link_id)) context_client.RemoveLink(link_id) - # Topology received else: + LOGGER.debug('Topology received') topology_details = TopologyDetails(**message_json) context = Context() @@ -187,12 +191,12 @@ class E2EOrchestratorServiceServicerImpl(E2EOrchestratorServiceServicer): i = 1 while True: try: - LOGGER.info(f'Retrieving external controller #{i}') ADD = str(get_setting(f'EXT_CONTROLLER{i}_ADD')) PORT = str(get_setting(f'EXT_CONTROLLER{i}_PORT')) except Exception as e: break try: + LOGGER.info(f'Retrieving external controller #{i}') url = f'http://{ADD}:{PORT}/tfs-api/context/{DEFAULT_CONTEXT_NAME}/topology_details/{DEFAULT_TOPOLOGY_NAME}' topo = requests.get(url).json() except Exception as e: diff --git a/src/e2e_orchestrator/service/__main__.py b/src/e2e_orchestrator/service/__main__.py index 0854aed2de9c748bab7c4d70f35dc6fd3e2ebfd3..4c0a6d471e2b6d7ae87aee666695ebdca6938491 100644 --- a/src/e2e_orchestrator/service/__main__.py +++ b/src/e2e_orchestrator/service/__main__.py @@ -28,7 +28,10 @@ from common.Settings import (ENVVAR_SUFIX_SERVICE_HOST, from .E2EOrchestratorService import E2EOrchestratorService terminate = threading.Event() -LOGGER = None + +LOG_LEVEL = get_log_level() +logging.basicConfig(level=LOG_LEVEL, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s") +LOGGER = logging.getLogger(__name__) def signal_handler(signal, frame): # pylint: disable=redefined-outer-name @@ -37,12 +40,6 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name def main(): - global LOGGER # pylint: disable=global-statement - - log_level = get_log_level() - logging.basicConfig(level=log_level) - LOGGER = logging.getLogger(__name__) - signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) diff --git a/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py b/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py index 484ccff660bee2687b2f500b6f120e0e9c451aab..0b99fba5042aea6d45f38b43ea9d74165425f113 100644 --- a/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/tfs_api/Resources.py @@ -300,21 +300,31 @@ class Links(_Resource): ] class Link(_Resource): + @staticmethod + def _virtual_link(link): + virtual_types = {LinkTypeEnum.LINKTYPE_VIRTUAL_COPPER, LinkTypeEnum.LINKTYPE_VIRTUAL_OPTICAL} + if link.link_type in virtual_types: + return True + return False + + def get(self, link_uuid : str): return format_grpc_to_json(self.context_client.GetLink(grpc_link_id(link_uuid))) def put(self, link_uuid : str): link_json = request.get_json() link = grpc_link(link_json) - virtual_types = {LinkTypeEnum.LINKTYPE_VIRTUAL_COPPER, LinkTypeEnum.LINKTYPE_VIRTUAL_OPTICAL} if link_uuid != link.link_id.link_uuid.uuid: raise BadRequest('Mismatching link_uuid') - elif link.link_type in virtual_types: + elif self._virtual_link(link): link = grpc_link(link_json) return format_grpc_to_json(self.vntmanager_client.SetVirtualLink(link)) - return format_grpc_to_json(self.context_client.SetLink(grpc_link(link))) + return format_grpc_to_json(self.context_client.SetLink(link)) def delete(self, link_uuid : str): + link = self.context_client.GetLink(grpc_link_id(link_uuid)) + if self._virtual_link(link): + format_grpc_to_json(self.vntmanager_client.RemoveVirtualLink(grpc_link_id(link_uuid))) return format_grpc_to_json(self.context_client.RemoveLink(grpc_link_id(link_uuid))) class ConnectionIds(_Resource): diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 9120d475b8b2bb88bfcf7d7da3cadba2bf9931e4..b6f8a7faf89acf25c84326132ff5e965aade61b2 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -384,10 +384,10 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): if "bandwidth" in constraint.custom.constraint_type: bitrate = int(float(constraint.custom.constraint_value)) break - - bitrate = int(float( - service.service_constraints[0].custom.constraint_value - )) + if service.service_constraints: + bitrate = int(float( + service.service_constraints[0].custom.constraint_value + )) if len(service.service_config.config_rules) > 0: c_rules_dict = json.loads( service.service_config.config_rules[0].custom.resource_value) diff --git a/src/tests/ecoc24/Dockerfile b/src/tests/ecoc24/Dockerfile index 727abbd3ac497e99c569095c245343c75c97bc84..f9a616e76c682dc79c76cab3b4b5295ba229e3c9 100644 --- a/src/tests/ecoc24/Dockerfile +++ b/src/tests/ecoc24/Dockerfile @@ -82,19 +82,22 @@ COPY src/tests/*.py ./tests/ COPY src/tests/ecoc24/__init__.py ./tests/ecoc24/__init__.py COPY src/tests/ecoc24/descriptors/descriptor_ip.json ./tests/ecoc24/descriptors/descriptor_ip.json COPY src/tests/ecoc24/descriptors/descriptor_opt.json ./tests/ecoc24/descriptors/descriptor_opt.json -COPY src/tests/ecoc24/descriptors/link_mapping.json ./tests/ecoc24/descriptors/link_mapping.json +COPY src/tests/ecoc24/descriptors/link_mapping.json ./tests/ecoc24/descriptors/descriptor_e2e.json COPY src/tests/ecoc24/tests/. ./tests/ecoc24/tests/ RUN tee ./run_tests.sh <<EOF #!/bin/bash -source /var/teraflow/tfs_runtime_env_vars.sh +source /var/teraflow/tfs_runtime_env_vars_e2e.sh export PYTHONPATH=/var/teraflow -pytest --verbose --log-level=INFO /var/teraflow/tests/ecoc24/tests/test_functional_bootstrap.py --junitxml=/opt/results/report_bootstrap.xml +pytest --verbose --log-level=INFO /var/teraflow/tests/ecoc24/tests/test_functional_bootstrap_opt.py --junitxml=/opt/results/report_bootstrap.xml +pytest --verbose --log-level=INFO /var/teraflow/tests/ecoc24/tests/test_functional_bootstrap_ip.py --junitxml=/opt/results/report_bootstrap.xml +sleep 5 +pytest --verbose --log-level=INFO /var/teraflow/tests/ecoc24/tests/test_functional_bootstrap_e2e.py --junitxml=/opt/results/report_bootstrap.xml #pytest --verbose --log-level=INFO /var/teraflow/tests/ofc24/tests/test_functional_create_service_unidir.py --junitxml=/opt/results/report_create_service_unidir.xml #pytest --verbose --log-level=INFO /var/teraflow/tests/ofc24/tests/test_functional_delete_service_unidir.py --junitxml=/opt/results/report_delete_service_unidir.xml #pytest --verbose --log-level=INFO /var/teraflow/tests/ofc24/tests/test_functional_create_service_bidir.py --junitxml=/opt/results/report_create_service_bidir.xml #pytest --verbose --log-level=INFO /var/teraflow/tests/ofc24/tests/test_functional_delete_service_bidir.py --junitxml=/opt/results/report_delete_service_bidir.xml -#pytest --verbose --log-level=INFO /var/teraflow/tests/ofc24/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml +pytest --verbose --log-level=INFO /var/teraflow/tests/ofc24/tests/test_functional_cleanup.py --junitxml=/opt/results/report_cleanup.xml EOF RUN chmod ug+x ./run_tests.sh diff --git a/src/tests/ecoc24/deploy_e2e.sh b/src/tests/ecoc24/deploy_e2e.sh index 12369f7575c0f1f111f399f89375adf3d156e25d..cbfcfdc21871b82c862f85dab1cefc136f2eb253 100755 --- a/src/tests/ecoc24/deploy_e2e.sh +++ b/src/tests/ecoc24/deploy_e2e.sh @@ -28,7 +28,7 @@ source src/tests/ecoc24/deploy_specs_e2e.sh # Change the name for the database cp manifests/contextservice.yaml manifests/contextservice.yaml.bak -sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfse2e_context"/}' manifests/contextservice.yaml +sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfs_e2e_context"/}' manifests/contextservice.yaml ./deploy/all.sh mv manifests/contextservice.yaml.bak manifests/contextservice.yaml diff --git a/src/tests/ecoc24/deploy_ip.sh b/src/tests/ecoc24/deploy_ip.sh index 46b99ad2708e77ebd50f58408bbce2b700fbfa6b..694b246980db904162e8c4b6cbd90dc294a9f19b 100755 --- a/src/tests/ecoc24/deploy_ip.sh +++ b/src/tests/ecoc24/deploy_ip.sh @@ -28,7 +28,7 @@ source src/tests/ecoc24/deploy_specs_ip.sh # Change the name for the database cp manifests/contextservice.yaml manifests/contextservice.yaml.bak -sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfsip_context"/}' manifests/contextservice.yaml +sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfs_ip_context"/}' manifests/contextservice.yaml ./deploy/all.sh mv manifests/contextservice.yaml.bak manifests/contextservice.yaml diff --git a/src/tests/ecoc24/deploy_opt.sh b/src/tests/ecoc24/deploy_opt.sh index 0dec83bb3158ad5e7831b63856ed52664e3879a0..c85b6e08bd9a6b3c9ed00decf022e27012783b8e 100755 --- a/src/tests/ecoc24/deploy_opt.sh +++ b/src/tests/ecoc24/deploy_opt.sh @@ -28,7 +28,7 @@ source src/tests/ecoc24/deploy_specs_opt.sh # Change the name for the database cp manifests/contextservice.yaml manifests/contextservice.yaml.bak -sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfsopt_context"/}' manifests/contextservice.yaml +sed -i '/name: CRDB_DATABASE/{n;s/value: .*/value: "tfs_opt_context"/}' manifests/contextservice.yaml ./deploy/all.sh mv manifests/contextservice.yaml.bak manifests/contextservice.yaml diff --git a/src/tests/ecoc24/deploy_specs_opt.sh b/src/tests/ecoc24/deploy_specs_opt.sh index d7a6093a317dd27bf64823106ba739944131732f..0622e258148468158a06e2f4c6b7208c5b75e81a 100755 --- a/src/tests/ecoc24/deploy_specs_opt.sh +++ b/src/tests/ecoc24/deploy_specs_opt.sh @@ -130,14 +130,14 @@ export CRDB_USERNAME="tfs" export CRDB_PASSWORD="tfs123" # Set the database name to be used by Context. -export CRDB_DATABASE="tfs_ip" +export CRDB_DATABASE="tfs_opt" # Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. # See ./deploy/all.sh or ./deploy/crdb.sh for additional details export CRDB_DEPLOY_MODE="single" # Disable flag for dropping database, if it exists. -export CRDB_DROP_DATABASE_IF_EXISTS="NO" +export CRDB_DROP_DATABASE_IF_EXISTS="YES" # Disable flag for re-deploying CockroachDB from scratch. export CRDB_REDEPLOY="" diff --git a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py index e18dc286dc79f9494d4ffafeff41107f9d79712f..79ff1b7523a521b98118d588c1ae3e900f61f01b 100644 --- a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py +++ b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py @@ -71,14 +71,12 @@ class VNTMEventDispatcher(threading.Thread): return message def run(self) -> None: - - time.sleep(5) events_collector = EventsCollector( context_client, log_events_received=True, - activate_context_collector = False, + activate_context_collector = True, activate_topology_collector = True, - activate_device_collector = False, - activate_link_collector = False, + activate_device_collector = True, + activate_link_collector = True, activate_service_collector = False, activate_slice_collector = False, activate_connection_collector = False,) @@ -92,7 +90,7 @@ class VNTMEventDispatcher(threading.Thread): LOGGER.info("Connecting to events server...: {}".format(url)) self.websocket = connect(url) except Exception as ex: - LOGGER.error('Error connecting to {}'.format(url)) + LOGGER.error(f'Error connecting to {url}\n\t{ex}') else: LOGGER.info('Connected to {}'.format(url)) context_id = json_context_id(DEFAULT_CONTEXT_NAME) @@ -108,12 +106,9 @@ class VNTMEventDispatcher(threading.Thread): while not self._terminate.is_set(): event = events_collector.get_event(block=True, timeout=GET_EVENT_TIMEOUT) if event is None: continue - LOGGER.info('Event type: {}'.format(event)) - LOGGER.debug('Received event: {}'.format(event)) + LOGGER.debug('Event type: {}'.format(event)) topology_details = context_client.GetTopologyDetails(TopologyId(**topology_id)) - to_send = grpc_message_to_json_string(topology_details) - self.send_msg(to_send) LOGGER.info('Exiting') @@ -135,6 +130,8 @@ class VNTManagerServiceServicerImpl(VNTManagerServiceServicer): self.event_dispatcher = VNTMEventDispatcher(request.host, int(request.port)) self.host = request.host self.port = request.port + LOGGER.info('sleeping 5...') + time.sleep(5) self.event_dispatcher.start() return reply @@ -158,12 +155,13 @@ class VNTManagerServiceServicerImpl(VNTManagerServiceServicer): link = Link(**message_json) context_client.SetLink(link) except Exception as e: - LOGGER.error('Exception setting virtual link={}\n\t{}'.format(request.link_id.link_uuid.uuid, e)) + LOGGER.error(f'Exception setting virtual link={request.link_id.link_uuid.uuid}\n\t{e}') return request.link_id @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def RemoveVirtualLink(self, request : LinkId, context : grpc.ServicerContext) -> Empty: try: + LOGGER.debug('Removing virtual link') self.event_dispatcher.send_msg(grpc_message_to_json_string(request)) # deconfigure('CSGW1', 'xe5', 'CSGW2', 'xe5', 'ecoc2024-1') response = self.event_dispatcher.recv_msg()