Merge branch... (7a70472f) · Commits · TFS / controller

src/dbscanserving/.gitlab-ci.yml

+40 −31

Original line number	Diff line number	Diff line
		@@ -30,6 +30,8 @@ build dbscanserving:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" \|\| $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
		- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
		- changes:
		- src/common/*/.py
		- proto/*.proto
		- src/$IMAGE_NAME/*/.{py,in,yml}
		- src/$IMAGE_NAME/Dockerfile
		- src/$IMAGE_NAME/tests/*.py
		@@ -48,25 +50,33 @@ unit_test dbscanserving:
		- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
		- if docker network list \| grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi
		- if docker container ls \| grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi
		- docker container prune -f
		script:
		- docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
		- docker run --name $IMAGE_NAME -d -p 10008:10008 -v "$PWD/src/$IMAGE_NAME/tests:/home/${IMAGE_NAME}/results" --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
		- sleep 5
		- docker ps -a
		- docker logs $IMAGE_NAME
		- >
		docker run --name $IMAGE_NAME -d -p 10008:10008
		--volume "$PWD/src/$IMAGE_NAME/tests:/opt/results"
		--network=teraflowbridge
		$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
		- docker ps -a
		- sleep 5
		- docker logs $IMAGE_NAME
		- docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/home/${IMAGE_NAME}/results/${IMAGE_NAME}_report.xml"
		- >
		docker exec -i $IMAGE_NAME bash -c
		"coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_unitary.py"
		- docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
		coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
		after_script:
		- docker rm -f $IMAGE_NAME
		- docker network rm teraflowbridge
		- docker volume prune --force
		- docker image prune --force
		rules:
		- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" \|\| $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
		- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
		- changes:
		- src/common/*/.py
		- proto/*.proto
		- src/$IMAGE_NAME/*/.{py,in,yml}
		- src/$IMAGE_NAME/Dockerfile
		- src/$IMAGE_NAME/tests/*.py
		@@ -78,7 +88,6 @@ unit_test dbscanserving:
		reports:
		junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml


		# Deployment of the dbscanserving service in Kubernetes Cluster
		#deploy dbscanserving:
		# variables:

src/dbscanserving/Dockerfile

+15 −29

Original line number	Diff line number	Diff line
		@@ -27,19 +27,6 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
		wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
		chmod +x /bin/grpc_health_probe

		# Creating a user for security reasons
		RUN groupadd -r teraflow && useradd -u 1001 --no-log-init -r -m -g teraflow teraflow
		USER teraflow

		# set working directory
		RUN mkdir -p /home/teraflow/controller/common
		WORKDIR /home/teraflow/controller

		# Get Python packages per module
		ENV VIRTUAL_ENV=/home/teraflow/venv
		RUN python3 -m venv ${VIRTUAL_ENV}
		ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

		# Get generic Python packages
		RUN python3 -m pip install --upgrade pip
		RUN python3 -m pip install --upgrade setuptools wheel
		@@ -47,36 +34,35 @@ RUN python3 -m pip install --upgrade pip-tools

		# Get common Python packages
		# Note: this step enables sharing the previous Docker build steps among all the Python components
		COPY --chown=teraflow:teraflow common_requirements.in common_requirements.in
		WORKDIR /var/teraflow
		COPY common_requirements.in common_requirements.in
		RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in
		RUN python3 -m pip install -r common_requirements.txt

		# Add common files into working directory
		WORKDIR /home/teraflow/controller/common
		COPY --chown=teraflow:teraflow src/common/. ./
		WORKDIR /var/teraflow/common
		COPY src/common/. ./
		RUN rm -rf proto

		# Create proto sub-folder, copy .proto files, and generate Python code
		RUN mkdir -p /home/teraflow/controller/common/proto
		WORKDIR /home/teraflow/controller/common/proto
		RUN mkdir -p /var/teraflow/common/proto
		WORKDIR /var/teraflow/common/proto
		RUN touch __init__.py
		COPY --chown=teraflow:teraflow proto/*.proto ./
		COPY proto/*.proto ./
		RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto
		RUN rm *.proto
		RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;

		# Create module sub-folders
		RUN mkdir -p /home/teraflow/controller/dbscanserving
		WORKDIR /home/teraflow/controller

		# Get Python packages per module
		COPY --chown=teraflow:teraflow ./src/dbscanserving/requirements.in dbscanserving/requirements.in
		# consider common and specific requirements to avoid inconsistencies with dependencies
		RUN pip-compile --quiet --output-file=dbscanserving/requirements.txt dbscanserving/requirements.in common_requirements.in
		RUN python3 -m pip install -r dbscanserving/requirements.txt
		# Create component sub-folders, get specific Python packages
		RUN mkdir -p /var/teraflow/dbscanserving
		WORKDIR /var/teraflow/dbscanserving
		COPY src/dbscanserving/requirements.in requirements.in
		RUN pip-compile --quiet --output-file=requirements.txt requirements.in
		RUN python3 -m pip install -r requirements.txt

		# Add component files into working directory
		COPY --chown=teraflow:teraflow ./src/dbscanserving/. dbscanserving
		WORKDIR /var/teraflow
		COPY src/dbscanserving/. dbscanserving/

		# Start the service
		ENTRYPOINT ["python", "-m", "dbscanserving.service"]

src/service/service/task_scheduler/TaskExecutor.py

+1 −2

Original line number	Diff line number	Diff line
		@@ -14,10 +14,9 @@

		import json, logging
		from enum import Enum
		from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
		from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
		from common.DeviceTypes import DeviceTypeEnum
		from common.method_wrappers.ServiceExceptions import NotFoundException
		from typing import List
		from common.proto.qkd_app_pb2 import QKDAppStatusEnum
		from common.proto.context_pb2 import (
		Connection, ConnectionId, Device, DeviceDriverEnum, DeviceId, Service, ServiceId,

src/service/tests/test_l3nm_gnmi_static_rule_gen/MockServiceHandler.py

+18 −3

Original line number	Diff line number	Diff line
		@@ -14,13 +14,15 @@

		import json, logging
		from typing import Any, Dict, List, Optional, Tuple, Union
		from common.proto.context_pb2 import ConfigRule, DeviceId, Service
		from common.proto.context_pb2 import ConfigRule, ConnectionId, DeviceId, Service
		from common.tools.object_factory.Connection import json_connection_id
		from common.tools.object_factory.Device import json_device_id
		from common.type_checkers.Checkers import chk_type
		from service.service.service_handler_api._ServiceHandler import _ServiceHandler
		from service.service.service_handler_api.SettingsHandler import SettingsHandler
		from service.service.service_handler_api.Tools import get_device_endpoint_uuids, get_endpoint_matching
		from .MockTaskExecutor import MockTaskExecutor
		from service.service.tools.EndpointIdFormatters import endpointids_to_raw
		from service.service.service_handlers.l3nm_gnmi_openconfig.ConfigRuleComposer import ConfigRuleComposer
		from service.service.service_handlers.l3nm_gnmi_openconfig.StaticRouteGenerator import StaticRouteGenerator

		@@ -48,17 +50,20 @@ class MockServiceHandler(_ServiceHandler):

		device_obj = self.__task_executor.get_device(DeviceId(**json_device_id(device_uuid)))
		device_settings = self.__settings_handler.get_device_settings(device_obj)
		self.__config_rule_composer.set_device_alias(device_obj.name, device_uuid)
		_device = self.__config_rule_composer.get_device(device_obj.name)
		_device.configure(device_obj, device_settings)

		endpoint_obj = get_endpoint_matching(device_obj, endpoint_uuid)
		endpoint_settings = self.__settings_handler.get_endpoint_settings(device_obj, endpoint_obj)
		_device.set_endpoint_alias(endpoint_obj.name, endpoint_uuid)
		_endpoint = _device.get_endpoint(endpoint_obj.name)
		_endpoint.configure(endpoint_obj, endpoint_settings)

		self.__endpoint_map[(device_uuid, endpoint_uuid)] = (device_obj.name, endpoint_obj.name)

		self.__static_route_generator.compose(endpoints)
		LOGGER.debug('config_rule_composer = {:s}'.format(json.dumps(self.__config_rule_composer.dump())))

		def _do_configurations(
		self, config_rules_per_device : Dict[str, List[Dict]], endpoints : List[Tuple[str, str, Optional[str]]],
		@@ -94,7 +99,12 @@ class MockServiceHandler(_ServiceHandler):
		) -> List[Union[bool, Exception]]:
		chk_type('endpoints', endpoints, list)
		if len(endpoints) == 0: return []
		self._compose_config_rules(endpoints)
		#service_uuid = self.__service.service_id.service_uuid.uuid
		connection = self.__task_executor.get_connection(ConnectionId(**json_connection_id(connection_uuid)))
		connection_endpoint_ids = endpointids_to_raw(connection.path_hops_endpoint_ids)
		self._compose_config_rules(connection_endpoint_ids)
		#network_instance_name = service_uuid.split('-')[0]
		#config_rules_per_device = self.__config_rule_composer.get_config_rules(network_instance_name, delete=False)
		config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=False)
		LOGGER.debug('config_rules_per_device={:s}'.format(str(config_rules_per_device)))
		results = self._do_configurations(config_rules_per_device, endpoints)
		@@ -106,7 +116,12 @@ class MockServiceHandler(_ServiceHandler):
		) -> List[Union[bool, Exception]]:
		chk_type('endpoints', endpoints, list)
		if len(endpoints) == 0: return []
		self._compose_config_rules(endpoints)
		#service_uuid = self.__service.service_id.service_uuid.uuid
		connection = self.__task_executor.get_connection(ConnectionId(**json_connection_id(connection_uuid)))
		connection_endpoint_ids = endpointids_to_raw(connection.path_hops_endpoint_ids)
		self._compose_config_rules(connection_endpoint_ids)
		#network_instance_name = service_uuid.split('-')[0]
		#config_rules_per_device = self.__config_rule_composer.get_config_rules(network_instance_name, delete=True)
		config_rules_per_device = self.__config_rule_composer.get_config_rules(delete=True)
		LOGGER.debug('config_rules_per_device={:s}'.format(str(config_rules_per_device)))
		results = self._do_configurations(config_rules_per_device, endpoints, delete=True)

src/service/tests/test_l3nm_gnmi_static_rule_gen/MockTaskExecutor.py

+31 −3

Original line number	Diff line number	Diff line
		@@ -14,10 +14,12 @@

		import logging
		from enum import Enum
		from typing import Dict, Optional, Union
		from typing import Any, Dict, Optional, Union
		from common.method_wrappers.ServiceExceptions import NotFoundException
		from common.proto.context_pb2 import Connection, Device, DeviceId, Service
		from service.service.tools.ObjectKeys import get_device_key
		from common.proto.context_pb2 import (
		Connection, ConnectionId, Device, DeviceId, Service
		)
		from service.service.tools.ObjectKeys import get_connection_key, get_device_key

		LOGGER = logging.getLogger(__name__)

		@@ -46,6 +48,32 @@ class MockTaskExecutor:
		object_key = '{:s}:{:s}'.format(object_type.value, object_key)
		self._grpc_objects_cache.pop(object_key, None)

		def _store_editable_grpc_object(
		self, object_type : CacheableObjectType, object_key : str, grpc_class, grpc_ro_object
		) -> Any:
		grpc_rw_object = grpc_class()
		grpc_rw_object.CopyFrom(grpc_ro_object)
		self._store_grpc_object(object_type, object_key, grpc_rw_object)
		return grpc_rw_object

		# ----- Connection-related methods ---------------------------------------------------------------------------------

		def get_connection(self, connection_id : ConnectionId) -> Connection:
		connection_key = get_connection_key(connection_id)
		connection = self._load_grpc_object(CacheableObjectType.CONNECTION, connection_key)
		if connection is None: raise NotFoundException('Connection', connection_key)
		return connection

		def set_connection(self, connection : Connection) -> None:
		connection_key = get_connection_key(connection.connection_id)
		self._store_grpc_object(CacheableObjectType.CONNECTION, connection_key, connection)

		def delete_connection(self, connection_id : ConnectionId) -> None:
		connection_key = get_connection_key(connection_id)
		self._delete_grpc_object(CacheableObjectType.CONNECTION, connection_key)

		# ----- Device-related methods -------------------------------------------------------------------------------------

		def get_device(self, device_id : DeviceId) -> Device:
		device_key = get_device_key(device_id)
		device = self._load_grpc_object(CacheableObjectType.DEVICE, device_key)