diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1aa5e597dde27c8a4af294b193afb189a8bf03a9..0da5a74316fb348af940eeb142f085f35d53677d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,7 @@ include: - local: '/src/qos_profile/.gitlab-ci.yml' - local: '/src/vnt_manager/.gitlab-ci.yml' - local: '/src/e2e_orchestrator/.gitlab-ci.yml' + - local: '/src/ztp_server/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' diff --git a/src/ztp_server/.gitlab-ci.yml b/src/ztp_server/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..11058048f20da390155a3647f4894076415bdb7a --- /dev/null +++ b/src/ztp_server/.gitlab-ci.yml @@ -0,0 +1,100 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build ztp_server: + variables: + IMAGE_NAME: 'ztp_server' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test ztp_server: + variables: + IMAGE_NAME: 'ztp_server' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build ztp_server + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - docker ps -aq | xargs -r docker rm -f + - > + if docker network list | grep teraflowbridge; then + echo "teraflowbridge is already created"; + else + docker network create -d bridge teraflowbridge; + fi + - > + if docker container ls | grep $IMAGE_NAME; then + docker rm -f $IMAGE_NAME; + else + echo "$IMAGE_NAME image is not in the system"; + fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + - > + docker run --name $IMAGE_NAME -d -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + --env LOG_LEVEL=DEBUG + --env FLASK_ENV=development + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - while ! docker logs $IMAGE_NAME 2>&1 | grep -q 'Configured Resources:'; do sleep 1; done + - sleep 5 # Give extra time to container to get ready + - docker ps -a + - docker logs $IMAGE_NAME + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_core.py --junitxml=/opt/results/${IMAGE_NAME}_report_core.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage run --append -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report_unitary.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker logs $IMAGE_NAME + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report_*.xml diff --git a/src/ztp_server/requirements.in b/src/ztp_server/requirements.in index 6ac992ba8f54e81cbe9ef2ec145e0c252baffff6..829ad27b3b13faa05a6e7219bce94bea559f3f41 100644 --- a/src/ztp_server/requirements.in +++ b/src/ztp_server/requirements.in @@ -12,17 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -deepdiff==6.7.* -deepmerge==1.1.* Flask==2.1.3 Flask-HTTPAuth==4.5.0 Flask-RESTful==0.3.9 jsonschema==4.4.0 -libyang==2.8.4 -netaddr==0.9.0 -pyang==2.6.0 -git+https://github.com/robshakir/pyangbind.git pydantic==2.6.3 requests==2.27.1 werkzeug==2.3.7 -websockets==12.0 diff --git a/src/ztp_server/service/ZtpServerServiceServicerImpl.py b/src/ztp_server/service/ZtpServerServiceServicerImpl.py index ef06cf9e4a8c17a06c7ec821338db93e08afed2c..6ee240f0fbabd4d9925d98cc353749c3ecd9c588 100755 --- a/src/ztp_server/service/ZtpServerServiceServicerImpl.py +++ b/src/ztp_server/service/ZtpServerServiceServicerImpl.py @@ -28,9 +28,9 @@ class ZtpServerServiceServicerImpl(ZtpServerServiceServicer): LOGGER.info('Servicer Created') @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def GetZtpProvisioning(self, request : ProvisioningScriptName, context : grpc.ServicerContext) -> ProvisioningScript: + def GetProvisioningScript(self, request : ProvisioningScriptName, context : grpc.ServicerContext) -> ProvisioningScript: try: - provisioningPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'data', request.scriptname) + provisioningPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', request.scriptname) with open(provisioningPath, 'r') as provisioning_file: provisioning_content = provisioning_file.read() return ProvisioningScript(script=provisioning_content) diff --git a/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/Resources.py b/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/Resources.py index 20321ae3a1924bbe3eb7bb243fdf43aff55bcab9..522bc0f1b147a1cf0db9c65350a6501128f80ae7 100755 --- a/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/Resources.py +++ b/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/Resources.py @@ -33,5 +33,4 @@ class config(_Resource): @HTTP_AUTH.login_required def get(self, config_db : str): content = returnConfigFile(config_db) - return content diff --git a/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/__init__.py b/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/__init__.py index 42ac3e5390ec32abee96cd562e79cd13fd462ca1..b1c9a961984700fd26084005f3d52d42161b3600 100755 --- a/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/__init__.py +++ b/src/ztp_server/service/rest_server/ztpServer_plugins/ztp_provisioning_api/__init__.py @@ -14,7 +14,7 @@ from ztp_server.service.rest_server.RestServer import RestServer from .Resources import ( - Config + config ) URL_PREFIX = '/provisioning' @@ -22,7 +22,7 @@ URL_PREFIX = '/provisioning' # Use 'path' type since some identifiers might contain char '/' and Flask is unable to recognize them in 'string' type. RESOURCES = [ # (endpoint_name, resource_class, resource_url) - ('api.config', Config, '/config/'), + ('api.config', config, '/config/'), ] def register_ztp_provisioning(rest_server : RestServer): diff --git a/src/ztp_server/tests/Constants.py b/src/ztp_server/tests/Constants.py new file mode 100644 index 0000000000000000000000000000000000000000..eb5f632d630327bc36654d2aa61d38e4d49b399d --- /dev/null +++ b/src/ztp_server/tests/Constants.py @@ -0,0 +1,28 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_baseurl_http, get_service_port_http + + +USERNAME = 'admin' +PASSWORD = 'admin' +LOCAL_HOST = '127.0.0.1' +MOCKSERVICE_PORT = 10000 +ZTP_SERVICE_PORT = get_service_port_http(ServiceNameEnum.ZTP_SERVER) + MOCKSERVICE_PORT # avoid privileged ports +ZTP_SERVICE_PREFIX_URL = get_service_baseurl_http(ServiceNameEnum.ZTP_SERVER) or '' +ZTP_SERVICE_BASE_URL = 'http://{:s}:{:s}@{:s}:{:d}{:s}'.format( + USERNAME, PASSWORD, LOCAL_HOST, ZTP_SERVICE_PORT, ZTP_SERVICE_PREFIX_URL +) diff --git a/src/ztp_server/tests/PrepareTestScenario.py b/src/ztp_server/tests/PrepareTestScenario.py new file mode 100644 index 0000000000000000000000000000000000000000..cda9702f78258d5e81cca9d6ddfd30ac19c56bda --- /dev/null +++ b/src/ztp_server/tests/PrepareTestScenario.py @@ -0,0 +1,132 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum, logging, os, pytest, requests, time # subprocess, threading +from typing import Any, Dict, List, Optional, Set, Union + +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, + ENVVAR_SUFIX_SERVICE_PORT_HTTP, get_env_var_name +) + +from common.tools.service.GenericRestServer import GenericRestServer +from ztp_server.service.rest_server.ztpServer_plugins.ztp_provisioning_api import register_ztp_provisioning +from ztp_server.client.ZtpClient import ZtpClient + +from .Constants import ( + LOCAL_HOST, MOCKSERVICE_PORT, ZTP_SERVICE_BASE_URL, ZTP_SERVICE_PORT, + USERNAME, PASSWORD +) + +os.environ[get_env_var_name(ServiceNameEnum.ZTP_SERVER, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.ZTP_SERVER, ENVVAR_SUFIX_SERVICE_PORT_HTTP)] = str(ZTP_SERVICE_PORT) + +@pytest.fixture(scope='session') +def ztp_server_application(): + _rest_server = GenericRestServer(ZTP_SERVICE_PORT, None, bind_address='127.0.0.1') + register_ztp_provisioning(_rest_server) + _rest_server.start() + time.sleep(1) # bring time for the server to start + yield _rest_server + _rest_server.shutdown() + _rest_server.join() + +@pytest.fixture(scope='session') +def ztp_client(): # pylint: disable=redefined-outer-name + _client = ZtpClient() + yield _client + _client.close() + +class RestRequestMethod(enum.Enum): + GET = 'get' + POST = 'post' + PUT = 'put' + PATCH = 'patch' + DELETE = 'delete' + +EXPECTED_STATUS_CODES : Set[int] = { + requests.codes['OK' ], + requests.codes['CREATED' ], + requests.codes['ACCEPTED' ], + requests.codes['NO_CONTENT'], +} + +def do_rest_request( + method : RestRequestMethod, url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + request_url = ZTP_SERVICE_BASE_URL + url + if logger is not None: + msg = 'Request: {:s} {:s}'.format(str(method.value).upper(), str(request_url)) + if body is not None: msg += ' body={:s}'.format(str(body)) + logger.warning(msg) + reply = requests.request(method.value, request_url, timeout=timeout, json=body, allow_redirects=allow_redirects) + if logger is not None: + logger.warning('Reply: {:s}'.format(str(reply.text))) + assert reply.status_code in expected_status_codes, 'Reply failed with status code {:d}'.format(reply.status_code) + + if reply.content and len(reply.content) > 0: return reply.json() + return None + +def do_rest_get_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.GET, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_post_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.POST, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_put_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.PUT, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_patch_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.PATCH, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) + +def do_rest_delete_request( + url : str, body : Optional[Any] = None, timeout : int = 10, + allow_redirects : bool = True, expected_status_codes : Set[int] = EXPECTED_STATUS_CODES, + logger : Optional[logging.Logger] = None +) -> Optional[Union[Dict, List]]: + return do_rest_request( + RestRequestMethod.DELETE, url, body=body, timeout=timeout, allow_redirects=allow_redirects, + expected_status_codes=expected_status_codes, logger=logger + ) diff --git a/src/ztp_server/tests/__init__.py b/src/ztp_server/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ccc21c7db78aac26daa1f8c5ff8e1ffd3f35460 --- /dev/null +++ b/src/ztp_server/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/ztp_server/tests/test_core.py b/src/ztp_server/tests/test_core.py new file mode 100644 index 0000000000000000000000000000000000000000..016b3ad9342f687aa1e7fc2762cec711ec11c97b --- /dev/null +++ b/src/ztp_server/tests/test_core.py @@ -0,0 +1,41 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import json + +from .PrepareTestScenario import ( # pylint: disable=unused-import + # be careful, order of symbols is important here! + ztp_server_application, do_rest_get_request +) + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +BASE_URL = '/provisioning' + +def test_get_config_file(ztp_server_application, # pylint: disable=redefined-outer-name, unused-argument +) -> None: + URL = BASE_URL + '/config/ztp.json' + data = { + "ztp": { + "01-provisioning-script": { + "plugin": { + "url": "http://localhost:9001/provisioning/provisioning_script_sonic.sh" + }, + "reboot-on-success": True + }} + } + retrieved_data = do_rest_get_request(URL, body=data, logger=LOGGER, expected_status_codes={200}) + LOGGER.debug('retrieved_data={:s}'.format(json.dumps(retrieved_data, sort_keys=True))) diff --git a/src/ztp_server/tests/test_unitary.py b/src/ztp_server/tests/test_unitary.py new file mode 100644 index 0000000000000000000000000000000000000000..bfe3b0904cad34ed2fdbf0255fda1b8619d50789 --- /dev/null +++ b/src/ztp_server/tests/test_unitary.py @@ -0,0 +1,38 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, pytest, logging, json +from ztp_server.client.ZtpClient import ZtpClient +from common.proto.ztp_server_pb2 import ProvisioningScriptName, ProvisioningScript, ZtpFileName, ZtpFile + +from common.tools.grpc.Tools import grpc_message_to_json_string + +from .PrepareTestScenario import ( # pylint: disable=unused-import + # be careful, order of symbols is important here! + ztp_client +) + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +def test_GetProvisioningScript( + ztp_client : ZtpClient, +): # pylint: disable=redefined-outer-name + + ztp_provisioning_file_request = ProvisioningScriptName() + ztp_provisioning_file_request.scriptname = "provisioning_script_sonic.sh" # pylint: disable=no-member + ztp_reply = ztp_client.GetProvisioningScript(ztp_provisioning_file_request) + + LOGGER.debug('retrieved_data={:s}'.format(grpc_message_to_json_string(ztp_reply))) + assert ztp_reply.script.startswith("#!/bin/bash")