From 5fd7570698c26fcb4dda88eaa1e7dbcd1f6dea3c Mon Sep 17 00:00:00 2001 From: "agbarneo@optaresolutions.com" Date: Thu, 26 Sep 2024 11:37:48 +0000 Subject: [PATCH 1/2] Added left files for qkd_app --- src/qkd_app/.gitlab-ci.yml | 80 +++++++++++++++++++++++++++ src/qkd_app/Config.py | 14 +++++ src/qkd_app/Dockerfile | 58 +++++++++++++++++++ src/qkd_app/requirements.in | 12 ++++ src/qkd_app/service/__init__.py | 13 +++++ src/qkd_app/{ => service}/__main__.py | 0 6 files changed, 177 insertions(+) create mode 100644 src/qkd_app/.gitlab-ci.yml create mode 100644 src/qkd_app/Config.py create mode 100644 src/qkd_app/Dockerfile create mode 100644 src/qkd_app/requirements.in create mode 100644 src/qkd_app/service/__init__.py rename src/qkd_app/{ => service}/__main__.py (100%) diff --git a/src/qkd_app/.gitlab-ci.yml b/src/qkd_app/.gitlab-ci.yml new file mode 100644 index 000000000..490070321 --- /dev/null +++ b/src/qkd_app/.gitlab-ci.yml @@ -0,0 +1,80 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build app: + variables: + IMAGE_NAME: 'qkd_app' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test app: + variables: + IMAGE_NAME: 'qkd_app' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build app + - unit_test service + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker run --name $IMAGE_NAME -d -p 10070:10070 -p 8005:8005 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - sleep 5 + - docker ps -a + - docker logs $IMAGE_NAME + - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker rm -f $IMAGE_NAME + - docker network rm teraflowbridge + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml diff --git a/src/qkd_app/Config.py b/src/qkd_app/Config.py new file mode 100644 index 000000000..a5d4f2e0b --- /dev/null +++ b/src/qkd_app/Config.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/qkd_app/Dockerfile b/src/qkd_app/Dockerfile new file mode 100644 index 000000000..7f8819c32 --- /dev/null +++ b/src/qkd_app/Dockerfile @@ -0,0 +1,58 @@ +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/qkd_app +WORKDIR /var/teraflow/qkd_app +COPY src/qkd_app/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/. context/ +COPY src/interdomain/. interdomain/ +COPY src/pathcomp/. pathcomp/ +COPY src/service/. service/ +COPY src/qkd_app/. qkd_app/ + +# Start the service +ENTRYPOINT ["python", "-m", "qkd_app.service"] diff --git a/src/qkd_app/requirements.in b/src/qkd_app/requirements.in new file mode 100644 index 000000000..47dc59064 --- /dev/null +++ b/src/qkd_app/requirements.in @@ -0,0 +1,12 @@ +Flask==2.1.3 +Flask-HTTPAuth==4.5.0 +Flask-RESTful==0.3.9 +jsonschema==4.4.0 +requests==2.27.1 +werkzeug==2.3.7 +nats-py==2.6.* +psycopg2-binary==2.9.* +SQLAlchemy==1.4.* +sqlalchemy-cockroachdb==1.4.* +SQLAlchemy-Utils==0.38.* + diff --git a/src/qkd_app/service/__init__.py b/src/qkd_app/service/__init__.py new file mode 100644 index 000000000..07d088140 --- /dev/null +++ b/src/qkd_app/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/qkd_app/__main__.py b/src/qkd_app/service/__main__.py similarity index 100% rename from src/qkd_app/__main__.py rename to src/qkd_app/service/__main__.py -- GitLab From e8615355b4ff24349e034bcf81304a61b299f92f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 26 Sep 2024 11:46:31 +0000 Subject: [PATCH 2/2] Pre-merge code cleanup --- src/qkd_app/.gitlab-ci.yml | 2 +- src/qkd_app/Config.py | 3 +-- src/qkd_app/Dockerfile | 16 ++++++++++++++-- src/qkd_app/requirements.in | 15 ++++++++++++++- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/qkd_app/.gitlab-ci.yml b/src/qkd_app/.gitlab-ci.yml index 490070321..85ee2b5e0 100644 --- a/src/qkd_app/.gitlab-ci.yml +++ b/src/qkd_app/.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/qkd_app/Config.py b/src/qkd_app/Config.py index a5d4f2e0b..07d088140 100644 --- a/src/qkd_app/Config.py +++ b/src/qkd_app/Config.py @@ -1,4 +1,4 @@ -# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/src/qkd_app/Dockerfile b/src/qkd_app/Dockerfile index 7f8819c32..4d2b8d61b 100644 --- a/src/qkd_app/Dockerfile +++ b/src/qkd_app/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM python:3.9-slim # Install dependencies @@ -49,8 +63,6 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow COPY src/context/. context/ -COPY src/interdomain/. interdomain/ -COPY src/pathcomp/. pathcomp/ COPY src/service/. service/ COPY src/qkd_app/. qkd_app/ diff --git a/src/qkd_app/requirements.in b/src/qkd_app/requirements.in index 47dc59064..a9bce93b5 100644 --- a/src/qkd_app/requirements.in +++ b/src/qkd_app/requirements.in @@ -1,3 +1,17 @@ +# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + Flask==2.1.3 Flask-HTTPAuth==4.5.0 Flask-RESTful==0.3.9 @@ -9,4 +23,3 @@ psycopg2-binary==2.9.* SQLAlchemy==1.4.* sqlalchemy-cockroachdb==1.4.* SQLAlchemy-Utils==0.38.* - -- GitLab