From 4f9d8b551d72665035f5e8dffc230b074e2aa77d Mon Sep 17 00:00:00 2001
From: Lluis Gifre <lluis.gifre@cttc.es>
Date: Thu, 14 Jul 2022 20:08:14 +0200
Subject: [PATCH] Context component:
- Arranged requirements to accelerate Docker image creation
- Updated Dockerfile
- Arranged GitLab CI/CD pipeline
---
common_requirements.in | 8 +++++++
src/context/.gitlab-ci.yml | 4 ++++
src/context/Dockerfile | 45 ++++++++++++++++++++++++++-----------
src/context/requirements.in | 7 ------
4 files changed, 44 insertions(+), 20 deletions(-)
create mode 100644 common_requirements.in
diff --git a/common_requirements.in b/common_requirements.in
new file mode 100644
index 000000000..2616cf851
--- /dev/null
+++ b/common_requirements.in
@@ -0,0 +1,8 @@
+coverage==6.3
+grpcio==1.43.0
+grpcio-health-checking==1.43.0
+grpcio-tools==1.43.0
+prometheus-client==0.13.0
+protobuf==3.19.3
+pytest==6.2.5
+pytest-benchmark==3.4.1
diff --git a/src/context/.gitlab-ci.yml b/src/context/.gitlab-ci.yml
index dc302cbd3..58fba75e3 100644
--- a/src/context/.gitlab-ci.yml
+++ b/src/context/.gitlab-ci.yml
@@ -30,6 +30,8 @@ build context:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
- changes:
+ - src/common/**/*.py
+ - proto/*.proto
- src/$IMAGE_NAME/**/*.{py,in,yml}
- src/$IMAGE_NAME/Dockerfile
- src/$IMAGE_NAME/tests/*.py
@@ -68,6 +70,8 @@ unit test context:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
- changes:
+ - src/common/**/*.py
+ - proto/*.proto
- src/$IMAGE_NAME/**/*.{py,in,yml}
- src/$IMAGE_NAME/Dockerfile
- src/$IMAGE_NAME/tests/*.py
diff --git a/src/context/Dockerfile b/src/context/Dockerfile
index 1599e588f..97aa46964 100644
--- a/src/context/Dockerfile
+++ b/src/context/Dockerfile
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM python:3-slim
+FROM python:3.9-slim
# Install dependencies
RUN apt-get --yes --quiet --quiet update && \
@@ -28,22 +28,41 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
chmod +x /bin/grpc_health_probe
# Get generic Python packages
-RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --upgrade setuptools wheel
+RUN python3 -m pip install --upgrade pip-tools
-# Set working directory
+# Get common Python packages
+# Note: this step enables sharing the previous Docker build steps among all the Python components
WORKDIR /var/teraflow
+COPY common_requirements.in common_requirements.in
+RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in
+RUN python3 -m pip install -r common_requirements.txt
-# Create module sub-folders
-RUN mkdir -p /var/teraflow/context
+# Add common files into working directory
+WORKDIR /var/teraflow/common
+COPY src/common/. ./
+RUN rm -rf proto
+
+# Create proto sub-folder, copy .proto files, and generate Python code
+RUN mkdir -p /var/teraflow/common/proto
+WORKDIR /var/teraflow/common/proto
+RUN touch __init__.py
+COPY proto/*.proto ./
+RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto
+RUN rm *.proto
+RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;
-# Get Python packages per module
-COPY context/requirements.in context/requirements.in
-RUN pip-compile --output-file=context/requirements.txt context/requirements.in
-RUN python3 -m pip install -r context/requirements.txt
+# Create component sub-folder, get specific Python packages
+RUN mkdir -p /var/teraflow/context
+WORKDIR /var/teraflow/context
+COPY src/context/requirements.in requirements.in
+RUN pip-compile --quiet --output-file=requirements.txt requirements.in
+RUN python3 -m pip install -r requirements.txt
-# Add files into working directory
-COPY common/. common
-COPY context/. context
+# Add component files into working directory
+WORKDIR /var/teraflow
+COPY src/context/. context/
-# Start context service
+# Start the service
ENTRYPOINT ["python", "-m", "context.service"]
diff --git a/src/context/requirements.in b/src/context/requirements.in
index 4c32f5c71..83193ae70 100644
--- a/src/context/requirements.in
+++ b/src/context/requirements.in
@@ -1,11 +1,4 @@
Flask==2.0.2
Flask-RESTful==0.3.9
-grpcio==1.43.0
-grpcio-health-checking==1.43.0
-prometheus-client==0.13.0
-protobuf==3.19.3
-pytest==6.2.5
-pytest-benchmark==3.4.1
redis==4.1.2
requests==2.27.1
-coverage==6.3
--
GitLab