Context component:

- Arranged requirements to accelerate Docker image creation
- Updated Dockerfile
- Arranged GitLab CI/CD pipeline

common_requirements.in

+coverage==6.3
+grpcio==1.43.0
+grpcio-health-checking==1.43.0
+grpcio-tools==1.43.0
+prometheus-client==0.13.0
+protobuf==3.19.3
+pytest==6.2.5
+pytest-benchmark==3.4.1

src/context/.gitlab-ci.yml

@@ -30,6 +30,8 @@ build context:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 
     - changes:
+      - src/common/**/*.py
+      - proto/*.proto
       - src/$IMAGE_NAME/**/*.{py,in,yml}
       - src/$IMAGE_NAME/Dockerfile
       - src/$IMAGE_NAME/tests/*.py
@@ -68,6 +70,8 @@ unit test context:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 
     - changes:
+      - src/common/**/*.py
+      - proto/*.proto
       - src/$IMAGE_NAME/**/*.{py,in,yml}
       - src/$IMAGE_NAME/Dockerfile
       - src/$IMAGE_NAME/tests/*.py

src/context/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM python:3-slim
+FROM python:3.9-slim
 
 # Install dependencies
 RUN apt-get --yes --quiet --quiet update && \
@@ -28,22 +28,41 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
     chmod +x /bin/grpc_health_probe
 
 # Get generic Python packages
-RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --upgrade setuptools wheel
+RUN python3 -m pip install --upgrade pip-tools
 
-# Set working directory
+# Get common Python packages
+# Note: this step enables sharing the previous Docker build steps among all the Python components
 WORKDIR /var/teraflow
+COPY common_requirements.in common_requirements.in
+RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in
+RUN python3 -m pip install -r common_requirements.txt
 
-# Create module sub-folders
-RUN mkdir -p /var/teraflow/context
+# Add common files into working directory
+WORKDIR /var/teraflow/common
+COPY src/common/. ./
+RUN rm -rf proto
+
+# Create proto sub-folder, copy .proto files, and generate Python code
+RUN mkdir -p /var/teraflow/common/proto
+WORKDIR /var/teraflow/common/proto
+RUN touch __init__.py
+COPY proto/*.proto ./
+RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto
+RUN rm *.proto
+RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;
 
-# Get Python packages per module
-COPY context/requirements.in context/requirements.in
-RUN pip-compile --output-file=context/requirements.txt context/requirements.in
-RUN python3 -m pip install -r context/requirements.txt
+# Create component sub-folder, get specific Python packages
+RUN mkdir -p /var/teraflow/context
+WORKDIR /var/teraflow/context
+COPY src/context/requirements.in requirements.in
+RUN pip-compile --quiet --output-file=requirements.txt requirements.in
+RUN python3 -m pip install -r requirements.txt
 
-# Add files into working directory
-COPY common/. common
-COPY context/. context
+# Add component files into working directory
+WORKDIR /var/teraflow
+COPY src/context/. context/
 
-# Start context service
+# Start the service
 ENTRYPOINT ["python", "-m", "context.service"]

src/context/requirements.in

 Flask==2.0.2
 Flask-RESTful==0.3.9
-grpcio==1.43.0
-grpcio-health-checking==1.43.0
-prometheus-client==0.13.0
-protobuf==3.19.3
-pytest==6.2.5
-pytest-benchmark==3.4.1
 redis==4.1.2
 requests==2.27.1
-coverage==6.3