diff --git a/common_requirements.in b/common_requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..2616cf85133f9f4e623f9a5c7f9e91a02083b2e3 --- /dev/null +++ b/common_requirements.in @@ -0,0 +1,8 @@ +coverage==6.3 +grpcio==1.43.0 +grpcio-health-checking==1.43.0 +grpcio-tools==1.43.0 +prometheus-client==0.13.0 +protobuf==3.19.3 +pytest==6.2.5 +pytest-benchmark==3.4.1 diff --git a/src/context/.gitlab-ci.yml b/src/context/.gitlab-ci.yml index dc302cbd3aa7dbe197450c91a0e9c90714df6110..58fba75e311f7bae371849b56fa8922badeaf83e 100644 --- a/src/context/.gitlab-ci.yml +++ b/src/context/.gitlab-ci.yml @@ -30,6 +30,8 @@ build context: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: + - src/common/**/*.py + - proto/*.proto - src/$IMAGE_NAME/**/*.{py,in,yml} - src/$IMAGE_NAME/Dockerfile - src/$IMAGE_NAME/tests/*.py @@ -68,6 +70,8 @@ unit test context: - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' - changes: + - src/common/**/*.py + - proto/*.proto - src/$IMAGE_NAME/**/*.{py,in,yml} - src/$IMAGE_NAME/Dockerfile - src/$IMAGE_NAME/tests/*.py diff --git a/src/context/Dockerfile b/src/context/Dockerfile index 1599e588fb6d1dc6acae5f664208b4ce8d2d921b..97aa46964c65773e8e14401b044ff2e331f16c03 100644 --- a/src/context/Dockerfile +++ b/src/context/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3-slim +FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ @@ -28,22 +28,41 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ chmod +x /bin/grpc_health_probe # Get generic Python packages -RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools -# Set working directory +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt -# Create module sub-folders -RUN mkdir -p /var/teraflow/context +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -# Get Python packages per module -COPY context/requirements.in context/requirements.in -RUN pip-compile --output-file=context/requirements.txt context/requirements.in -RUN python3 -m pip install -r context/requirements.txt +# Create component sub-folder, get specific Python packages +RUN mkdir -p /var/teraflow/context +WORKDIR /var/teraflow/context +COPY src/context/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt -# Add files into working directory -COPY common/. common -COPY context/. context +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/context/. context/ -# Start context service +# Start the service ENTRYPOINT ["python", "-m", "context.service"] diff --git a/src/context/requirements.in b/src/context/requirements.in index 4c32f5c71c828a57ceeb90f2739e3b551c99abfe..83193ae7012c81881dbbb0412878cd759252f65e 100644 --- a/src/context/requirements.in +++ b/src/context/requirements.in @@ -1,11 +1,4 @@ Flask==2.0.2 Flask-RESTful==0.3.9 -grpcio==1.43.0 -grpcio-health-checking==1.43.0 -prometheus-client==0.13.0 -protobuf==3.19.3 -pytest==6.2.5 -pytest-benchmark==3.4.1 redis==4.1.2 requests==2.27.1 -coverage==6.3