Adapted Common, Context, Device, Monitoring to common proto folder

Common:
- Corrected Python package versions to prevent collisions with p4runtime

Context:
- Minor corrections in GitLab CI/CD pipeline

Device:
- Corrected Python package versions to prevent collisions with p4runtime

Monitoring:
- Arranged requirements to accelerate Docker image creation
- Corrected Python package versions to prevent collisions with p4runtime
- Updated Dockerfile
- Arranged GitLab CI/CD pipeline

common_requirements.in

 coverage==6.3
-grpcio==1.43.0
-grpcio-health-checking==1.43.0
-grpcio-tools==1.43.0
+grpcio==1.47.*
+grpcio-health-checking==1.47.*
+grpcio-tools==1.47.*
 prometheus-client==0.13.0
-protobuf==3.19.3
+protobuf==3.20.*
 pytest==6.2.5
 pytest-benchmark==3.4.1

src/context/.gitlab-ci.yml

@@ -55,8 +55,8 @@ unit test context:
     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
     - docker pull "redis:6.2"
     - docker run --name redis -d --network=teraflowbridge redis:6.2
-    - docker run --name $IMAGE_NAME -d -p 1010:1010 --env "DB_BACKEND=redis" --env "REDIS_SERVICE_HOST=redis" --env "REDIS_SERVICE_PORT=6379" --env "REDIS_DATABASE_ID=0" -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
     - sleep 10
+    - docker run --name $IMAGE_NAME -d -p 1010:1010 --env "DB_BACKEND=redis" --env "REDIS_SERVICE_HOST=redis" --env "REDIS_SERVICE_PORT=6379" --env "REDIS_DATABASE_ID=0" -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
     - docker ps -a
     - docker logs $IMAGE_NAME
     - docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml"

src/device/requirements.in

@@ -11,3 +11,9 @@ pytz==2021.3
 redis==4.1.2
 requests==2.27.1
 xmltodict==0.12.0
+
+# pip's dependency resolver does not take into account installed packages.
+# p4runtime does not specify the version of grpcio/protobuf it needs, so it tries to install latest one
+# adding here again grpcio==1.47.* and protobuf==3.20.* with explicit versions to prevent collisions
+grpcio==1.47.*
+protobuf==3.20.*

src/monitoring/.gitlab-ci.yml

@@ -21,7 +21,7 @@ build monitoring:
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/
+    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile .
     - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
     - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
   after_script:
@@ -30,6 +30,8 @@ build monitoring:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 
     - changes:
+      - src/common/**/*.py
+      - proto/*.proto
       - src/$IMAGE_NAME/**/*.{py,in,yml}
       - src/$IMAGE_NAME/Dockerfile
       - src/$IMAGE_NAME/tests/*.py
@@ -51,9 +53,10 @@ unit test monitoring:
     - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
   script:
     - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
-    - docker run --name influxdb -d -p 8086:8086 -e INFLUXDB_DB=$INFLUXDB_DATABASE -e INFLUXDB_ADMIN_USER=$INFLUXDB_USER -e INFLUXDB_ADMIN_PASSWORD=$INFLUXDB_PASSWORD -e INFLUXDB_HTTP_AUTH_ENABLED=True --network=teraflowbridge --rm influxdb:1.8
+    - docker pull "influxdb:1.8"
+    - docker run --name influxdb -d -p 8086:8086 -e INFLUXDB_DB=$INFLUXDB_DATABASE -e INFLUXDB_ADMIN_USER=$INFLUXDB_USER -e INFLUXDB_ADMIN_PASSWORD=$INFLUXDB_PASSWORD -e INFLUXDB_HTTP_AUTH_ENABLED=True --network=teraflowbridge influxdb:1.8
     - sleep 10
-    - docker run --name $IMAGE_NAME -d -p 7070:7070 --env INFLUXDB_USER=$INFLUXDB_USER --env INFLUXDB_PASSWORD=$INFLUXDB_PASSWORD --env INFLUXDB_DATABASE=$INFLUXDB_DATABASE --env INFLUXDB_HOSTNAME=influxdb --env INFLUXDB_PORT=8086 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge --rm $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+    - docker run --name $IMAGE_NAME -d -p 7070:7070 --env INFLUXDB_USER=$INFLUXDB_USER --env INFLUXDB_PASSWORD=$INFLUXDB_PASSWORD --env INFLUXDB_DATABASE=$INFLUXDB_DATABASE --env INFLUXDB_HOSTNAME=influxdb --env INFLUXDB_PORT=8086 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
     - sleep 30
     - docker ps -a
     - docker logs $IMAGE_NAME
@@ -68,6 +71,8 @@ unit test monitoring:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' 
     - changes:
+      - src/common/**/*.py
+      - proto/*.proto
       - src/$IMAGE_NAME/**/*.{py,in,yml}
       - src/$IMAGE_NAME/Dockerfile
       - src/$IMAGE_NAME/tests/*.py
@@ -78,7 +83,7 @@ unit test monitoring:
       when: always
       reports:
         junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml
- 
+
 # Deployment of the service in Kubernetes Cluster
 deploy monitoring:
   variables:

src/monitoring/Dockerfile

@@ -12,50 +12,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM python:3-slim
+FROM python:3.9-slim
 
 # Install dependencies
 RUN apt-get --yes --quiet --quiet update && \
     apt-get --yes --quiet --quiet install wget g++ && \
     rm -rf /var/lib/apt/lists/*
 
-# show python logs as they occur
+# Set Python to show logs as they occur
 ENV PYTHONUNBUFFERED=0
 
-# download the grpc health probe
+# Download the gRPC health probe
 RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
     chmod +x /bin/grpc_health_probe
 
 # Get generic Python packages
-RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --upgrade setuptools wheel
+RUN python3 -m pip install --upgrade pip-tools
 
-# Set working directory
+# Get common Python packages
+# Note: this step enables sharing the previous Docker build steps among all the Python components
 WORKDIR /var/teraflow
+COPY common_requirements.in common_requirements.in
+RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in
+RUN python3 -m pip install -r common_requirements.txt
 
-# Create module sub-folders
+# Add common files into working directory
+WORKDIR /var/teraflow/common
+COPY src/common/. ./
+RUN rm -rf proto
+
+# Create proto sub-folder, copy .proto files, and generate Python code
+RUN mkdir -p /var/teraflow/common/proto
+WORKDIR /var/teraflow/common/proto
+RUN touch __init__.py
+COPY proto/*.proto ./
+RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto
+RUN rm *.proto
+RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;
+
+# Create component sub-folders, get specific Python packages
 RUN mkdir -p /var/teraflow/monitoring
-RUN mkdir -p /var/teraflow/common
-RUN mkdir -p /var/teraflow/common/tools
-RUN mkdir -p /var/teraflow/common/rpc_method_wrapper
-RUN mkdir -p /var/teraflow/device
-RUN mkdir -p /var/teraflow/context
-
-# Get Python packages per module
-COPY monitoring/requirements.in requirements.in
-RUN pip-compile --output-file=requirements.txt requirements.in
+WORKDIR /var/teraflow/monitoring
+COPY src/monitoring/requirements.in requirements.in
+RUN pip-compile --quiet --output-file=requirements.txt requirements.in
 RUN python3 -m pip install -r requirements.txt
 
-# add files into working directory
-COPY monitoring/. monitoring
-COPY common/. common
-COPY context/. context
-COPY device/. device
-
-RUN rm -r common/message_broker/tests
-RUN rm -r common/orm/tests
-RUN rm -r common/rpc_method_wrapper/tests
-RUN rm -r context/tests/test_unitary.py
+# Add component files into working directory
+WORKDIR /var/teraflow
+COPY src/context/. context/
+COPY src/device/. device/
+COPY src/monitoring/. monitoring/
 
+# Start the service
 ENTRYPOINT ["python", "-m", "monitoring.service"]
-

src/monitoring/requirements.in

 anytree==2.8.0
 APScheduler==3.8.1
 fastcache==1.1.0
-grpcio==1.43.0
-grpcio-health-checking==1.43.0
 #google-api-core
 #opencensus[stackdriver]
 #google-cloud-profiler
@@ -11,10 +9,6 @@ Jinja2==3.0.3
 ncclient==0.6.13
 p4runtime==1.3.0
 paramiko==2.9.2
-prometheus-client==0.13.0
-protobuf==3.19.3
-pytest==6.2.5
-pytest-benchmark==3.4.1
 influxdb
 python-dateutil==2.8.2
 python-json-logger==2.0.2
@@ -22,4 +16,9 @@ pytz==2021.3
 redis==4.1.2
 requests==2.27.1
 xmltodict==0.12.0
-coverage==6.3
+
+# pip's dependency resolver does not take into account installed packages.
+# p4runtime does not specify the version of grpcio/protobuf it needs, so it tries to install latest one
+# adding here again grpcio==1.47.* and protobuf==3.20.* with explicit versions to prevent collisions
+grpcio==1.47.*
+protobuf==3.20.*