From d5b09cd93c6511daa442be783405ea1ddcfbfeb3 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Mon, 8 Sep 2025 11:09:30 +0200 Subject: [PATCH 1/5] Initial migration of the context component to Python 3.13. --- common_requirements.in | 24 ++++++++++++------------ src/common/tools/grpc/Tools.py | 19 ++++++++++++++++--- src/context/Dockerfile | 4 ++-- src/context/requirements.in | 8 ++++---- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/common_requirements.in b/common_requirements.in index 12c6c778d..de1a6ed20 100644 --- a/common_requirements.in +++ b/common_requirements.in @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -coverage==6.3 -grpcio==1.47.* -grpcio-health-checking==1.47.* -grpcio-reflection==1.47.* -grpcio-tools==1.47.* -grpclib==0.4.4 -prettytable==3.5.0 -prometheus-client==0.13.0 -protobuf==3.20.* -pytest==6.2.5 -pytest-benchmark==3.4.1 -python-dateutil==2.8.2 +coverage==7.9.* +grpcio==1.73.* +grpcio-health-checking==1.73.* +grpcio-reflection==1.73.* +grpcio-tools==1.73.* +grpclib==0.4.8 +prettytable==3.16.0 +prometheus-client==0.22.* +protobuf==6.32.* +pytest==8.4.1 +pytest-benchmark==5.1.0 +python-dateutil==2.9.0 pytest-depends==1.0.1 diff --git a/src/common/tools/grpc/Tools.py b/src/common/tools/grpc/Tools.py index 9a64c4870..8184343d6 100644 --- a/src/common/tools/grpc/Tools.py +++ b/src/common/tools/grpc/Tools.py @@ -13,15 +13,28 @@ # limitations under the License. import json +import warnings +from google import protobuf from google.protobuf.json_format import MessageToDict def grpc_message_to_json( message, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=False ): if not hasattr(message, 'DESCRIPTOR'): return json.dumps(str(message), sort_keys=True) # not a gRPC message - return MessageToDict( - message, including_default_value_fields=including_default_value_fields, - preserving_proto_field_name=preserving_proto_field_name, use_integers_for_enums=use_integers_for_enums) + if including_default_value_fields is not True: + warnings.warn( + "The 'including_default_value_fields' argument is deprecated and will have no effect in future versions.", + DeprecationWarning, + stacklevel=2 + ) + if protobuf.__version__ < '4.0.0': + return MessageToDict( + message, including_default_value_fields=including_default_value_fields, + preserving_proto_field_name=preserving_proto_field_name, use_integers_for_enums=use_integers_for_enums) + else: + return MessageToDict( + message, + preserving_proto_field_name=preserving_proto_field_name, use_integers_for_enums=use_integers_for_enums) def grpc_message_list_to_json(message_list): if message_list is None: return None diff --git a/src/context/Dockerfile b/src/context/Dockerfile index a4bf84153..eb999c3e4 100644 --- a/src/context/Dockerfile +++ b/src/context/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.9-slim +FROM python:3.13-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ @@ -23,7 +23,7 @@ RUN apt-get --yes --quiet --quiet update && \ ENV PYTHONUNBUFFERED=0 # Download the gRPC health probe -RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ +RUN GRPC_HEALTH_PROBE_VERSION=v0.4.40 && \ wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ chmod +x /bin/grpc_health_probe diff --git a/src/context/requirements.in b/src/context/requirements.in index f5c809461..907cd1ac6 100644 --- a/src/context/requirements.in +++ b/src/context/requirements.in @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -nats-py==2.6.* +nats-py==2.10.* psycopg2-binary==2.9.* -SQLAlchemy==1.4.* -sqlalchemy-cockroachdb==1.4.* -SQLAlchemy-Utils==0.38.* +SQLAlchemy==1.4.* # TODO: Update to 2.0 due to deprecated build step +sqlalchemy-cockroachdb==1.4.* # TODO: Update to 2.0 due to deprecated build step +SQLAlchemy-Utils==0.41.* -- GitLab From 4aae0fda86d3dad02379011d563008c860ada749 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Mon, 8 Sep 2025 12:07:56 +0200 Subject: [PATCH 2/5] Upgrading version of the SQLAlchemy for the context component. --- src/context/requirements.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/context/requirements.in b/src/context/requirements.in index 907cd1ac6..2bb37aa88 100644 --- a/src/context/requirements.in +++ b/src/context/requirements.in @@ -14,6 +14,6 @@ nats-py==2.10.* psycopg2-binary==2.9.* -SQLAlchemy==1.4.* # TODO: Update to 2.0 due to deprecated build step -sqlalchemy-cockroachdb==1.4.* # TODO: Update to 2.0 due to deprecated build step -SQLAlchemy-Utils==0.41.* +SQLAlchemy==2.0.* +sqlalchemy-cockroachdb==2.0.* +SQLAlchemy-Utils==0.42.* -- GitLab From 52fd48942be6a38662b86ed9a6ed9409dc410b52 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Fri, 19 Sep 2025 08:59:20 +0200 Subject: [PATCH 3/5] Testing a new strategy for upgrading through a different common_requirements file. --- common_requirements.in | 25 ++++++++++++------------- common_requirements_py313.in | 27 +++++++++++++++++++++++++++ src/service/Dockerfile | 2 +- 3 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 common_requirements_py313.in diff --git a/common_requirements.in b/common_requirements.in index de1a6ed20..ebd726e41 100644 --- a/common_requirements.in +++ b/common_requirements.in @@ -12,16 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -coverage==7.9.* -grpcio==1.73.* -grpcio-health-checking==1.73.* -grpcio-reflection==1.73.* -grpcio-tools==1.73.* -grpclib==0.4.8 -prettytable==3.16.0 -prometheus-client==0.22.* -protobuf==6.32.* -pytest==8.4.1 -pytest-benchmark==5.1.0 -python-dateutil==2.9.0 -pytest-depends==1.0.1 +coverage==6.3 +grpcio==1.47.* +grpcio-health-checking==1.47.* +grpcio-reflection==1.47.* +grpcio-tools==1.47.* +grpclib==0.4.4 +prettytable==3.5.0 +prometheus-client==0.13.0 +protobuf==3.20.* +pytest==6.2.5 +pytest-benchmark==3.4.1 +python-dateutil==2.8.2 diff --git a/common_requirements_py313.in b/common_requirements_py313.in new file mode 100644 index 000000000..de1a6ed20 --- /dev/null +++ b/common_requirements_py313.in @@ -0,0 +1,27 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +coverage==7.9.* +grpcio==1.73.* +grpcio-health-checking==1.73.* +grpcio-reflection==1.73.* +grpcio-tools==1.73.* +grpclib==0.4.8 +prettytable==3.16.0 +prometheus-client==0.22.* +protobuf==6.32.* +pytest==8.4.1 +pytest-benchmark==5.1.0 +python-dateutil==2.9.0 +pytest-depends==1.0.1 diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 49efe9829..b415f55a3 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -35,7 +35,7 @@ RUN python3 -m pip install --upgrade pip-tools # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components WORKDIR /var/teraflow -COPY common_requirements.in common_requirements.in +COPY common_requirements_py313.in common_requirements.in RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in RUN python3 -m pip install -r common_requirements.txt -- GitLab From 06eb46c6747bcea6757bc5844e2f6d3e36403aed Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Mon, 22 Sep 2025 14:41:09 +0200 Subject: [PATCH 4/5] Improving probes for Kubernetes --- manifests/contextservice.yaml | 13 +++++++++---- manifests/serviceservice.yaml | 13 +++++++++---- src/context/Dockerfile | 10 +++++----- src/context/requirements.in | 6 +++--- src/service/Dockerfile | 10 +++++----- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml index 5592864d6..3ac332389 100644 --- a/manifests/contextservice.yaml +++ b/manifests/contextservice.yaml @@ -52,12 +52,17 @@ spec: name: crdb-data - secretRef: name: nats-data + startupProbe: + grpc: + port: 1010 + failureThreshold: 30 + periodSeconds: 1 readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:1010"] + grpc: + port: 1010 livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:1010"] + grpc: + port: 1010 resources: requests: cpu: 250m diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index 8262550ef..4f65cd137 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -37,12 +37,17 @@ spec: env: - name: LOG_LEVEL value: "INFO" + startupProbe: + grpc: + port: 3030 + failureThreshold: 30 + periodSeconds: 1 readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:3030"] + grpc: + port: 3030 livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:3030"] + grpc: + port: 3030 resources: requests: cpu: 250m diff --git a/src/context/Dockerfile b/src/context/Dockerfile index eb999c3e4..add63fe65 100644 --- a/src/context/Dockerfile +++ b/src/context/Dockerfile @@ -23,9 +23,9 @@ RUN apt-get --yes --quiet --quiet update && \ ENV PYTHONUNBUFFERED=0 # Download the gRPC health probe -RUN GRPC_HEALTH_PROBE_VERSION=v0.4.40 && \ - wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ - chmod +x /bin/grpc_health_probe +# RUN GRPC_HEALTH_PROBE_VERSION=v0.4.40 && \ +# wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ +# chmod +x /bin/grpc_health_probe # Get generic Python packages RUN python3 -m pip install --upgrade pip @@ -35,7 +35,7 @@ RUN python3 -m pip install --upgrade pip-tools # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components WORKDIR /var/teraflow -COPY common_requirements.in common_requirements.in +COPY common_requirements_py313.in common_requirements.in RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in RUN python3 -m pip install -r common_requirements.txt @@ -57,7 +57,7 @@ RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; RUN mkdir -p /var/teraflow/context WORKDIR /var/teraflow/context COPY src/context/requirements.in requirements.in -RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in /var/teraflow/common_requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory diff --git a/src/context/requirements.in b/src/context/requirements.in index 2bb37aa88..907cd1ac6 100644 --- a/src/context/requirements.in +++ b/src/context/requirements.in @@ -14,6 +14,6 @@ nats-py==2.10.* psycopg2-binary==2.9.* -SQLAlchemy==2.0.* -sqlalchemy-cockroachdb==2.0.* -SQLAlchemy-Utils==0.42.* +SQLAlchemy==1.4.* # TODO: Update to 2.0 due to deprecated build step +sqlalchemy-cockroachdb==1.4.* # TODO: Update to 2.0 due to deprecated build step +SQLAlchemy-Utils==0.41.* diff --git a/src/service/Dockerfile b/src/service/Dockerfile index b415f55a3..493094769 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.9-slim +FROM python:3.13-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ @@ -23,9 +23,9 @@ RUN apt-get --yes --quiet --quiet update && \ ENV PYTHONUNBUFFERED=0 # Download the gRPC health probe -RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ - wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ - chmod +x /bin/grpc_health_probe +# RUN GRPC_HEALTH_PROBE_VERSION=v0.4.40 && \ +# wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ +# chmod +x /bin/grpc_health_probe # Get generic Python packages RUN python3 -m pip install --upgrade pip @@ -57,7 +57,7 @@ RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; RUN mkdir -p /var/teraflow/service WORKDIR /var/teraflow/service COPY src/service/requirements.in requirements.in -RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in /var/teraflow/common_requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory -- GitLab From 0f3f508db06806e37326250595aca2648b0c5e17 Mon Sep 17 00:00:00 2001 From: Carlos Natalino Date: Mon, 22 Sep 2025 15:03:22 +0200 Subject: [PATCH 5/5] Including version compatibility for the Decorator class. --- src/common/method_wrappers/Decorator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/common/method_wrappers/Decorator.py b/src/common/method_wrappers/Decorator.py index 6bd9f8217..ef0c6f99e 100644 --- a/src/common/method_wrappers/Decorator.py +++ b/src/common/method_wrappers/Decorator.py @@ -128,7 +128,10 @@ class MetricsPool: else: raise Exception('Unsupported metric: {:s}'.format(raw_metric_name)) # pragma: no cover metric_data = method_to_metric_fields.setdefault(method_name, dict()).setdefault(metric_name, dict()) - for field_name,labels,value,_,_ in raw_metric_data._child_samples(): + for values in raw_metric_data._child_samples(): + field_name = values[0] + labels = values[1] + value = values[2] if field_name == '_bucket': bucket_bounds.add(labels['le']) if len(labels) > 0: field_name = '{:s}:{:s}'.format(field_name, json.dumps(labels, sort_keys=True)) metric_data[field_name] = value -- GitLab