From 98f84af7e3a7a76c7ed7cfce2cc34bca07d6c14e Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Fri, 24 Oct 2025 06:06:52 +0000 Subject: [PATCH 01/15] feat: Implement DSCM Pluggable Service with InMemory Dictionary - Added Dockerfile for building the DSCM service container. - Created service and client modules for managing pluggable devices. - Implemented gRPC methods for Create, Get, List, Delete, and Configure pluggables. - Developed unit tests for all service functionalities. - Added message creation utilities for test cases. - Established logging and error handling mechanisms. - Included README files for documentation. --- proto/dscm_pluggable.proto | 123 +++++++++ proto/policy.proto | 2 +- .../run_tests_locally-service-pluggable.sh | 23 ++ src/common/Constants.py | 2 + .../method_wrappers/ServiceExceptions.py | 10 +- src/dscm/.gitlab-ci.yml | 13 + src/dscm/Dockerfile | 68 +++++ src/dscm/README.md | 213 ++++++++++++++++ src/dscm/__init__.py | 14 ++ src/dscm/client/DscmPluggableClient.py | 94 +++++++ src/dscm/client/__init__.py | 14 ++ src/dscm/requirements.in | 0 src/dscm/service/DscmPluggableService.py | 28 +++ .../DscmPluggableServiceServicerImpl.py | 201 +++++++++++++++ src/dscm/service/__init__.py | 14 ++ src/dscm/service/__main__.py | 51 ++++ src/dscm/tests/__init__.py | 14 ++ src/dscm/tests/test_DscmPluggables.py | 236 ++++++++++++++++++ src/dscm/tests/testmessages.py | 230 +++++++++++++++++ 19 files changed, 1344 insertions(+), 6 deletions(-) create mode 100644 proto/dscm_pluggable.proto create mode 100755 scripts/run_tests_locally-service-pluggable.sh create mode 100644 src/dscm/.gitlab-ci.yml create mode 100644 src/dscm/Dockerfile create mode 100644 src/dscm/README.md create mode 100644 src/dscm/__init__.py create mode 100644 src/dscm/client/DscmPluggableClient.py create mode 100644 src/dscm/client/__init__.py create mode 100644 src/dscm/requirements.in create mode 100644 src/dscm/service/DscmPluggableService.py create mode 100644 src/dscm/service/DscmPluggableServiceServicerImpl.py create mode 100644 src/dscm/service/__init__.py create mode 100644 src/dscm/service/__main__.py create mode 100644 src/dscm/tests/__init__.py create mode 100644 src/dscm/tests/test_DscmPluggables.py create mode 100644 src/dscm/tests/testmessages.py diff --git a/proto/dscm_pluggable.proto b/proto/dscm_pluggable.proto new file mode 100644 index 000000000..95ec83360 --- /dev/null +++ b/proto/dscm_pluggable.proto @@ -0,0 +1,123 @@ +syntax = "proto3"; + +package tfs.dscm.v0; + +import "context.proto"; +import "google/protobuf/field_mask.proto"; + +service DscmPluggableService { + rpc CreatePluggable (CreatePluggableRequest) returns (Pluggable) {} + rpc ListPluggables (ListPluggablesRequest) returns (ListPluggablesResponse) {} + rpc GetPluggable (GetPluggableRequest) returns (Pluggable) {} + rpc DeletePluggable (DeletePluggableRequest) returns (context.Empty) {} + rpc ConfigurePluggable (ConfigurePluggableRequest) returns (Pluggable) {} +} + + + +message PluggableId { + context.DeviceId device = 1; + int32 pluggable_index = 2; // physical slot number in the device +} + +message DigitalSubcarrierGroupId { + PluggableId pluggable = 1; + int32 group_index = 2; // Group id within the pluggable +} + +message DigitalSubcarrierId { + DigitalSubcarrierGroupId group = 1; + int32 subcarrier_index = 2; // Subcarrier index within the group +} + +message DigitalSubcarrierConfig { + DigitalSubcarrierId id = 1; + bool active = 2; + double target_output_power_dbm = 3; + double center_frequency_hz = 10; + double symbol_rate_baud = 11; +} + +message DigitalSubcarrierState { + DigitalSubcarrierId id = 1; + bool active = 2; + double measured_output_power_dbm = 3; + double osnr_db = 4; + context.Timestamp updated_at = 10; +} + +message DigitalSubcarrierGroupConfig { + DigitalSubcarrierGroupId id = 1; + int32 group_size = 2; // expected number of DSCs + double group_capacity_gbps = 3; // from YANG group capacity in Gbps + double subcarrier_spacing_mhz = 4; // from YANG digital-subcarrier-spacing + repeated DigitalSubcarrierConfig subcarriers = 10; +} + +message DigitalSubcarrierGroupState { + DigitalSubcarrierGroupId id = 1; + int32 count = 2; // available DSCs + double group_capacity_gbps = 3; + double subcarrier_spacing_mhz = 4; + repeated DigitalSubcarrierState subcarriers = 10; + context.Timestamp updated_at = 20; +} + +message PluggableConfig { + PluggableId id = 1; + repeated DigitalSubcarrierGroupConfig dsc_groups = 10; +} + +message PluggableState { + PluggableId id = 1; + repeated DigitalSubcarrierGroupState dsc_groups = 10; + context.Timestamp updated_at = 20; +} + +message Pluggable { + PluggableId id = 1; + PluggableConfig config = 2; + PluggableState state = 3; +} + +// ----------------------------------------------------------------------------- +// RPC I/O Messages +// ----------------------------------------------------------------------------- +message CreatePluggableRequest { + context.DeviceId device = 1; + int32 preferred_pluggable_index = 2; // -1 for auto (physical slot number in the device(router)) + PluggableConfig initial_config = 10; +} + +message ListPluggablesRequest { + context.DeviceId device = 1; + View view_level = 2; +} + +message ListPluggablesResponse { + repeated Pluggable pluggables = 1; +} + +message GetPluggableRequest { + PluggableId id = 1; + View view_level = 2; +} + +message DeletePluggableRequest { + PluggableId id = 1; +} + +message ConfigurePluggableRequest { + PluggableConfig config = 1; + google.protobuf.FieldMask update_mask = 2; // Not Implemented yet (for partial updates) + View view_level = 3; + int32 apply_timeout_seconds = 10; // Not Implemented yet (for timeout) +} + +// to control the level of detail in responses +enum View { + VIEW_UNSPECIFIED = 0; + VIEW_CONFIG = 1; + VIEW_STATE = 2; + VIEW_FULL = 3; +} diff --git a/proto/policy.proto b/proto/policy.proto index 51ea63b7f..d788f86aa 100644 --- a/proto/policy.proto +++ b/proto/policy.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package policy; import "context.proto"; -import "policy_condition.proto"; +import "policy_condition.proto"; // WARNING: Not used import "policy_action.proto"; import "monitoring.proto"; // to be migrated to: "kpi_manager.proto" diff --git a/scripts/run_tests_locally-service-pluggable.sh b/scripts/run_tests_locally-service-pluggable.sh new file mode 100755 index 000000000..be977b5a6 --- /dev/null +++ b/scripts/run_tests_locally-service-pluggable.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECTDIR=`pwd` +cd $PROJECTDIR/src +# RCFILE=$PROJECTDIR/coverage/.coveragerc + +python3 -m pytest --log-level=info --log-cli-level=info --verbose \ + dscm/tests/test_DscmPluggables.py + +echo "Bye!" diff --git a/src/common/Constants.py b/src/common/Constants.py index a5dca46a2..d00599ac2 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -75,6 +75,7 @@ class ServiceNameEnum(Enum): ANALYTICSBACKEND = 'analytics-backend' QOSPROFILE = 'qos-profile' OSMCLIENT = 'osm-client' + DSCMPLUGGABLE = 'dscm-pluggable' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -117,6 +118,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.ANALYTICSBACKEND .value : 30090, ServiceNameEnum.AUTOMATION .value : 30200, ServiceNameEnum.OSMCLIENT .value : 30210, + ServiceNameEnum.DSCMPLUGGABLE .value : 30220, # Used for test and debugging only ServiceNameEnum.DLT_GATEWAY .value : 50051, diff --git a/src/common/method_wrappers/ServiceExceptions.py b/src/common/method_wrappers/ServiceExceptions.py index 3d06c00a5..842318832 100644 --- a/src/common/method_wrappers/ServiceExceptions.py +++ b/src/common/method_wrappers/ServiceExceptions.py @@ -33,21 +33,21 @@ class NotFoundException(ServiceException): class AlreadyExistsException(ServiceException): def __init__( - self, object_name : str, object_uuid: str, extra_details : Union[str, Iterable[str]] = None + self, object_name : str, object_uuid: str, extra_details : Union[str, Iterable[str]] = [] ) -> None: details = '{:s}({:s}) already exists'.format(str(object_name), str(object_uuid)) super().__init__(grpc.StatusCode.ALREADY_EXISTS, details, extra_details=extra_details) class InvalidArgumentException(ServiceException): def __init__( - self, argument_name : str, argument_value: str, extra_details : Union[str, Iterable[str]] = None + self, argument_name : str, argument_value: str, extra_details : Union[str, Iterable[str]] = [] ) -> None: details = '{:s}({:s}) is invalid'.format(str(argument_name), str(argument_value)) super().__init__(grpc.StatusCode.INVALID_ARGUMENT, details, extra_details=extra_details) class InvalidArgumentsException(ServiceException): def __init__( - self, arguments : List[Tuple[str, str]], extra_details : Union[str, Iterable[str]] = None + self, arguments : List[Tuple[str, str]], extra_details : Union[str, Iterable[str]] = [] ) -> None: str_arguments = ', '.join(['{:s}({:s})'.format(name, value) for name,value in arguments]) details = 'Arguments {:s} are invalid'.format(str_arguments) @@ -55,14 +55,14 @@ class InvalidArgumentsException(ServiceException): class OperationFailedException(ServiceException): def __init__( - self, operation : str, extra_details : Union[str, Iterable[str]] = None + self, operation : str, extra_details : Union[str, Iterable[str]] = [] ) -> None: details = 'Operation({:s}) failed'.format(str(operation)) super().__init__(grpc.StatusCode.INTERNAL, details, extra_details=extra_details) class NotImplementedException(ServiceException): def __init__( - self, operation : str, extra_details : Union[str, Iterable[str]] = None + self, operation : str, extra_details : Union[str, Iterable[str]] = [] ) -> None: details = 'Operation({:s}) not implemented'.format(str(operation)) super().__init__(grpc.StatusCode.UNIMPLEMENTED, details, extra_details=extra_details) diff --git a/src/dscm/.gitlab-ci.yml b/src/dscm/.gitlab-ci.yml new file mode 100644 index 000000000..7363515f0 --- /dev/null +++ b/src/dscm/.gitlab-ci.yml @@ -0,0 +1,13 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/dscm/Dockerfile b/src/dscm/Dockerfile new file mode 100644 index 000000000..22c1c92ae --- /dev/null +++ b/src/dscm/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ git && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +WORKDIR /var/teraflow +COPY common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /var/teraflow/common +COPY src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /var/teraflow/common/proto +WORKDIR /var/teraflow/common/proto +RUN touch __init__.py +COPY proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create component sub-folders, get specific Python packages +RUN mkdir -p /var/teraflow/dscm +WORKDIR /var/teraflow/dscm +COPY src/dscm/requirements.in requirements.in +RUN pip-compile --quiet --output-file=requirements.txt requirements.in +RUN python3 -m pip install -r requirements.txt + +# Add component files into working directory +WORKDIR /var/teraflow +COPY src/dscm/. dscm/ + +# # Start the service +ENTRYPOINT [ "python", "-m", "dscm.service" ] diff --git a/src/dscm/README.md b/src/dscm/README.md new file mode 100644 index 000000000..b76723ecf --- /dev/null +++ b/src/dscm/README.md @@ -0,0 +1,213 @@ +# DSCM (Digital Subcarrier Multipelxed) Service + +## Overview + +The DSCM service provides gRPC-based management for optical pluggables and their digital subcarrier groups. It enables configuration and monitoring of coherent optical transceivers with support for multi-carrier operation. + +## Key Concepts + +### Pluggable +An optical transceiver module installed in a device (router/switch) at a specific physical slot index. + +### Digital Subcarrier Group (DSC Group) +A logical grouping of digital subcarriers within a pluggable, representing a coherent optical channel with shared parameters: +- **group_size**: Expected number of subcarriers in the group +- **group_capacity_gbps**: Total capacity in Gbps (e.g., 400G) +- **subcarrier_spacing_mhz**: Frequency spacing between subcarriers (e.g., 75 MHz) + +### Digital Subcarrier +Individual frequency channels within a DSC group with configurable parameters: +- **active**: Enable/disable the subcarrier +- **target_output_power_dbm**: Transmit power in dBm +- **center_frequency_hz**: Carrier frequency in Hz (e.g., 193.1 THz) +- **symbol_rate_baud**: Symbol rate in Baud (e.g., 64 GBaud) + +## Service API + +### gRPC Methods + +| Method | Request | Response | Description | +|--------|---------|----------|-------------| +| `CreatePluggable` | `CreatePluggableRequest` | `Pluggable` | Create a new pluggable with optional initial configuration | +| `GetPluggable` | `GetPluggableRequest` | `Pluggable` | Retrieve a specific pluggable by ID | +| `ListPluggables` | `ListPluggablesRequest` | `ListPluggablesResponse` | List all pluggables for a device | +| `DeletePluggable` | `DeletePluggableRequest` | `Empty` | Remove a pluggable from management | +| `ConfigurePluggable` | `ConfigurePluggableRequest` | `Pluggable` | Apply full configuration to a pluggable | + +### View Levels + +Control the level of detail in responses: +- `VIEW_UNSPECIFIED (0)`: Default, returns full pluggable +- `VIEW_CONFIG (1)`: Returns only configuration data +- `VIEW_STATE (2)`: Returns only state/telemetry data +- `VIEW_FULL (3)`: Returns complete pluggable (config + state) + +## Usage Examples + +### 1. Creating a Pluggable Without Configuration + +```python +from dscm.client.DscmPluggableClient import DscmPluggableClient +from dscm.tests.testmessages import create_pluggable_request + +# Create client +client = DscmPluggableClient() + +# Create pluggable request (auto-assign index) +request = create_pluggable_request( + device_uuid="550e8400-e29b-41d4-a716-446655440000", + preferred_pluggable_index=-1 # -1 for auto-assignment +) + +# Create pluggable +pluggable = client.CreatePluggable(request) +print(f"Created pluggable at index: {pluggable.id.pluggable_index}") + +# Close client +client.close() +``` + +### 2. Creating a Pluggable With Initial Configuration + +```python +from dscm.tests.testmessages import create_pluggable_request + +# Create pluggable with optical channel configuration +request = create_pluggable_request( + device_uuid="550e8400-e29b-41d4-a716-446655440000", + preferred_pluggable_index=0, # Physical slot 0 + with_initial_config=True # Include DSC group configuration +) + +pluggable = client.CreatePluggable(request) + +# Verify configuration +assert len(pluggable.config.dsc_groups) == 1 +dsc_group = pluggable.config.dsc_groups[0] +assert dsc_group.group_size == 4 +assert dsc_group.group_capacity_gbps == 400.0 +``` + +### 3. Listing Pluggables with View Filtering + +```python +from common.proto.dscm_pluggable_pb2 import View +from dscm.tests.testmessages import create_list_pluggables_request + +# List only configuration (no state data) +request = create_list_pluggables_request( + device_uuid="550e8400-e29b-41d4-a716-446655440000", + view_level=View.VIEW_CONFIG +) + +response = client.ListPluggables(request) +for pluggable in response.pluggables: + print(f"Pluggable {pluggable.id.pluggable_index}: {len(pluggable.config.dsc_groups)} DSC groups") +``` + +### 4. Getting a Specific Pluggable + +```python +from dscm.tests.testmessages import create_get_pluggable_request + +# Get full pluggable details +request = create_get_pluggable_request( + device_uuid="550e8400-e29b-41d4-a716-446655440000", + pluggable_index=0, + view_level=View.VIEW_FULL +) + +pluggable = client.GetPluggable(request) +print(f"Device: {pluggable.id.device.device_uuid.uuid}") +print(f"Index: {pluggable.id.pluggable_index}") +print(f"DSC Groups: {len(pluggable.config.dsc_groups)}") +``` + +### 5. Configuring a Pluggable + +```python +from dscm.tests.testmessages import create_configure_pluggable_request + +# Apply full configuration (reconfigure optical channels) +request = create_configure_pluggable_request( + device_uuid="550e8400-e29b-41d4-a716-446655440000", + pluggable_index=0, + view_level=View.VIEW_CONFIG, + apply_timeout_seconds=30 +) + +# Configuration includes: +# - 1 DSC group with 400G capacity, 75 MHz spacing +# - 2 digital subcarriers at 193.1 THz and 193.175 THz +# - Each subcarrier: -10 dBm power, 64 GBaud symbol rate + +pluggable = client.ConfigurePluggable(request) +print(f"Configured {len(pluggable.config.dsc_groups[0].subcarriers)} subcarriers") +``` + +### 6. Deleting a Pluggable + +```python +from dscm.tests.testmessages import create_delete_pluggable_request + +# Delete pluggable from management +request = create_delete_pluggable_request( + device_uuid="550e8400-e29b-41d4-a716-446655440000", + pluggable_index=0 +) + +response = client.DeletePluggable(request) +print("Pluggable deleted successfully") +``` + +## Configuration Message Structure + +### Complete Configuration Example + +```python +from common.proto import dscm_pluggable_pb2 + +# Create configuration request +request = dscm_pluggable_pb2.ConfigurePluggableRequest() + +# Set pluggable ID +request.config.id.device.device_uuid.uuid = "550e8400-e29b-41d4-a716-446655440000" +request.config.id.pluggable_index = 0 + +# Add DSC group +dsc_group = request.config.dsc_groups.add() +dsc_group.id.pluggable.device.device_uuid.uuid = "550e8400-e29b-41d4-a716-446655440000" +dsc_group.id.pluggable.pluggable_index = 0 +dsc_group.id.group_index = 0 +dsc_group.group_size = 2 +dsc_group.group_capacity_gbps = 400.0 +dsc_group.subcarrier_spacing_mhz = 75.0 + +# Add subcarrier 1 +subcarrier = dsc_group.subcarriers.add() +subcarrier.id.group.pluggable.device.device_uuid.uuid = "550e8400-e29b-41d4-a716-446655440000" +subcarrier.id.group.pluggable.pluggable_index = 0 +subcarrier.id.group.group_index = 0 +subcarrier.id.subcarrier_index = 0 +subcarrier.active = True +subcarrier.target_output_power_dbm = -10.0 +subcarrier.center_frequency_hz = 193100000000000 # 193.1 THz +subcarrier.symbol_rate_baud = 64000000000 # 64 GBaud + +# Add subcarrier 2 +subcarrier2 = dsc_group.subcarriers.add() +# ... (similar configuration for second subcarrier) + +# Set view level and timeout +request.view_level = dscm_pluggable_pb2.VIEW_FULL +request.apply_timeout_seconds = 30 +``` + +## API Reference + +For complete API documentation, see: +- Protocol Buffer definitions: `/home/ubuntu/tfs-ctrl/proto/dscm_pluggable.proto` +- Client implementation: `/home/ubuntu/tfs-ctrl/src/dscm/client/DscmPluggableClient.py` +- Service implementation: `/home/ubuntu/tfs-ctrl/src/dscm/service/DscmPluggableServiceServicerImpl.py` +- Test examples: `/home/ubuntu/tfs-ctrl/src/dscm/tests/test_DscmPluggables.py` +- Message helpers: `/home/ubuntu/tfs-ctrl/src/dscm/tests/testmessages.py` diff --git a/src/dscm/__init__.py b/src/dscm/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/dscm/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/dscm/client/DscmPluggableClient.py b/src/dscm/client/DscmPluggableClient.py new file mode 100644 index 000000000..e261c6bb6 --- /dev/null +++ b/src/dscm/client/DscmPluggableClient.py @@ -0,0 +1,94 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from common.Constants import ServiceNameEnum +from common.Settings import get_service_host, get_service_port_grpc + +from common.proto.context_pb2 import Empty +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.client.RetryDecorator import retry, delay_exponential + +from common.proto.dscm_pluggable_pb2_grpc import DscmPluggableServiceStub +from common.proto.dscm_pluggable_pb2 import ( + PluggableId, Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, + GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest, DeconfigurePluggableRequest) + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 10 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') + +class DscmPluggableClient: + def __init__(self, host=None, port=None): + if not host: host = get_service_host(ServiceNameEnum.DSCMPLUGGABLE) + if not port: port = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) + self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) + LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) + + self.channel = None + self.stub = None + self.connect() + LOGGER.debug('Channel created') + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = DscmPluggableServiceStub(self.channel) + + def close(self): + if self.channel is not None: self.channel.close() + self.channel = None + self.stub = None + + @RETRY_DECORATOR + def CreatePluggable(self, request : CreatePluggableRequest) -> Pluggable: # pyright: ignore[reportInvalidTypeForm] + LOGGER.debug('CreatePluggable: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.CreatePluggable(request) + LOGGER.debug('CreatePluggable result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def ListPluggables(self, request : ListPluggablesRequest) -> ListPluggablesResponse: # pyright: ignore[reportInvalidTypeForm] + LOGGER.debug('ListPluggables: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.ListPluggables(request) + LOGGER.debug('ListPluggables result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def GetPluggable(self, request : GetPluggableRequest) -> Pluggable: # pyright: ignore[reportInvalidTypeForm] + LOGGER.debug('GetPluggable: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.GetPluggable(request) + LOGGER.debug('GetPluggable result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeletePluggable(self, request : DeletePluggableRequest) -> Empty: # pyright: ignore[reportInvalidTypeForm] + LOGGER.debug('DeletePluggable: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeletePluggable(request) + LOGGER.debug('DeletePluggable result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def ConfigurePluggable(self, request : ConfigurePluggableRequest) -> Pluggable: # pyright: ignore[reportInvalidTypeForm] + LOGGER.debug('ConfigurePluggable: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.ConfigurePluggable(request) + LOGGER.debug('ConfigurePluggable result: {:s}'.format(grpc_message_to_json_string(response))) + return response + + @RETRY_DECORATOR + def DeconfigurePluggable(self, request : DeconfigurePluggableRequest) -> Empty: # pyright: ignore[reportInvalidTypeForm] + LOGGER.debug('DeconfigurePluggable: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.DeconfigurePluggable(request) + LOGGER.debug('DeconfigurePluggable result: {:s}'.format(grpc_message_to_json_string(response))) + return response diff --git a/src/dscm/client/__init__.py b/src/dscm/client/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/dscm/client/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/dscm/requirements.in b/src/dscm/requirements.in new file mode 100644 index 000000000..e69de29bb diff --git a/src/dscm/service/DscmPluggableService.py b/src/dscm/service/DscmPluggableService.py new file mode 100644 index 000000000..aac2ecce1 --- /dev/null +++ b/src/dscm/service/DscmPluggableService.py @@ -0,0 +1,28 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.Constants import ServiceNameEnum +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.proto.dscm_pluggable_pb2_grpc import add_DscmPluggableServiceServicer_to_server +from dscm.service.DscmPluggableServiceServicerImpl import DscmPluggableServiceServicerImpl + +class DscmPluggableService(GenericGrpcService): + def __init__(self, cls_name: str = __name__) -> None: + port = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) + super().__init__(port, cls_name=cls_name) + self.dscmPluggableService_servicer = DscmPluggableServiceServicerImpl() + + def install_servicers(self): + add_DscmPluggableServiceServicer_to_server(self.dscmPluggableService_servicer, self.server) diff --git a/src/dscm/service/DscmPluggableServiceServicerImpl.py b/src/dscm/service/DscmPluggableServiceServicerImpl.py new file mode 100644 index 000000000..e6d80d512 --- /dev/null +++ b/src/dscm/service/DscmPluggableServiceServicerImpl.py @@ -0,0 +1,201 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, grpc +from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from common.proto.context_pb2 import Empty +from common.proto.dscm_pluggable_pb2_grpc import DscmPluggableServiceServicer +from common.proto.dscm_pluggable_pb2 import ( + Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, + GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest) +from common.method_wrappers.ServiceExceptions import NotFoundException, AlreadyExistsException + +LOGGER = logging.getLogger(__name__) +METRICS_POOL = MetricsPool('DscmPluggable', 'ServicegRPC') + +class DscmPluggableServiceServicerImpl(DscmPluggableServiceServicer): + def __init__(self): + LOGGER.info('Init DscmPluggableService') + self.pluggables = {} # In-memory store for pluggables + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def CreatePluggable( + self, request: CreatePluggableRequest, context: grpc.ServicerContext # type: ignore + ) -> Pluggable: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(request)) + + device_uuid = request.device.device_uuid.uuid + + if request.preferred_pluggable_index and request.preferred_pluggable_index >= 0: + pluggable_index = request.preferred_pluggable_index + else: + pluggable_index = -1 + + pluggable_key = f"{device_uuid}:{pluggable_index}" + + if pluggable_key in self.pluggables: + LOGGER.warning(f"Pluggable already exists at device {device_uuid} index {pluggable_index}") + raise AlreadyExistsException('Pluggable', pluggable_key) + + pluggable = Pluggable() + + pluggable.id.device.device_uuid.uuid = device_uuid + pluggable.id.pluggable_index = pluggable_index + + if request.HasField('initial_config'): + pluggable.config.CopyFrom(request.initial_config) + # The below ensure ID consistency across all levels are maintained + # User might send incorrect/inconsistent IDs in nested structures + pluggable.config.id.device.device_uuid.uuid = device_uuid + pluggable.config.id.pluggable_index = pluggable_index + + for dsc_group in pluggable.config.dsc_groups: + dsc_group.id.pluggable.device.device_uuid.uuid = device_uuid + dsc_group.id.pluggable.pluggable_index = pluggable_index + + for subcarrier in dsc_group.subcarriers: + subcarrier.id.group.pluggable.device.device_uuid.uuid = device_uuid + subcarrier.id.group.pluggable.pluggable_index = pluggable_index + + pluggable.state.id.device.device_uuid.uuid = device_uuid + pluggable.state.id.pluggable_index = pluggable_index + + self.pluggables[pluggable_key] = pluggable + + LOGGER.info(f"Created pluggable: device={device_uuid}, index={pluggable_index}") + return pluggable + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def GetPluggable( + self, request: GetPluggableRequest, context: grpc.ServicerContext # type: ignore + ) -> Pluggable: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(request)) + + device_uuid = request.id.device.device_uuid.uuid + pluggable_index = request.id.pluggable_index + pluggable_key = f"{device_uuid}:{pluggable_index}" + + if pluggable_key not in self.pluggables: + LOGGER.warning(f'No matching pluggable found: device={device_uuid}, index={pluggable_index}') + raise NotFoundException('Pluggable', pluggable_key) + + pluggable = self.pluggables[pluggable_key] + + if request.view_level == 1: + filtered = Pluggable() + filtered.id.CopyFrom(pluggable.id) + filtered.config.CopyFrom(pluggable.config) + return filtered + elif request.view_level == 2: + filtered = Pluggable() + filtered.id.CopyFrom(pluggable.id) + filtered.state.CopyFrom(pluggable.state) + return filtered + else: + return pluggable + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def DeletePluggable( + self, request: DeletePluggableRequest, context: grpc.ServicerContext # type: ignore + ) -> Empty: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(request)) + + device_uuid = request.id.device.device_uuid.uuid + pluggable_index = request.id.pluggable_index + pluggable_key = f"{device_uuid}:{pluggable_index}" + + if pluggable_key not in self.pluggables: + LOGGER.info(f'No matching pluggable found: device={device_uuid}, index={pluggable_index}') + raise NotFoundException('Pluggable', pluggable_key) + + del self.pluggables[pluggable_key] + LOGGER.info(f"Deleted pluggable: device={device_uuid}, index={pluggable_index}") + + return Empty() + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def ListPluggables( + self, request: ListPluggablesRequest, context: grpc.ServicerContext # type: ignore + ) -> ListPluggablesResponse: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(request)) + + response = ListPluggablesResponse() + device_uuid = request.device.device_uuid.uuid if request.HasField('device') else None + + for pluggable in self.pluggables.values(): + if device_uuid and pluggable.id.device.device_uuid.uuid != device_uuid: + continue + + if request.view_level == 1: + filtered = Pluggable() + filtered.id.CopyFrom(pluggable.id) + filtered.config.CopyFrom(pluggable.config) + response.pluggables.append(filtered) + elif request.view_level == 2: + filtered = Pluggable() + filtered.id.CopyFrom(pluggable.id) + filtered.state.CopyFrom(pluggable.state) + response.pluggables.append(filtered) + else: + response.pluggables.append(pluggable) + + LOGGER.info(f"Returning {len(response.pluggables)} pluggable(s)") + return response + + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def ConfigurePluggable( + self, request: ConfigurePluggableRequest, context: grpc.ServicerContext # type: ignore + ) -> Pluggable: # type: ignore + LOGGER.info("Received gRPC message object: {:}".format(request)) + + device_uuid = request.config.id.device.device_uuid.uuid + pluggable_index = request.config.id.pluggable_index + pluggable_key = f"{device_uuid}:{pluggable_index}" + + if pluggable_key not in self.pluggables: + LOGGER.info(f'No matching pluggable found: device={device_uuid}, index={pluggable_index}') + raise NotFoundException('Pluggable', pluggable_key) + + pluggable = self.pluggables[pluggable_key] + + LOGGER.info(f"Applying full configuration to pluggable: device={device_uuid}, index={pluggable_index}") + pluggable.config.CopyFrom(request.config) + + # To ensure ID consistency across all levels are maintained + # User might send incorrect/inconsistent IDs in nested structures + pluggable.config.id.device.device_uuid.uuid = device_uuid + pluggable.config.id.pluggable_index = pluggable_index + for dsc_group in pluggable.config.dsc_groups: + dsc_group.id.pluggable.device.device_uuid.uuid = device_uuid + dsc_group.id.pluggable.pluggable_index = pluggable_index + for subcarrier in dsc_group.subcarriers: + subcarrier.id.group.pluggable.device.device_uuid.uuid = device_uuid + subcarrier.id.group.pluggable.pluggable_index = pluggable_index + + has_config = len(pluggable.config.dsc_groups) > 0 + state_msg = "configured" if has_config else "deconfigured (empty config)" + LOGGER.info(f"Successfully {state_msg} pluggable: device={device_uuid}, index={pluggable_index}") + + if request.view_level == 1: + filtered = Pluggable() + filtered.id.CopyFrom(pluggable.id) + filtered.config.CopyFrom(pluggable.config) + return filtered + elif request.view_level == 2: + filtered = Pluggable() + filtered.id.CopyFrom(pluggable.id) + filtered.state.CopyFrom(pluggable.state) + return filtered + else: + return pluggable diff --git a/src/dscm/service/__init__.py b/src/dscm/service/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/dscm/service/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/dscm/service/__main__.py b/src/dscm/service/__main__.py new file mode 100644 index 000000000..f21f35fe6 --- /dev/null +++ b/src/dscm/service/__main__.py @@ -0,0 +1,51 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, signal, sys, threading +from common.Settings import get_log_level +from .DscmPluggableService import DscmPluggableService + +terminate = threading.Event() +LOGGER = None + +def signal_handler(signal, frame): + LOGGER.warning('Terminate signal received') + terminate.set() + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.debug('Starting...') + + grpc_service = DscmPluggableService() + grpc_service.start() + + # Wait for Ctrl+C or termination signal + while not terminate.wait(timeout=1.0): pass + + LOGGER.debug('Terminating...') + grpc_service.stop() + + LOGGER.debug('Bye') + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/dscm/tests/__init__.py b/src/dscm/tests/__init__.py new file mode 100644 index 000000000..3ccc21c7d --- /dev/null +++ b/src/dscm/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/dscm/tests/test_DscmPluggables.py b/src/dscm/tests/test_DscmPluggables.py new file mode 100644 index 000000000..876bccb8e --- /dev/null +++ b/src/dscm/tests/test_DscmPluggables.py @@ -0,0 +1,236 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import grpc +import os, pytest +import logging +from typing import Union + +from common.proto.context_pb2 import Empty +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, + get_env_var_name, get_service_port_grpc) +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server + +from common.proto.dscm_pluggable_pb2 import (PluggableId, Pluggable, + CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, View) +from common.tools.service.GenericGrpcService import GenericGrpcService + +from dscm.client.DscmPluggableClient import DscmPluggableClient +from dscm.service.DscmPluggableService import DscmPluggableService +from dscm.tests.testmessages import (create_pluggable_request, + create_list_pluggables_request, create_get_pluggable_request, + create_delete_pluggable_request, create_configure_pluggable_request) + + +########################### +# Tests Setup +########################### + +LOCAL_HOST = '127.0.0.1' + +DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) # type: ignore +os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) + +LOGGER = logging.getLogger(__name__) + +class MockContextService(GenericGrpcService): + # Mock Service implementing Context to simplify unitary tests of DSCM pluggables + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockService') + + # pylint: disable=attribute-defined-outside-init + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +# This fixture will be requested by test cases and last during testing session +@pytest.fixture(scope='session') +def dscm_pluggable_service(): + LOGGER.info('Initializing DscmPluggableService...') + _service = DscmPluggableService() + _service.start() + + # yield the server, when test finishes, execution will resume to stop it + LOGGER.info('Yielding DscmPluggableService...') + yield _service + + LOGGER.info('Terminating DscmPluggableService...') + _service.stop() + + LOGGER.info('Terminated DscmPluggableService...') + +@pytest.fixture(scope='function') +def dscm_pluggable_client(dscm_pluggable_service : DscmPluggableService): + LOGGER.info('Creating DscmPluggableClient...') + _client = DscmPluggableClient() + + LOGGER.info('Yielding DscmPluggableClient...') + yield _client + + LOGGER.info('Closing DscmPluggableClient...') + _client.close() + + LOGGER.info('Closed DscmPluggableClient...') + +@pytest.fixture(autouse=True) +def log_all_methods(request): + ''' + This fixture logs messages before and after each test function runs, indicating the start and end of the test. + The autouse=True parameter ensures that this logging happens automatically for all tests in the module. + ''' + LOGGER.info(f" >>>>> Starting test: {request.node.name} ") + yield + LOGGER.info(f" <<<<< Finished test: {request.node.name} ") + +########################### +# Test Cases +########################### + +# CreatePluggable Test without configuration +def test_CreatePluggable(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Creating Pluggable for test...') + _pluggable_request = create_pluggable_request(preferred_pluggable_index=-1) + _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + LOGGER.info('Created Pluggable for test: %s', _pluggable) + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.pluggable_index == _pluggable_request.preferred_pluggable_index + assert _pluggable.id.device.device_uuid.uuid == _pluggable_request.device.device_uuid.uuid + + +# CreatePluggable Test with configuration +def test_CreatePluggable_with_config(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Creating Pluggable with initial configuration for test...') + _pluggable_request = create_pluggable_request( + device_uuid = "9bbf1937-db9e-45bc-b2c6-3214a9d42157", + preferred_pluggable_index = -1, + with_initial_config = True) + _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + LOGGER.info('Created Pluggable with initial configuration for test: %s', _pluggable) + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.pluggable_index == _pluggable_request.preferred_pluggable_index + assert _pluggable.id.device.device_uuid.uuid == _pluggable_request.device.device_uuid.uuid + assert _pluggable.config is not None + assert len(_pluggable.config.dsc_groups) == 1 + dsc_group = _pluggable.config.dsc_groups[0] + assert dsc_group.group_size == 4 + assert len(dsc_group.subcarriers) == 2 + +# create pluggable request with pluggable key already exists error +def test_CreatePluggable_already_exists(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Creating Pluggable for test...') + _pluggable_request = create_pluggable_request(preferred_pluggable_index=5) + _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + LOGGER.info('Created Pluggable for test: %s', _pluggable) + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.pluggable_index == _pluggable_request.preferred_pluggable_index + assert _pluggable.id.device.device_uuid.uuid == _pluggable_request.device.device_uuid.uuid + # Try to create the same pluggable again, should raise ALREADY_EXISTS + with pytest.raises(grpc.RpcError) as e: + dscm_pluggable_client.CreatePluggable(_pluggable_request) + assert e.value.code() == grpc.StatusCode.ALREADY_EXISTS + +# ListPluggables Test +def test_ListPluggables(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Listing Pluggables for test...') + _list_request = create_list_pluggables_request( + view_level = View.VIEW_CONFIG # View.VIEW_STATE + ) + _pluggables = dscm_pluggable_client.ListPluggables(_list_request) + LOGGER.info('Listed Pluggables for test: %s', _pluggables) + assert isinstance(_pluggables, ListPluggablesResponse) + if len(_pluggables.pluggables) != 0: + assert len(_pluggables.pluggables) >= 1 + for p in _pluggables.pluggables: + assert isinstance(p, Pluggable) + assert isinstance(p.id, PluggableId) + else: + assert len(_pluggables.pluggables) == 0 + +# GetPluggable Test +def test_GetPluggable(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Starting GetPluggable test...') + LOGGER.info('Getting Pluggable for test...') + # First create a pluggable to get it later + _pluggable_request = create_pluggable_request(preferred_pluggable_index=1) + _created_pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + LOGGER.info('Created Pluggable for GetPluggable test: %s', _created_pluggable) + + _get_request = create_get_pluggable_request( + device_uuid = _created_pluggable.id.device.device_uuid.uuid, + pluggable_index = _created_pluggable.id.pluggable_index, + view_level = View.VIEW_FULL + ) + _pluggable = dscm_pluggable_client.GetPluggable(_get_request) + LOGGER.info('Got Pluggable for test: %s', _pluggable) + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.pluggable_index == _created_pluggable.id.pluggable_index + assert _pluggable.id.device.device_uuid.uuid == _created_pluggable.id.device.device_uuid.uuid + + +# DeletePluggable Test +def test_DeletePluggable(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Starting DeletePluggable test...') + LOGGER.info('Creating Pluggable to delete for test...') + + # First create a pluggable to delete it later + _pluggable_request = create_pluggable_request(preferred_pluggable_index=2) + _created_pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + LOGGER.info('Created Pluggable to delete for test: %s', _created_pluggable) + + _delete_request = create_delete_pluggable_request( + device_uuid = _created_pluggable.id.device.device_uuid.uuid, + pluggable_index = _created_pluggable.id.pluggable_index + ) + _response = dscm_pluggable_client.DeletePluggable(_delete_request) + LOGGER.info('Deleted Pluggable for test, response: %s', _response) + assert isinstance(_response, Empty) + + # Try to get the deleted pluggable, should raise NOT_FOUND + with pytest.raises(grpc.RpcError) as e: + dscm_pluggable_client.GetPluggable( + create_get_pluggable_request( + device_uuid = _created_pluggable.id.device.device_uuid.uuid, + pluggable_index = _created_pluggable.id.pluggable_index + ) + ) + assert e.value.code() == grpc.StatusCode.NOT_FOUND + +# ConfigurePluggable Test +def test_ConfigurePluggable(dscm_pluggable_client : DscmPluggableClient): + LOGGER.info('Starting ConfigurePluggable test...') + LOGGER.info('Creating Pluggable to configure for test...') + + # First create a pluggable to configure it later + _pluggable_request = create_pluggable_request(preferred_pluggable_index=3) + _created_pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + LOGGER.info('Created Pluggable to configure for test: %s', _created_pluggable) + + _configure_request = create_configure_pluggable_request( + device_uuid = _created_pluggable.id.device.device_uuid.uuid, + pluggable_index = _created_pluggable.id.pluggable_index, + ) + _pluggable = dscm_pluggable_client.ConfigurePluggable(_configure_request) + LOGGER.info('Configured Pluggable for test: %s', _pluggable) + assert isinstance(_pluggable, Pluggable) + assert _pluggable.config is not None + assert len(_pluggable.config.dsc_groups) == 1 + dsc_group = _pluggable.config.dsc_groups[0] + assert dsc_group.group_size == 2 + assert len(dsc_group.subcarriers) == 2 diff --git a/src/dscm/tests/testmessages.py b/src/dscm/tests/testmessages.py new file mode 100644 index 000000000..bf6858854 --- /dev/null +++ b/src/dscm/tests/testmessages.py @@ -0,0 +1,230 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid +from typing import Optional, List, Dict, Any +from common.proto import dscm_pluggable_pb2 + + +########################### +# CreatePluggableRequest +########################### + +def create_pluggable_request( + device_uuid: Optional[str] = None, + preferred_pluggable_index: Optional[int] = None, + with_initial_config: bool = False +) -> dscm_pluggable_pb2.CreatePluggableRequest: + """ + Create a CreatePluggableRequest message. + + Args: + device_uuid: UUID of the device. If None, generates a random UUID. + preferred_pluggable_index: Preferred index for the pluggable. If None, not set. + with_initial_config: If True, includes initial configuration. + + Returns: + CreatePluggableRequest message + """ + _request = dscm_pluggable_pb2.CreatePluggableRequest() + + # Set device ID + if device_uuid is None: + device_uuid = str(uuid.uuid4()) + _request.device.device_uuid.uuid = device_uuid + + # Set preferred pluggable index if provided + if preferred_pluggable_index is not None: + _request.preferred_pluggable_index = preferred_pluggable_index + + # Add initial configuration if requested + if with_initial_config: + _request.initial_config.id.device.device_uuid.uuid = device_uuid + _request.initial_config.id.pluggable_index = preferred_pluggable_index or 0 + + # Add sample DSC group configuration + dsc_group = _request.initial_config.dsc_groups.add() + dsc_group.id.pluggable.device.device_uuid.uuid = device_uuid + dsc_group.id.pluggable.pluggable_index = preferred_pluggable_index or 0 + dsc_group.id.group_index = 0 + dsc_group.group_size = 4 + dsc_group.group_capacity_gbps = 400.0 + dsc_group.subcarrier_spacing_mhz = 75.0 + + # Add sample subcarrier configurations + for i in range(2): + subcarrier = dsc_group.subcarriers.add() + subcarrier.id.group.pluggable.device.device_uuid.uuid = device_uuid + subcarrier.id.group.pluggable.pluggable_index = preferred_pluggable_index or 0 + subcarrier.id.group.group_index = 0 + subcarrier.id.subcarrier_index = i + subcarrier.active = True + subcarrier.target_output_power_dbm = -10.0 + subcarrier.center_frequency_hz = 193100000000000 + (i * 75000000) # 193.1 THz + spacing + subcarrier.symbol_rate_baud = 64000000000 # 64 GBaud + + return _request + + +########################### +# ListPluggablesRequest +########################### + +def create_list_pluggables_request( + device_uuid: Optional[str] = None, + view_level: dscm_pluggable_pb2.View = dscm_pluggable_pb2.VIEW_FULL +) -> dscm_pluggable_pb2.ListPluggablesRequest: + """ + Create a ListPluggablesRequest message. + + Args: + device_uuid: UUID of the device to filter by. If None, generates a random UUID. + view_level: View level (VIEW_CONFIG, VIEW_STATE, VIEW_FULL, VIEW_UNSPECIFIED) + + Returns: + ListPluggablesRequest message + """ + _request = dscm_pluggable_pb2.ListPluggablesRequest() + + if device_uuid is None: + device_uuid = "9bbf1937-db9e-45bc-b2c6-3214a9d42157" + # device_uuid = str(uuid.uuid4()) + _request.device.device_uuid.uuid = device_uuid + _request.view_level = view_level + + return _request + + +########################### +# GetPluggableRequest +########################### + +def create_get_pluggable_request( + device_uuid: str, + pluggable_index: int, + view_level: dscm_pluggable_pb2.View = dscm_pluggable_pb2.VIEW_FULL +) -> dscm_pluggable_pb2.GetPluggableRequest: + """ + Create a GetPluggableRequest message. + + Args: + device_uuid: UUID of the device + pluggable_index: Index of the pluggable + view_level: View level (VIEW_CONFIG, VIEW_STATE, VIEW_FULL, VIEW_UNSPECIFIED) + + Returns: + GetPluggableRequest message + """ + _request = dscm_pluggable_pb2.GetPluggableRequest() + _request.id.device.device_uuid.uuid = device_uuid + _request.id.pluggable_index = pluggable_index + _request.view_level = view_level + return _request + + +########################### +# DeletePluggableRequest +########################### + +def create_delete_pluggable_request( + device_uuid: str, + pluggable_index: int +) -> dscm_pluggable_pb2.DeletePluggableRequest: + """ + Create a DeletePluggableRequest message. + + Args: + device_uuid: UUID of the device + pluggable_index: Index of the pluggable + + Returns: + DeletePluggableRequest message + """ + _request = dscm_pluggable_pb2.DeletePluggableRequest() + _request.id.device.device_uuid.uuid = device_uuid + _request.id.pluggable_index = pluggable_index + + return _request + + +########################### +# ConfigurePluggableRequest +########################### + +def create_configure_pluggable_request( + device_uuid: str, + pluggable_index: int, + update_mask_paths: Optional[list] = None, + view_level: dscm_pluggable_pb2.View = dscm_pluggable_pb2.VIEW_FULL, # pyright: ignore[reportInvalidTypeForm] + apply_timeout_seconds: int = 30, + parameters: Optional[List[Dict[str, Any]]] = [], +) -> dscm_pluggable_pb2.ConfigurePluggableRequest: # pyright: ignore[reportInvalidTypeForm] + """ + Create a ConfigurePluggableRequest message. + + Args: + device_uuid: UUID of the device + pluggable_index: Index of the pluggable + update_mask_paths: List of field paths to update. If None, updates all fields. + view_level: View level for response + apply_timeout_seconds: Timeout in seconds for applying configuration + + Returns: + ConfigurePluggableRequest message + """ + _request = dscm_pluggable_pb2.ConfigurePluggableRequest() + + # Set pluggable configuration + _request.config.id.device.device_uuid.uuid = device_uuid + _request.config.id.pluggable_index = pluggable_index + + # Add DSC group configuration + group_1 = _request.config.dsc_groups.add() + group_1.id.pluggable.device.device_uuid.uuid = device_uuid + group_1.id.pluggable.pluggable_index = pluggable_index + group_1.id.group_index = 0 + group_1.group_size = 2 + group_1.group_capacity_gbps = 400.0 + group_1.subcarrier_spacing_mhz = 75.0 + + # Add digital-subcarrier configuration (to group group_1) + subcarrier_1 = group_1.subcarriers.add() + subcarrier_1.id.group.pluggable.device.device_uuid.uuid = device_uuid + subcarrier_1.id.group.pluggable.pluggable_index = pluggable_index + subcarrier_1.id.group.group_index = 0 + subcarrier_1.id.subcarrier_index = 0 + subcarrier_1.active = True + subcarrier_1.target_output_power_dbm = -10.0 + subcarrier_1.center_frequency_hz = 193100000000000 # 193.1 THz + subcarrier_1.symbol_rate_baud = 64000000000 # 64 GBaud + + # Add another digital-subcarrier configuration (to group group_1) + subcarrier_2 = group_1.subcarriers.add() + subcarrier_2.id.group.pluggable.device.device_uuid.uuid = device_uuid + subcarrier_2.id.group.pluggable.pluggable_index = pluggable_index + subcarrier_2.id.group.group_index = 0 + subcarrier_2.id.subcarrier_index = 1 + subcarrier_2.active = True + subcarrier_2.target_output_power_dbm = -10.0 + subcarrier_2.center_frequency_hz = 193100075000000 # 193.175 THz + subcarrier_2.symbol_rate_baud = 64000000000 # 64 GBaud + + # Set update mask if provided + if update_mask_paths: + _request.update_mask.paths.extend(update_mask_paths) + + _request.view_level = view_level + _request.apply_timeout_seconds = apply_timeout_seconds + + return _request -- GitLab From a7c55b2ae47d578eb4c0030b7924e074e1d295df Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Mon, 27 Oct 2025 10:07:01 +0000 Subject: [PATCH 02/15] Remove "DSCM" for component and proto file. --- ...{dscm_pluggable.proto => pluggables.proto} | 4 +- .../run_tests_locally-service-pluggable.sh | 4 +- src/{dscm => pluggables}/.gitlab-ci.yml | 0 src/{dscm => pluggables}/Dockerfile | 0 src/{dscm => pluggables}/README.md | 0 src/{dscm => pluggables}/__init__.py | 0 .../client/PluggablesClient.py} | 19 +++------- src/{dscm => pluggables}/client/__init__.py | 0 src/{dscm => pluggables}/requirements.in | 0 .../service/PluggablesService.py} | 10 ++--- .../service/PluggablesServiceServicerImpl.py} | 8 ++-- src/{dscm => pluggables}/service/__init__.py | 0 src/{dscm => pluggables}/service/__main__.py | 4 +- src/{dscm => pluggables}/tests/__init__.py | 0 .../tests/test_Pluggables.py} | 38 +++++++++---------- .../tests/testmessages.py | 29 +++++++------- 16 files changed, 54 insertions(+), 62 deletions(-) rename proto/{dscm_pluggable.proto => pluggables.proto} (98%) rename src/{dscm => pluggables}/.gitlab-ci.yml (100%) rename src/{dscm => pluggables}/Dockerfile (100%) rename src/{dscm => pluggables}/README.md (100%) rename src/{dscm => pluggables}/__init__.py (100%) rename src/{dscm/client/DscmPluggableClient.py => pluggables/client/PluggablesClient.py} (83%) rename src/{dscm => pluggables}/client/__init__.py (100%) rename src/{dscm => pluggables}/requirements.in (100%) rename src/{dscm/service/DscmPluggableService.py => pluggables/service/PluggablesService.py} (70%) rename src/{dscm/service/DscmPluggableServiceServicerImpl.py => pluggables/service/PluggablesServiceServicerImpl.py} (97%) rename src/{dscm => pluggables}/service/__init__.py (100%) rename src/{dscm => pluggables}/service/__main__.py (93%) rename src/{dscm => pluggables}/tests/__init__.py (100%) rename src/{dscm/tests/test_DscmPluggables.py => pluggables/tests/test_Pluggables.py} (90%) rename src/{dscm => pluggables}/tests/testmessages.py (87%) diff --git a/proto/dscm_pluggable.proto b/proto/pluggables.proto similarity index 98% rename from proto/dscm_pluggable.proto rename to proto/pluggables.proto index 95ec83360..96661c02a 100644 --- a/proto/dscm_pluggable.proto +++ b/proto/pluggables.proto @@ -1,11 +1,11 @@ syntax = "proto3"; -package tfs.dscm.v0; +package tfs.pluggables.v0; import "context.proto"; import "google/protobuf/field_mask.proto"; -service DscmPluggableService { +service PluggablesService { rpc CreatePluggable (CreatePluggableRequest) returns (Pluggable) {} rpc ListPluggables (ListPluggablesRequest) returns (ListPluggablesResponse) {} rpc GetPluggable (GetPluggableRequest) returns (Pluggable) {} diff --git a/scripts/run_tests_locally-service-pluggable.sh b/scripts/run_tests_locally-service-pluggable.sh index be977b5a6..600c1edf7 100755 --- a/scripts/run_tests_locally-service-pluggable.sh +++ b/scripts/run_tests_locally-service-pluggable.sh @@ -15,9 +15,9 @@ PROJECTDIR=`pwd` cd $PROJECTDIR/src -# RCFILE=$PROJECTDIR/coverage/.coveragerc +RCFILE=$PROJECTDIR/coverage/.coveragerc python3 -m pytest --log-level=info --log-cli-level=info --verbose \ - dscm/tests/test_DscmPluggables.py + pluggables/tests/test_Pluggables.py echo "Bye!" diff --git a/src/dscm/.gitlab-ci.yml b/src/pluggables/.gitlab-ci.yml similarity index 100% rename from src/dscm/.gitlab-ci.yml rename to src/pluggables/.gitlab-ci.yml diff --git a/src/dscm/Dockerfile b/src/pluggables/Dockerfile similarity index 100% rename from src/dscm/Dockerfile rename to src/pluggables/Dockerfile diff --git a/src/dscm/README.md b/src/pluggables/README.md similarity index 100% rename from src/dscm/README.md rename to src/pluggables/README.md diff --git a/src/dscm/__init__.py b/src/pluggables/__init__.py similarity index 100% rename from src/dscm/__init__.py rename to src/pluggables/__init__.py diff --git a/src/dscm/client/DscmPluggableClient.py b/src/pluggables/client/PluggablesClient.py similarity index 83% rename from src/dscm/client/DscmPluggableClient.py rename to src/pluggables/client/PluggablesClient.py index e261c6bb6..f8e7deadb 100644 --- a/src/dscm/client/DscmPluggableClient.py +++ b/src/pluggables/client/PluggablesClient.py @@ -20,17 +20,17 @@ from common.proto.context_pb2 import Empty from common.tools.grpc.Tools import grpc_message_to_json_string from common.tools.client.RetryDecorator import retry, delay_exponential -from common.proto.dscm_pluggable_pb2_grpc import DscmPluggableServiceStub -from common.proto.dscm_pluggable_pb2 import ( - PluggableId, Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, - GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest, DeconfigurePluggableRequest) +from common.proto.pluggables_pb2_grpc import PluggablesServiceStub +from common.proto.pluggables_pb2 import ( + Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, + GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest) LOGGER = logging.getLogger(__name__) MAX_RETRIES = 10 DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') -class DscmPluggableClient: +class PluggablesClient: def __init__(self, host=None, port=None): if not host: host = get_service_host(ServiceNameEnum.DSCMPLUGGABLE) if not port: port = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) @@ -44,7 +44,7 @@ class DscmPluggableClient: def connect(self): self.channel = grpc.insecure_channel(self.endpoint) - self.stub = DscmPluggableServiceStub(self.channel) + self.stub = PluggablesServiceStub(self.channel) def close(self): if self.channel is not None: self.channel.close() @@ -85,10 +85,3 @@ class DscmPluggableClient: response = self.stub.ConfigurePluggable(request) LOGGER.debug('ConfigurePluggable result: {:s}'.format(grpc_message_to_json_string(response))) return response - - @RETRY_DECORATOR - def DeconfigurePluggable(self, request : DeconfigurePluggableRequest) -> Empty: # pyright: ignore[reportInvalidTypeForm] - LOGGER.debug('DeconfigurePluggable: {:s}'.format(grpc_message_to_json_string(request))) - response = self.stub.DeconfigurePluggable(request) - LOGGER.debug('DeconfigurePluggable result: {:s}'.format(grpc_message_to_json_string(response))) - return response diff --git a/src/dscm/client/__init__.py b/src/pluggables/client/__init__.py similarity index 100% rename from src/dscm/client/__init__.py rename to src/pluggables/client/__init__.py diff --git a/src/dscm/requirements.in b/src/pluggables/requirements.in similarity index 100% rename from src/dscm/requirements.in rename to src/pluggables/requirements.in diff --git a/src/dscm/service/DscmPluggableService.py b/src/pluggables/service/PluggablesService.py similarity index 70% rename from src/dscm/service/DscmPluggableService.py rename to src/pluggables/service/PluggablesService.py index aac2ecce1..92e9aa6e1 100644 --- a/src/dscm/service/DscmPluggableService.py +++ b/src/pluggables/service/PluggablesService.py @@ -15,14 +15,14 @@ from common.Constants import ServiceNameEnum from common.Settings import get_service_port_grpc from common.tools.service.GenericGrpcService import GenericGrpcService -from common.proto.dscm_pluggable_pb2_grpc import add_DscmPluggableServiceServicer_to_server -from dscm.service.DscmPluggableServiceServicerImpl import DscmPluggableServiceServicerImpl +from common.proto.pluggables_pb2_grpc import add_PluggablesServiceServicer_to_server +from pluggables.service.PluggablesServiceServicerImpl import PluggablesServiceServicerImpl -class DscmPluggableService(GenericGrpcService): +class PluggablesService(GenericGrpcService): def __init__(self, cls_name: str = __name__) -> None: port = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) super().__init__(port, cls_name=cls_name) - self.dscmPluggableService_servicer = DscmPluggableServiceServicerImpl() + self.dscmPluggableService_servicer = PluggablesServiceServicerImpl() def install_servicers(self): - add_DscmPluggableServiceServicer_to_server(self.dscmPluggableService_servicer, self.server) + add_PluggablesServiceServicer_to_server(self.dscmPluggableService_servicer, self.server) diff --git a/src/dscm/service/DscmPluggableServiceServicerImpl.py b/src/pluggables/service/PluggablesServiceServicerImpl.py similarity index 97% rename from src/dscm/service/DscmPluggableServiceServicerImpl.py rename to src/pluggables/service/PluggablesServiceServicerImpl.py index e6d80d512..a12be3ca4 100644 --- a/src/dscm/service/DscmPluggableServiceServicerImpl.py +++ b/src/pluggables/service/PluggablesServiceServicerImpl.py @@ -15,8 +15,8 @@ import logging, grpc from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.proto.context_pb2 import Empty -from common.proto.dscm_pluggable_pb2_grpc import DscmPluggableServiceServicer -from common.proto.dscm_pluggable_pb2 import ( +from common.proto.pluggables_pb2_grpc import PluggablesServiceServicer +from common.proto.pluggables_pb2 import ( Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest) from common.method_wrappers.ServiceExceptions import NotFoundException, AlreadyExistsException @@ -24,7 +24,7 @@ from common.method_wrappers.ServiceExceptions import NotFoundException, AlreadyE LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('DscmPluggable', 'ServicegRPC') -class DscmPluggableServiceServicerImpl(DscmPluggableServiceServicer): +class PluggablesServiceServicerImpl(PluggablesServiceServicer): def __init__(self): LOGGER.info('Init DscmPluggableService') self.pluggables = {} # In-memory store for pluggables @@ -55,7 +55,7 @@ class DscmPluggableServiceServicerImpl(DscmPluggableServiceServicer): if request.HasField('initial_config'): pluggable.config.CopyFrom(request.initial_config) - # The below ensure ID consistency across all levels are maintained + # The below code ensures ID consistency across all levels are maintained # User might send incorrect/inconsistent IDs in nested structures pluggable.config.id.device.device_uuid.uuid = device_uuid pluggable.config.id.pluggable_index = pluggable_index diff --git a/src/dscm/service/__init__.py b/src/pluggables/service/__init__.py similarity index 100% rename from src/dscm/service/__init__.py rename to src/pluggables/service/__init__.py diff --git a/src/dscm/service/__main__.py b/src/pluggables/service/__main__.py similarity index 93% rename from src/dscm/service/__main__.py rename to src/pluggables/service/__main__.py index f21f35fe6..fc99a41cb 100644 --- a/src/dscm/service/__main__.py +++ b/src/pluggables/service/__main__.py @@ -14,7 +14,7 @@ import logging, signal, sys, threading from common.Settings import get_log_level -from .DscmPluggableService import DscmPluggableService +from .PluggablesService import PluggablesService terminate = threading.Event() LOGGER = None @@ -35,7 +35,7 @@ def main(): LOGGER.debug('Starting...') - grpc_service = DscmPluggableService() + grpc_service = PluggablesService() grpc_service.start() # Wait for Ctrl+C or termination signal diff --git a/src/dscm/tests/__init__.py b/src/pluggables/tests/__init__.py similarity index 100% rename from src/dscm/tests/__init__.py rename to src/pluggables/tests/__init__.py diff --git a/src/dscm/tests/test_DscmPluggables.py b/src/pluggables/tests/test_Pluggables.py similarity index 90% rename from src/dscm/tests/test_DscmPluggables.py rename to src/pluggables/tests/test_Pluggables.py index 876bccb8e..5e12ad1a3 100644 --- a/src/dscm/tests/test_DscmPluggables.py +++ b/src/pluggables/tests/test_Pluggables.py @@ -26,13 +26,13 @@ from common.Settings import ( from common.tests.MockServicerImpl_Context import MockServicerImpl_Context from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server -from common.proto.dscm_pluggable_pb2 import (PluggableId, Pluggable, +from common.proto.pluggables_pb2 import (PluggableId, Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, View) from common.tools.service.GenericGrpcService import GenericGrpcService -from dscm.client.DscmPluggableClient import DscmPluggableClient -from dscm.service.DscmPluggableService import DscmPluggableService -from dscm.tests.testmessages import (create_pluggable_request, +from pluggables.client.PluggablesClient import PluggablesClient +from pluggables.service.PluggablesService import PluggablesService +from pluggables.tests.testmessages import (create_pluggable_request, create_list_pluggables_request, create_get_pluggable_request, create_delete_pluggable_request, create_configure_pluggable_request) @@ -62,9 +62,9 @@ class MockContextService(GenericGrpcService): # This fixture will be requested by test cases and last during testing session @pytest.fixture(scope='session') -def dscm_pluggable_service(): +def pluggables_service(): LOGGER.info('Initializing DscmPluggableService...') - _service = DscmPluggableService() + _service = PluggablesService() _service.start() # yield the server, when test finishes, execution will resume to stop it @@ -77,17 +77,17 @@ def dscm_pluggable_service(): LOGGER.info('Terminated DscmPluggableService...') @pytest.fixture(scope='function') -def dscm_pluggable_client(dscm_pluggable_service : DscmPluggableService): - LOGGER.info('Creating DscmPluggableClient...') - _client = DscmPluggableClient() +def dscm_pluggable_client(pluggables_service : PluggablesService): + LOGGER.info('Creating PluggablesClient...') + _client = PluggablesClient() - LOGGER.info('Yielding DscmPluggableClient...') + LOGGER.info('Yielding PluggablesClient...') yield _client - LOGGER.info('Closing DscmPluggableClient...') + LOGGER.info('Closing PluggablesClient...') _client.close() - LOGGER.info('Closed DscmPluggableClient...') + LOGGER.info('Closed PluggablesClient...') @pytest.fixture(autouse=True) def log_all_methods(request): @@ -104,7 +104,7 @@ def log_all_methods(request): ########################### # CreatePluggable Test without configuration -def test_CreatePluggable(dscm_pluggable_client : DscmPluggableClient): +def test_CreatePluggable(dscm_pluggable_client : PluggablesClient): LOGGER.info('Creating Pluggable for test...') _pluggable_request = create_pluggable_request(preferred_pluggable_index=-1) _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) @@ -115,7 +115,7 @@ def test_CreatePluggable(dscm_pluggable_client : DscmPluggableClient): # CreatePluggable Test with configuration -def test_CreatePluggable_with_config(dscm_pluggable_client : DscmPluggableClient): +def test_CreatePluggable_with_config(dscm_pluggable_client : PluggablesClient): LOGGER.info('Creating Pluggable with initial configuration for test...') _pluggable_request = create_pluggable_request( device_uuid = "9bbf1937-db9e-45bc-b2c6-3214a9d42157", @@ -133,7 +133,7 @@ def test_CreatePluggable_with_config(dscm_pluggable_client : DscmPluggableClient assert len(dsc_group.subcarriers) == 2 # create pluggable request with pluggable key already exists error -def test_CreatePluggable_already_exists(dscm_pluggable_client : DscmPluggableClient): +def test_CreatePluggable_already_exists(dscm_pluggable_client : PluggablesClient): LOGGER.info('Creating Pluggable for test...') _pluggable_request = create_pluggable_request(preferred_pluggable_index=5) _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) @@ -147,7 +147,7 @@ def test_CreatePluggable_already_exists(dscm_pluggable_client : DscmPluggableCli assert e.value.code() == grpc.StatusCode.ALREADY_EXISTS # ListPluggables Test -def test_ListPluggables(dscm_pluggable_client : DscmPluggableClient): +def test_ListPluggables(dscm_pluggable_client : PluggablesClient): LOGGER.info('Listing Pluggables for test...') _list_request = create_list_pluggables_request( view_level = View.VIEW_CONFIG # View.VIEW_STATE @@ -164,7 +164,7 @@ def test_ListPluggables(dscm_pluggable_client : DscmPluggableClient): assert len(_pluggables.pluggables) == 0 # GetPluggable Test -def test_GetPluggable(dscm_pluggable_client : DscmPluggableClient): +def test_GetPluggable(dscm_pluggable_client : PluggablesClient): LOGGER.info('Starting GetPluggable test...') LOGGER.info('Getting Pluggable for test...') # First create a pluggable to get it later @@ -185,7 +185,7 @@ def test_GetPluggable(dscm_pluggable_client : DscmPluggableClient): # DeletePluggable Test -def test_DeletePluggable(dscm_pluggable_client : DscmPluggableClient): +def test_DeletePluggable(dscm_pluggable_client : PluggablesClient): LOGGER.info('Starting DeletePluggable test...') LOGGER.info('Creating Pluggable to delete for test...') @@ -213,7 +213,7 @@ def test_DeletePluggable(dscm_pluggable_client : DscmPluggableClient): assert e.value.code() == grpc.StatusCode.NOT_FOUND # ConfigurePluggable Test -def test_ConfigurePluggable(dscm_pluggable_client : DscmPluggableClient): +def test_ConfigurePluggable(dscm_pluggable_client : PluggablesClient): LOGGER.info('Starting ConfigurePluggable test...') LOGGER.info('Creating Pluggable to configure for test...') diff --git a/src/dscm/tests/testmessages.py b/src/pluggables/tests/testmessages.py similarity index 87% rename from src/dscm/tests/testmessages.py rename to src/pluggables/tests/testmessages.py index bf6858854..d67a022c4 100644 --- a/src/dscm/tests/testmessages.py +++ b/src/pluggables/tests/testmessages.py @@ -14,8 +14,7 @@ import uuid from typing import Optional, List, Dict, Any -from common.proto import dscm_pluggable_pb2 - +from common.proto import pluggables_pb2 ########################### # CreatePluggableRequest @@ -25,7 +24,7 @@ def create_pluggable_request( device_uuid: Optional[str] = None, preferred_pluggable_index: Optional[int] = None, with_initial_config: bool = False -) -> dscm_pluggable_pb2.CreatePluggableRequest: +) -> pluggables_pb2.CreatePluggableRequest: # pyright: ignore[reportInvalidTypeForm] """ Create a CreatePluggableRequest message. @@ -37,7 +36,7 @@ def create_pluggable_request( Returns: CreatePluggableRequest message """ - _request = dscm_pluggable_pb2.CreatePluggableRequest() + _request = pluggables_pb2.CreatePluggableRequest() # Set device ID if device_uuid is None: @@ -83,8 +82,8 @@ def create_pluggable_request( def create_list_pluggables_request( device_uuid: Optional[str] = None, - view_level: dscm_pluggable_pb2.View = dscm_pluggable_pb2.VIEW_FULL -) -> dscm_pluggable_pb2.ListPluggablesRequest: + view_level: pluggables_pb2.View = pluggables_pb2.VIEW_FULL # pyright: ignore[reportInvalidTypeForm] +) -> pluggables_pb2.ListPluggablesRequest: # pyright: ignore[reportInvalidTypeForm] """ Create a ListPluggablesRequest message. @@ -95,7 +94,7 @@ def create_list_pluggables_request( Returns: ListPluggablesRequest message """ - _request = dscm_pluggable_pb2.ListPluggablesRequest() + _request = pluggables_pb2.ListPluggablesRequest() if device_uuid is None: device_uuid = "9bbf1937-db9e-45bc-b2c6-3214a9d42157" @@ -113,8 +112,8 @@ def create_list_pluggables_request( def create_get_pluggable_request( device_uuid: str, pluggable_index: int, - view_level: dscm_pluggable_pb2.View = dscm_pluggable_pb2.VIEW_FULL -) -> dscm_pluggable_pb2.GetPluggableRequest: + view_level: pluggables_pb2.View = pluggables_pb2.VIEW_FULL # pyright: ignore[reportInvalidTypeForm] +) -> pluggables_pb2.GetPluggableRequest: # pyright: ignore[reportInvalidTypeForm] """ Create a GetPluggableRequest message. @@ -126,7 +125,7 @@ def create_get_pluggable_request( Returns: GetPluggableRequest message """ - _request = dscm_pluggable_pb2.GetPluggableRequest() + _request = pluggables_pb2.GetPluggableRequest() _request.id.device.device_uuid.uuid = device_uuid _request.id.pluggable_index = pluggable_index _request.view_level = view_level @@ -140,7 +139,7 @@ def create_get_pluggable_request( def create_delete_pluggable_request( device_uuid: str, pluggable_index: int -) -> dscm_pluggable_pb2.DeletePluggableRequest: +) -> pluggables_pb2.DeletePluggableRequest: # pyright: ignore[reportInvalidTypeForm] """ Create a DeletePluggableRequest message. @@ -151,7 +150,7 @@ def create_delete_pluggable_request( Returns: DeletePluggableRequest message """ - _request = dscm_pluggable_pb2.DeletePluggableRequest() + _request = pluggables_pb2.DeletePluggableRequest() _request.id.device.device_uuid.uuid = device_uuid _request.id.pluggable_index = pluggable_index @@ -166,10 +165,10 @@ def create_configure_pluggable_request( device_uuid: str, pluggable_index: int, update_mask_paths: Optional[list] = None, - view_level: dscm_pluggable_pb2.View = dscm_pluggable_pb2.VIEW_FULL, # pyright: ignore[reportInvalidTypeForm] + view_level: pluggables_pb2.View = pluggables_pb2.VIEW_FULL, # pyright: ignore[reportInvalidTypeForm] apply_timeout_seconds: int = 30, parameters: Optional[List[Dict[str, Any]]] = [], -) -> dscm_pluggable_pb2.ConfigurePluggableRequest: # pyright: ignore[reportInvalidTypeForm] +) -> pluggables_pb2.ConfigurePluggableRequest: # pyright: ignore[reportInvalidTypeForm] """ Create a ConfigurePluggableRequest message. @@ -183,7 +182,7 @@ def create_configure_pluggable_request( Returns: ConfigurePluggableRequest message """ - _request = dscm_pluggable_pb2.ConfigurePluggableRequest() + _request = pluggables_pb2.ConfigurePluggableRequest() # Set pluggable configuration _request.config.id.device.device_uuid.uuid = device_uuid -- GitLab From c6085005b0987b2bf76b79d76764c426f46a09fc Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Wed, 29 Oct 2025 17:43:12 +0000 Subject: [PATCH 03/15] Refactor Pluggables Service and add SBI support - Removed unused import of google.protobuf.field_mask from pluggables.proto. - Updated DigitalSubcarrierGroupState and PluggableConfig messages in pluggables.proto to include additional fields for configuration management. - Enhanced PluggablesServiceServicerImpl to support pushing configurations to devices, including error handling and logging. - Added new methods for translating pluggable configurations to NETCONF format in config_translator.py. - Created CommonObjects.py and PreparePluggablesTestScenario.py for test setup and device configuration. - Implemented comprehensive tests for creating, configuring, retrieving, and deleting pluggables with NETCONF devices in test_pluggables_with_SBI.py. - Updated pytest.ini to include integration test marker for better test categorization. --- proto/pluggables.proto | 14 +- proto/policy.proto | 2 +- src/device/service/drivers/__init__.py | 26 +- .../service/PluggablesServiceServicerImpl.py | 139 +++++++- src/pluggables/service/config_translator.py | 101 ++++++ src/pluggables/tests/CommonObjects.py | 176 +++++++++++ .../tests/PreparePluggablesTestScenario.py | 170 ++++++++++ .../tests/test_pluggables_with_SBI.py | 296 ++++++++++++++++++ src/pluggables/tests/testmessages.py | 12 + src/pytest.ini | 18 ++ 10 files changed, 928 insertions(+), 26 deletions(-) create mode 100644 src/pluggables/service/config_translator.py create mode 100644 src/pluggables/tests/CommonObjects.py create mode 100644 src/pluggables/tests/PreparePluggablesTestScenario.py create mode 100644 src/pluggables/tests/test_pluggables_with_SBI.py create mode 100644 src/pytest.ini diff --git a/proto/pluggables.proto b/proto/pluggables.proto index 96661c02a..d6abe9daf 100644 --- a/proto/pluggables.proto +++ b/proto/pluggables.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package tfs.pluggables.v0; import "context.proto"; -import "google/protobuf/field_mask.proto"; service PluggablesService { rpc CreatePluggable (CreatePluggableRequest) returns (Pluggable) {} @@ -56,7 +55,7 @@ message DigitalSubcarrierGroupConfig { message DigitalSubcarrierGroupState { DigitalSubcarrierGroupId id = 1; - int32 count = 2; // available DSCs + int32 count = 2; // available DSCs double group_capacity_gbps = 3; double subcarrier_spacing_mhz = 4; repeated DigitalSubcarrierState subcarriers = 10; @@ -65,6 +64,10 @@ message DigitalSubcarrierGroupState { message PluggableConfig { PluggableId id = 1; + double target_output_power_dbm = 2; // target output power for the pluggable + double center_frequency_mhz = 3; // center frequency in MHz + int32 operational_mode = 4; // e.g., 0=off and 1=on + int32 line_port = 5; // line port number repeated DigitalSubcarrierGroupConfig dsc_groups = 10; } @@ -108,10 +111,9 @@ message DeletePluggableRequest { } message ConfigurePluggableRequest { - PluggableConfig config = 1; - google.protobuf.FieldMask update_mask = 2; // Not Implemented yet (for partial updates) - View view_level = 3; - int32 apply_timeout_seconds = 10; // Not Implemented yet (for timeout) + PluggableConfig config = 1; + View view_level = 2; + int32 apply_timeout_seconds = 10; // Not Implemented yet (for timeout) } // to control the level of detail in responses diff --git a/proto/policy.proto b/proto/policy.proto index d788f86aa..51ea63b7f 100644 --- a/proto/policy.proto +++ b/proto/policy.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package policy; import "context.proto"; -import "policy_condition.proto"; // WARNING: Not used +import "policy_condition.proto"; import "policy_action.proto"; import "monitoring.proto"; // to be migrated to: "kpi_manager.proto" diff --git a/src/device/service/drivers/__init__.py b/src/device/service/drivers/__init__.py index 5768e2310..21a0ec111 100644 --- a/src/device/service/drivers/__init__.py +++ b/src/device/service/drivers/__init__.py @@ -98,19 +98,19 @@ if LOAD_ALL_DEVICE_DRIVERS: } ])) -if LOAD_ALL_DEVICE_DRIVERS: - from .gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position - DRIVERS.append( - (GnmiOpenConfigDriver, [ - { - # Real Packet Router, specifying gNMI OpenConfig Driver => use GnmiOpenConfigDriver - FilterFieldEnum.DEVICE_TYPE: [ - DeviceTypeEnum.PACKET_POP, - DeviceTypeEnum.PACKET_ROUTER, - ], - FilterFieldEnum.DRIVER : DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG, - } - ])) +# if LOAD_ALL_DEVICE_DRIVERS: +# from .gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position +# DRIVERS.append( +# (GnmiOpenConfigDriver, [ +# { +# # Real Packet Router, specifying gNMI OpenConfig Driver => use GnmiOpenConfigDriver +# FilterFieldEnum.DEVICE_TYPE: [ +# DeviceTypeEnum.PACKET_POP, +# DeviceTypeEnum.PACKET_ROUTER, +# ], +# FilterFieldEnum.DRIVER : DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG, +# } +# ])) if LOAD_ALL_DEVICE_DRIVERS: from .gnmi_nokia_srlinux.GnmiNokiaSrLinuxDriver import GnmiNokiaSrLinuxDriver # pylint: disable=wrong-import-position diff --git a/src/pluggables/service/PluggablesServiceServicerImpl.py b/src/pluggables/service/PluggablesServiceServicerImpl.py index a12be3ca4..5509b9d66 100644 --- a/src/pluggables/service/PluggablesServiceServicerImpl.py +++ b/src/pluggables/service/PluggablesServiceServicerImpl.py @@ -12,22 +12,98 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging, grpc +import json, logging, grpc from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method -from common.proto.context_pb2 import Empty +from common.proto.context_pb2 import Empty, Device, DeviceId from common.proto.pluggables_pb2_grpc import PluggablesServiceServicer from common.proto.pluggables_pb2 import ( Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest) -from common.method_wrappers.ServiceExceptions import NotFoundException, AlreadyExistsException +from common.method_wrappers.ServiceExceptions import ( + NotFoundException, AlreadyExistsException, InvalidArgumentException) +from common.tools.object_factory.ConfigRule import json_config_rule_set +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from .config_translator import translate_pluggable_config_to_netconf, create_config_rule_from_dict LOGGER = logging.getLogger(__name__) -METRICS_POOL = MetricsPool('DscmPluggable', 'ServicegRPC') +METRICS_POOL = MetricsPool('Pluggables', 'ServicegRPC') class PluggablesServiceServicerImpl(PluggablesServiceServicer): def __init__(self): - LOGGER.info('Init DscmPluggableService') + LOGGER.info('Initiate PluggablesService') self.pluggables = {} # In-memory store for pluggables + self.context_client = ContextClient() + self.device_client = DeviceClient() + + def _push_config_to_device(self, device_uuid: str, pluggable_index: int, pluggable_config): # type: ignore + """ + Push pluggable configuration to the actual device via DeviceClient. + Args: + device_uuid: UUID of the target device + pluggable_index: Index of the pluggable + pluggable_config: PluggableConfig protobuf message + """ + LOGGER.info(f"Configuring device {device_uuid}, pluggable index {pluggable_index}") + + # Step 1: Get the device from Context service (to extract IP address and determine template) + try: + device = self.context_client.GetDevice(DeviceId(device_uuid={'uuid': device_uuid})) # type: ignore + LOGGER.info(f"Retrieved device from Context: {device.name}") + except grpc.RpcError as e: + LOGGER.error(f"Failed to get device {device_uuid} from Context: {e}") + raise + + # Translate pluggable config to NETCONF format + component_name = f"channel-{pluggable_index}" + netconf_config = translate_pluggable_config_to_netconf(pluggable_config, component_name=component_name) + + LOGGER.info(f"Translated pluggable config to NETCONF format: {netconf_config}") + + # Step 2: Extract device IP address from _connect/address config rule + device_address = None + for config_rule in device.device_config.config_rules: # type: ignore + if config_rule.custom.resource_key == '_connect/address': # type: ignore + device_address = config_rule.custom.resource_value # type: ignore + break + + # Step 3: Determine the appropriate template based on device IP address (TODO: This need to be updated later) + if device_address == '10.30.7.7': + template_identifier = 'hub' + elif device_address == '10.30.7.8': + template_identifier = 'leaf' + else: + # Default to hub template if IP address cannot be determined + LOGGER.warning(f"Cannot determine device type from IP address {device_address}, defaulting to hub template") + raise InvalidArgumentException( 'Device IP address', device_address, extra_details='Unknown device IP adress') + + LOGGER.info(f"Using template identifier: {template_identifier} for device {device.name} (IP: {device_address})") + + # Step 4: Create configuration rule with template-specific resource key + # For simplicity, we use a fixed pluggable index of 1 for template lookup + template_index = 1 # TODO: This should be dynamic based on actual pluggable index + resource_key = f"/pluggable/{template_index}/config/{template_identifier}" + + # Create config rule dict and convert to protobuf + config_json = json.dumps(netconf_config) + config_rule_dict = json_config_rule_set(resource_key, config_json) + config_rule = create_config_rule_from_dict(config_rule_dict) + + # Step 5: Create a minimal Device object with only the DSCM config rule + config_device = Device() + config_device.device_id.device_uuid.uuid = device_uuid # type: ignore + config_device.device_config.config_rules.append(config_rule) # type: ignore + + LOGGER.info(f"Created minimal device with config rule: resource_key={resource_key}, template={template_identifier}") + + # Step 6: Call ConfigureDevice to push the configuration + try: + device_id = self.device_client.ConfigureDevice(config_device) + LOGGER.info(f"Successfully configured device {device_id.device_uuid.uuid}") # type: ignore + except grpc.RpcError as e: + LOGGER.error(f"Failed to configure device {device_uuid}: {e}") + raise InvalidArgumentException( + 'Device configuration', f'{device_uuid}:{pluggable_index}', extra_details=str(e)) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreatePluggable( @@ -37,7 +113,7 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): device_uuid = request.device.device_uuid.uuid - if request.preferred_pluggable_index and request.preferred_pluggable_index >= 0: + if request.preferred_pluggable_index >= 0: pluggable_index = request.preferred_pluggable_index else: pluggable_index = -1 @@ -70,6 +146,25 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): pluggable.state.id.device.device_uuid.uuid = device_uuid pluggable.state.id.pluggable_index = pluggable_index + + # Verify device exists in Context service + try: + device = self.context_client.GetDevice(DeviceId(device_uuid={'uuid': device_uuid})) # type: ignore + LOGGER.info(f"Device {device_uuid} found in Context service: {device.name}") + except grpc.RpcError as e: + LOGGER.error(f"Device {device_uuid} not found in Context service: {e}") + raise NotFoundException('Device', device_uuid, extra_details='Device must exist before creating pluggable') + + # If initial_config is provided, push configuration to device + if request.HasField('initial_config') and len(pluggable.config.dsc_groups) > 0: + LOGGER.info(f"Pushing initial configuration to device {device_uuid}") + try: + self._push_config_to_device(device_uuid, pluggable_index, pluggable.config) + except Exception as e: + LOGGER.error(f"Failed to push initial config to device: {e}") + raise + # We still create the pluggable in memory, but log the error + # In production, you might want to raise the exception instead self.pluggables[pluggable_key] = pluggable @@ -119,6 +214,21 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): LOGGER.info(f'No matching pluggable found: device={device_uuid}, index={pluggable_index}') raise NotFoundException('Pluggable', pluggable_key) + # Remove pluggable config from device via DSCM driver + try: + pluggable = self.pluggables[pluggable_key] + # Create empty config to trigger deletion + from common.proto.pluggables_pb2 import PluggableConfig + empty_config = PluggableConfig() + empty_config.id.device.device_uuid.uuid = device_uuid + empty_config.id.pluggable_index = pluggable_index + + LOGGER.info(f"Removing configuration from device {device_uuid}") + self._push_config_to_device(device_uuid, pluggable_index, empty_config) + except Exception as e: + LOGGER.error(f"Failed to remove config from device: {e}") + # Continue with deletion from memory even if device config removal fails + del self.pluggables[pluggable_key] LOGGER.info(f"Deleted pluggable: device={device_uuid}, index={pluggable_index}") @@ -184,6 +294,23 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): subcarrier.id.group.pluggable.pluggable_index = pluggable_index has_config = len(pluggable.config.dsc_groups) > 0 + + # Push pluggable config to device via DSCM driver + if has_config: + LOGGER.info(f"Pushing configuration to device {device_uuid}") + try: + self._push_config_to_device(device_uuid, pluggable_index, pluggable.config) + except Exception as e: + LOGGER.error(f"Failed to push config to device: {e}") + # Continue even if device configuration fails + # In production, you might want to raise the exception + else: + LOGGER.info(f"Empty configuration - removing config from device {device_uuid}") + try: + self._push_config_to_device(device_uuid, pluggable_index, pluggable.config) + except Exception as e: + LOGGER.error(f"Failed to remove config from device: {e}") + state_msg = "configured" if has_config else "deconfigured (empty config)" LOGGER.info(f"Successfully {state_msg} pluggable: device={device_uuid}, index={pluggable_index}") diff --git a/src/pluggables/service/config_translator.py b/src/pluggables/service/config_translator.py new file mode 100644 index 000000000..3405fbb66 --- /dev/null +++ b/src/pluggables/service/config_translator.py @@ -0,0 +1,101 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Dict, Any, List, Optional +from common.proto.pluggables_pb2 import PluggableConfig +from common.proto.context_pb2 import ConfigRule, ConfigActionEnum + +LOGGER = logging.getLogger(__name__) + + +def create_config_rule_from_dict(config_rule_dict: Dict[str, Any]) -> ConfigRule: # type: ignore + config_rule = ConfigRule() + config_rule.action = config_rule_dict['action'] + config_rule.custom.resource_key = config_rule_dict['custom']['resource_key'] + config_rule.custom.resource_value = config_rule_dict['custom']['resource_value'] + return config_rule + + +def translate_pluggable_config_to_netconf( + pluggable_config: PluggableConfig, # type: ignore + component_name: str = "channel-1" # channel-1 for HUB and channel-1/3/5 for LEAF +) -> Dict[str, Any]: + """ + Translate PluggableConfig protobuf message to the format expected by NetConfDriver. + Args: + pluggable_config: PluggableConfig message containing DSC groups and subcarriers + component_name: Name of the optical channel component (default: "channel-1") + Returns: + Dictionary in the format expected by NetConfDriver templates: + """ + + if not pluggable_config or not pluggable_config.dsc_groups: + LOGGER.warning("Empty pluggable config provided") + return { + "name": component_name, + "operation": "delete" + } + + if not hasattr(pluggable_config, 'center_frequency_mhz') or pluggable_config.center_frequency_mhz <= 0: + raise ValueError("center_frequency_mhz is required and must be greater than 0 in PluggableConfig") + center_frequency_mhz = int(pluggable_config.center_frequency_mhz) + + if not hasattr(pluggable_config, 'operational_mode') or pluggable_config.operational_mode <= 0: + raise ValueError("operational_mode is required and must be greater than 0 in PluggableConfig") + operational_mode = pluggable_config.operational_mode + + if not hasattr(pluggable_config, 'target_output_power_dbm'): + raise ValueError("target_output_power_dbm is required in PluggableConfig") + target_output_power = pluggable_config.target_output_power_dbm + + if not hasattr(pluggable_config, 'line_port'): + raise ValueError("line_port is required in PluggableConfig") + line_port = pluggable_config.line_port + + LOGGER.debug(f"Extracted config values: freq={center_frequency_mhz} MHz, " + f"op_mode={operational_mode}, power={target_output_power} dBm, line_port={line_port}") + + # Build digital subcarriers groups + digital_sub_carriers_groups = [] + + for group_dsc in pluggable_config.dsc_groups: + group_dsc_data = { + "digital_sub_carriers_group_id": group_dsc.id.group_index, + "digital_sub_carrier_id": [] + } + + for subcarrier in group_dsc.subcarriers: + # Only subcarrier_index and active status are needed for Jinja2 template + subcarrier_data = { + "sub_carrier_id": subcarrier.id.subcarrier_index, + "active": "true" if subcarrier.active else "false" + } + group_dsc_data["digital_sub_carrier_id"].append(subcarrier_data) + + digital_sub_carriers_groups.append(group_dsc_data) + + # Build the final configuration dictionary + config = { + "name": component_name, + "frequency": center_frequency_mhz, + "operational_mode": operational_mode, + "target_output_power": target_output_power, + "digital_sub_carriers_group": digital_sub_carriers_groups + } + + LOGGER.info(f"Translated pluggable config to NETCONF format: component={component_name}, " + f"frequency={center_frequency_mhz} MHz, groups={len(digital_sub_carriers_groups)}") + + return config diff --git a/src/pluggables/tests/CommonObjects.py b/src/pluggables/tests/CommonObjects.py new file mode 100644 index 000000000..4ddb57104 --- /dev/null +++ b/src/pluggables/tests/CommonObjects.py @@ -0,0 +1,176 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy, logging +from typing import Dict, Any, Optional +from common.Constants import DEFAULT_CONTEXT_NAME, DEFAULT_TOPOLOGY_NAME +from common.tools.object_factory.Context import json_context, json_context_id +from common.tools.object_factory.Topology import json_topology, json_topology_id +from common.proto.context_pb2 import Device +from common.tools.object_factory.Device import ( + json_device_connect_rules, json_device_id, json_device_packetrouter_disabled +) + +LOGGER = logging.getLogger(__name__) + + +# ----- Context -------------------------------------------------------------------------------------------------------- +CONTEXT_ID = json_context_id(DEFAULT_CONTEXT_NAME) +CONTEXT = json_context(DEFAULT_CONTEXT_NAME) + + +# ----- Topology ------------------------------------------------------------------------------------------------------- +TOPOLOGY_ID = json_topology_id(DEFAULT_TOPOLOGY_NAME, context_id=CONTEXT_ID) +TOPOLOGY = json_topology(DEFAULT_TOPOLOGY_NAME, context_id=CONTEXT_ID) + + +# ----- Hub Device Configuration --------------------------------------------- + +DEVICE_HUB_UUID = 'hub-device-uuid-001' +DEVICE_HUB_ADDRESS = '10.30.7.7' +DEVICE_HUB_PORT = 2023 +DEVICE_HUB_USERNAME = 'admin' +DEVICE_HUB_PASSWORD = 'admin' +DEVICE_HUB_TIMEOUT = 15 + +DEVICE_HUB_ID = json_device_id(DEVICE_HUB_UUID) +DEVICE_HUB = json_device_packetrouter_disabled(DEVICE_HUB_UUID, name='Hub-Router') + +DEVICE_HUB_CONNECT_RULES = json_device_connect_rules(DEVICE_HUB_ADDRESS, DEVICE_HUB_PORT, { + 'username': DEVICE_HUB_USERNAME, + 'password': DEVICE_HUB_PASSWORD, + 'force_running': False, + 'hostkey_verify': False, + 'look_for_keys': False, + 'allow_agent': False, + 'commit_per_rule': False, + 'device_params': {'name': 'default'}, + 'manager_params': {'timeout': DEVICE_HUB_TIMEOUT}, +}) + +# ----- Leaf Device Configuration -------------------------------------------- + +DEVICE_LEAF_UUID = 'leaf-device-uuid-001' +DEVICE_LEAF_ADDRESS = '10.30.7.8' +DEVICE_LEAF_PORT = 2023 +DEVICE_LEAF_USERNAME = 'admin' +DEVICE_LEAF_PASSWORD = 'admin' +DEVICE_LEAF_TIMEOUT = 15 + +DEVICE_LEAF_ID = json_device_id(DEVICE_LEAF_UUID) +DEVICE_LEAF = json_device_packetrouter_disabled(DEVICE_LEAF_UUID, name='Leaf-Router') + +DEVICE_LEAF_CONNECT_RULES = json_device_connect_rules(DEVICE_LEAF_ADDRESS, DEVICE_LEAF_PORT, { + 'username': DEVICE_LEAF_USERNAME, + 'password': DEVICE_LEAF_PASSWORD, + 'force_running': False, + 'hostkey_verify': False, + 'look_for_keys': False, + 'allow_agent': False, + 'commit_per_rule': False, + 'device_params': {'name': 'default'}, + 'manager_params': {'timeout': DEVICE_LEAF_TIMEOUT}, +}) + +# ----- Complete Device Objects with Connect Rules -------------------------- + +def get_device_hub_with_connect_rules() -> Device: + """ + Create a complete Hub device with connection rules. + + Returns: + Device protobuf object ready to be added to Context + """ + device_dict = copy.deepcopy(DEVICE_HUB) + device_dict['device_config']['config_rules'].extend(DEVICE_HUB_CONNECT_RULES) + return Device(**device_dict) + + +def get_device_leaf_with_connect_rules() -> Device: + """ + Create a complete Leaf device with connection rules. + + Returns: + Device protobuf object ready to be added to Context + """ + device_dict = copy.deepcopy(DEVICE_LEAF) + device_dict['device_config']['config_rules'].extend(DEVICE_LEAF_CONNECT_RULES) + return Device(**device_dict) + +# ----- Device Connection Mapping -------------------------------------------- + +DEVICES_CONNECTION_INFO = { + 'hub': { + 'uuid': DEVICE_HUB_UUID, + 'address': DEVICE_HUB_ADDRESS, + 'port': DEVICE_HUB_PORT, + 'settings': {}, + 'username': DEVICE_HUB_USERNAME, + 'password': DEVICE_HUB_PASSWORD + }, + 'leaf': { + 'uuid': DEVICE_LEAF_UUID, + 'address': DEVICE_LEAF_ADDRESS, + 'port': DEVICE_LEAF_PORT, + 'settings': {}, + 'username': DEVICE_LEAF_USERNAME, + 'password': DEVICE_LEAF_PASSWORD + }, +} + + +def get_device_connection_info(device_uuid: str) -> Optional[Dict[str, Any]]: + """ + Get device connection information for NETCONF driver. + + Args: + device_uuid: UUID of the device + + Returns: + Dictionary with connection info or None if not found + """ + # Map device UUIDs to device types + device_mapping = { + DEVICE_HUB_UUID: 'hub', + DEVICE_LEAF_UUID: 'leaf', + } + + device_type = device_mapping.get(device_uuid) + + if device_type and device_type in DEVICES_CONNECTION_INFO: + return DEVICES_CONNECTION_INFO[device_type] + + return None + + +def determine_node_identifier(device_uuid: str, pluggable_index: int) -> str: + """ + Determine the node identifier (e.g., 'T2.1', 'T1.1', etc.) for the device. + + Args: + device_uuid: UUID of the device + pluggable_index: Index of the pluggable + + Returns: + Node identifier string + """ + conn_info = get_device_connection_info(device_uuid) + + if conn_info and conn_info['address'] == DEVICE_HUB_ADDRESS: + return 'T2.1' # Hub node + elif conn_info and conn_info['address'] == DEVICE_LEAF_ADDRESS: + # For multiple leaf nodes, use pluggable_index to differentiate + return f'T1.{pluggable_index + 1}' + else: + return 'T1.1' # Default diff --git a/src/pluggables/tests/PreparePluggablesTestScenario.py b/src/pluggables/tests/PreparePluggablesTestScenario.py new file mode 100644 index 000000000..937f09099 --- /dev/null +++ b/src/pluggables/tests/PreparePluggablesTestScenario.py @@ -0,0 +1,170 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy, logging, os, pytest +from typing import Union +from common.Constants import ServiceNameEnum +from common.Settings import ( + ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, + get_env_var_name, get_service_port_grpc +) +from common.proto.context_pb2 import Device, DeviceId, Topology, Context +from common.tools.service.GenericGrpcService import GenericGrpcService +from common.tests.MockServicerImpl_Context import MockServicerImpl_Context +from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from device.service.DeviceService import DeviceService +from device.service.driver_api.DriverFactory import DriverFactory +from device.service.driver_api.DriverInstanceCache import DriverInstanceCache +from device.service.drivers import DRIVERS +from pluggables.client.PluggablesClient import PluggablesClient +from pluggables.service.PluggablesService import PluggablesService +from pluggables.tests.CommonObjects import ( + DEVICE_HUB, DEVICE_HUB_ID, DEVICE_HUB_UUID, DEVICE_HUB_CONNECT_RULES, + DEVICE_LEAF, DEVICE_LEAF_ID, DEVICE_LEAF_UUID, DEVICE_LEAF_CONNECT_RULES, + CONTEXT_ID, CONTEXT, TOPOLOGY_ID, TOPOLOGY, + get_device_hub_with_connect_rules, get_device_leaf_with_connect_rules +) +from common.tools.object_factory.Topology import json_topology + +LOGGER = logging.getLogger(__name__) + +LOCAL_HOST = '127.0.0.1' +MOCKSERVICE_PORT = 10000 + +# Configure service endpoints +CONTEXT_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.CONTEXT) +DEVICE_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.DEVICE) +DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) + +os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(CONTEXT_SERVICE_PORT) +os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DEVICE_SERVICE_PORT) +os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) + + +class MockContextService(GenericGrpcService): + """Mock Context Service for testing""" + + def __init__(self, bind_port: Union[str, int]) -> None: + super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockContextService') + + def install_servicers(self): + self.context_servicer = MockServicerImpl_Context() + add_ContextServiceServicer_to_server(self.context_servicer, self.server) + +@pytest.fixture(scope='session') +def mock_context_service(): + """Start mock Context service for the test session""" + LOGGER.info('Initializing MockContextService...') + _service = MockContextService(CONTEXT_SERVICE_PORT) + _service.start() + yield _service + LOGGER.info('Terminating MockContextService...') + _service.stop() + + +@pytest.fixture(scope='session') +def context_client(mock_context_service): # pylint: disable=redefined-outer-name + """Create Context client for the test session""" + LOGGER.info('Creating ContextClient...') + _client = ContextClient() + yield _client + LOGGER.info('Closing ContextClient...') + _client.close() + +@pytest.fixture(scope='session') +def device_service(context_client : ContextClient): # pylint: disable=redefined-outer-name + """Start Device service for the test session""" + LOGGER.info('Initializing DeviceService...') + _driver_factory = DriverFactory(DRIVERS) + _driver_instance_cache = DriverInstanceCache(_driver_factory) + _service = DeviceService(_driver_instance_cache) + _service.start() + yield _service + LOGGER.info('Terminating DeviceService...') + _service.stop() + +@pytest.fixture(scope='session') +def device_client(device_service: DeviceService): # pylint: disable=redefined-outer-name + """Create Device client for the test session""" + LOGGER.info('Creating DeviceClient...') + _client = DeviceClient() + yield _client + LOGGER.info('Closing DeviceClient...') + _client.close() + + +@pytest.fixture(scope='session') +def pluggables_service(context_client: ContextClient): + """Start Pluggables service for the test session""" + LOGGER.info('Initializing PluggablesService...') + _service = PluggablesService() + _service.start() + yield _service + LOGGER.info('Terminating PluggablesService...') + _service.stop() + + +@pytest.fixture(scope='session') +def pluggables_client(pluggables_service: PluggablesService, + context_client: ContextClient, + device_client: DeviceClient + ): + """Create Pluggables client for the test session""" + LOGGER.info('Creating PluggablesClient...') + _client = PluggablesClient() + yield _client + LOGGER.info('Closing PluggablesClient...') + _client.close() + + +def test_prepare_environment( + context_client: ContextClient, # pylint: disable=redefined-outer-name + device_client: DeviceClient, # pylint: disable=redefined-outer-name + pluggables_client: PluggablesClient, + device_service: DeviceService ): # pylint: disable=redefined-outer-name + """Prepare test environment by adding devices to Context""" + + LOGGER.info('Preparing test environment...') + + + context_client.SetContext(Context(**CONTEXT)) + context_client.SetTopology(Topology(**TOPOLOGY)) + LOGGER.info('Created admin Context and Topology') + + # Add Hub device with connect rules + hub_device = get_device_hub_with_connect_rules() + context_client.SetDevice(hub_device) + LOGGER.info(f'Added Hub device: {DEVICE_HUB_UUID}') + + # Add Leaf device with connect rules + leaf_device = get_device_leaf_with_connect_rules() + context_client.SetDevice(leaf_device) + LOGGER.info(f'Added Leaf device: {DEVICE_LEAF_UUID}') + + # Verify devices were added + hub_device_retrieved = context_client.GetDevice(DeviceId(**DEVICE_HUB_ID)) + assert hub_device_retrieved is not None + assert hub_device_retrieved.device_id.device_uuid.uuid == DEVICE_HUB_UUID + LOGGER.info(f'Verified Hub device: {hub_device_retrieved.name}') + + leaf_device_retrieved = context_client.GetDevice(DeviceId(**DEVICE_LEAF_ID)) + assert leaf_device_retrieved is not None + assert leaf_device_retrieved.device_id.device_uuid.uuid == DEVICE_LEAF_UUID + LOGGER.info(f'Verified Leaf device: {leaf_device_retrieved.name}') + diff --git a/src/pluggables/tests/test_pluggables_with_SBI.py b/src/pluggables/tests/test_pluggables_with_SBI.py new file mode 100644 index 000000000..7e8e56037 --- /dev/null +++ b/src/pluggables/tests/test_pluggables_with_SBI.py @@ -0,0 +1,296 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging, pytest +from common.proto.context_pb2 import Empty +from common.proto.pluggables_pb2 import Pluggable, View +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from pluggables.client.PluggablesClient import PluggablesClient +from pluggables.tests.testmessages import ( + create_pluggable_request, create_list_pluggables_request, + create_get_pluggable_request, create_delete_pluggable_request, + create_configure_pluggable_request +) +from pluggables.tests.CommonObjects import ( + DEVICE_HUB_UUID, DEVICE_LEAF_UUID +) +from pluggables.tests.PreparePluggablesTestScenario import ( # pylint: disable=unused-import + # be careful, order of symbols is important here! + mock_context_service, context_client, device_service, device_client, + pluggables_service, pluggables_client, test_prepare_environment +) + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +@pytest.fixture(autouse=True) +def log_all_methods(request): + ''' + This fixture logs messages before and after each test function runs, indicating the start and end of the test. + The autouse=True parameter ensures that this logging happens automatically for all tests in the module. + ''' + LOGGER.info(f" >>>>> Starting test: {request.node.name} ") + yield + LOGGER.info(f" <<<<< Finished test: {request.node.name} ") + +# ----- Pluggable Tests with NETCONF ----------------------------------------- + +# Number 1. +def test_create_pluggable_hub_without_config(pluggables_client: PluggablesClient): + """Test creating a pluggable on Hub device without initial configuration""" + LOGGER.info('Creating Pluggable on Hub device without config...') + + _request = create_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + preferred_pluggable_index=1, # set 1 for HUB and leaf-1, 2 for leaf-2 and 3 for leaf-3 + with_initial_config=False + ) + + _pluggable = pluggables_client.CreatePluggable(_request) + + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.device.device_uuid.uuid == DEVICE_HUB_UUID + assert _pluggable.id.pluggable_index == 1 + LOGGER.info(f'Created Pluggable on Hub: {_pluggable.id}') + +# Number 2. +@pytest.mark.integration +def test_create_pluggable_hub_with_config(pluggables_client: PluggablesClient): + """Test creating a pluggable on Hub device with initial configuration + + Requires: Real NETCONF device at 10.30.7.7:2023 + """ + LOGGER.info('Creating Pluggable on Hub device with config...') + + _request = create_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + preferred_pluggable_index=2, # Use index 2 to avoid conflict with test #1 + with_initial_config=True + ) + + _pluggable = pluggables_client.CreatePluggable(_request) + + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.device.device_uuid.uuid == DEVICE_HUB_UUID + assert _pluggable.id.pluggable_index == 2 + assert len(_pluggable.config.dsc_groups) == 1 # Should be 1, not 2 (check testmessages.py) + + # Verify DSC group configuration + dsc_group = _pluggable.config.dsc_groups[0] + assert dsc_group.group_size == 4 # From testmessages.py + assert len(dsc_group.subcarriers) == 2 + + LOGGER.info(f'Created Pluggable on Hub with {len(dsc_group.subcarriers)} subcarriers') + +# Number 3. +@pytest.mark.integration +def test_create_pluggable_leaf_with_config(pluggables_client: PluggablesClient): + """Test creating a pluggable on Leaf device with initial configuration + + Requires: Real NETCONF device at 10.30.7.8:2023 + """ + LOGGER.info('Creating Pluggable on Leaf device with config...') + + _request = create_pluggable_request( + device_uuid=DEVICE_LEAF_UUID, + preferred_pluggable_index=1, + with_initial_config=True + ) + + _pluggable = pluggables_client.CreatePluggable(_request) + + assert isinstance(_pluggable, Pluggable) + assert _pluggable.id.device.device_uuid.uuid == DEVICE_LEAF_UUID + assert _pluggable.id.pluggable_index == 1 # Should be 1, not 0 + assert len(_pluggable.config.dsc_groups) == 1 + + LOGGER.info(f'Created Pluggable on Leaf: {_pluggable.id}') + +# Number 4. +@pytest.mark.integration +def test_configure_pluggable_hub(pluggables_client: PluggablesClient): + """Test configuring an existing pluggable on Hub device""" + LOGGER.info('Configuring existing Pluggable on Hub device...') + + # First, create a pluggable without config + _create_request = create_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + preferred_pluggable_index=3, # Use index 3 to avoid conflicts + with_initial_config=False + ) + _created = pluggables_client.CreatePluggable(_create_request) + assert _created.id.pluggable_index == 3 + + # Now configure it + _config_request = create_configure_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + pluggable_index=3, # Match the created index + view_level=View.VIEW_FULL + ) + + _configured = pluggables_client.ConfigurePluggable(_config_request) + + assert isinstance(_configured, Pluggable) + assert _configured.id.device.device_uuid.uuid == DEVICE_HUB_UUID + assert _configured.id.pluggable_index == 3 + assert len(_configured.config.dsc_groups) == 1 + + # Verify configuration was applied + dsc_group = _configured.config.dsc_groups[0] + assert dsc_group.group_size == 2 + assert len(dsc_group.subcarriers) == 2 + + LOGGER.info(f'Configured Pluggable on Hub with {len(dsc_group.subcarriers)} subcarriers') + +# Number 5. +@pytest.mark.integration +def test_get_pluggable(pluggables_client: PluggablesClient): + """Test retrieving an existing pluggable + + Requires: Real NETCONF device at 10.30.7.7:2023 + """ + LOGGER.info('Getting existing Pluggable...') + + # Create a pluggable first + _create_request = create_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + preferred_pluggable_index=4, # Use index 4 to avoid conflicts + with_initial_config=True + ) + _created = pluggables_client.CreatePluggable(_create_request) + + # Now get it + _get_request = create_get_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + pluggable_index=4, # Match the created index + view_level=View.VIEW_FULL + ) + + _retrieved = pluggables_client.GetPluggable(_get_request) + + assert isinstance(_retrieved, Pluggable) + assert _retrieved.id.device.device_uuid.uuid == DEVICE_HUB_UUID + assert _retrieved.id.pluggable_index == 4 + assert len(_retrieved.config.dsc_groups) == len(_created.config.dsc_groups) + + LOGGER.info(f'Retrieved Pluggable: {_retrieved.id}') + +# Number 6. +def test_list_pluggables(pluggables_client: PluggablesClient): + """Test listing all pluggables for a device""" + LOGGER.info('Listing Pluggables for Hub device...') + + _list_request = create_list_pluggables_request( + device_uuid=DEVICE_HUB_UUID, + view_level=View.VIEW_CONFIG + ) + + _response = pluggables_client.ListPluggables(_list_request) + + assert _response is not None + assert len(_response.pluggables) >= 1 # At least one from previous tests + + for pluggable in _response.pluggables: + assert pluggable.id.device.device_uuid.uuid == DEVICE_HUB_UUID + LOGGER.info(f'Found Pluggable: index={pluggable.id.pluggable_index}') + +# Number 7. +@pytest.mark.integration +def test_delete_pluggable(pluggables_client: PluggablesClient): + """Test deleting a pluggable + + Requires: Real NETCONF device at 10.30.7.8:2023 + """ + LOGGER.info('Deleting Pluggable...') + + # Create a pluggable to delete + _create_request = create_pluggable_request( + device_uuid=DEVICE_LEAF_UUID, + preferred_pluggable_index=2, # Use index 2 to avoid conflict with test #3 + with_initial_config=True + ) + _created = pluggables_client.CreatePluggable(_create_request) + assert _created.id.pluggable_index == 2 + + # Delete it + _delete_request = create_delete_pluggable_request( + device_uuid=DEVICE_LEAF_UUID, + pluggable_index=2 + ) + + _response = pluggables_client.DeletePluggable(_delete_request) + assert isinstance(_response, Empty) + + # Verify it's deleted + with pytest.raises(grpc.RpcError) as e: + _get_request = create_get_pluggable_request( + device_uuid=DEVICE_LEAF_UUID, + pluggable_index=2 + ) + pluggables_client.GetPluggable(_get_request) + + assert e.value.code() == grpc.StatusCode.NOT_FOUND + LOGGER.info('Successfully deleted Pluggable and verified removal') + +# Number 8. +def test_pluggable_already_exists_error(pluggables_client: PluggablesClient): + """Test that creating a pluggable with same key raises ALREADY_EXISTS""" + LOGGER.info('Testing ALREADY_EXISTS error...') + + _request = create_pluggable_request( + device_uuid=DEVICE_LEAF_UUID, + preferred_pluggable_index=3, # Use index 3 + with_initial_config=False + ) + + # Create first time - should succeed + _pluggable = pluggables_client.CreatePluggable(_request) + assert _pluggable.id.pluggable_index == 3 # Should be 3, not 5 + + # Try to create again - should fail + with pytest.raises(grpc.RpcError) as e: + pluggables_client.CreatePluggable(_request) + + assert e.value.code() == grpc.StatusCode.ALREADY_EXISTS + LOGGER.info('Successfully caught ALREADY_EXISTS error') + +# Number 9. +def test_pluggable_not_found_error(pluggables_client: PluggablesClient): + """Test that getting non-existent pluggable raises NOT_FOUND""" + LOGGER.info('Testing NOT_FOUND error...') + + _request = create_get_pluggable_request( + device_uuid=DEVICE_HUB_UUID, + pluggable_index=999, # Non-existent index + view_level=View.VIEW_FULL + ) + + with pytest.raises(grpc.RpcError) as e: + pluggables_client.GetPluggable(_request) + + assert e.value.code() == grpc.StatusCode.NOT_FOUND + LOGGER.info('Successfully caught NOT_FOUND error') + + +# ----- Cleanup Tests -------------------------------------------------------- + +def test_cleanup_environment( + context_client: ContextClient, # pylint: disable=redefined-outer-name + pluggables_client: PluggablesClient): # pylint: disable=redefined-outer-name + """Cleanup test environment by removing test devices""" + + + LOGGER.info('Test environment cleanup completed') diff --git a/src/pluggables/tests/testmessages.py b/src/pluggables/tests/testmessages.py index d67a022c4..9a0fb5a05 100644 --- a/src/pluggables/tests/testmessages.py +++ b/src/pluggables/tests/testmessages.py @@ -52,6 +52,12 @@ def create_pluggable_request( _request.initial_config.id.device.device_uuid.uuid = device_uuid _request.initial_config.id.pluggable_index = preferred_pluggable_index or 0 + # Set top-level PluggableConfig fields + _request.initial_config.center_frequency_mhz = 193100000 # 193.1 THz in MHz + _request.initial_config.operational_mode = 1 # Operational mode + _request.initial_config.target_output_power_dbm = -10.0 # Target output power + _request.initial_config.line_port = 1 # Line port number + # Add sample DSC group configuration dsc_group = _request.initial_config.dsc_groups.add() dsc_group.id.pluggable.device.device_uuid.uuid = device_uuid @@ -188,6 +194,12 @@ def create_configure_pluggable_request( _request.config.id.device.device_uuid.uuid = device_uuid _request.config.id.pluggable_index = pluggable_index + # Set top-level PluggableConfig fields + _request.config.center_frequency_mhz = 193100000 # 193.1 THz in MHz + _request.config.operational_mode = 1 # Operational mode + _request.config.target_output_power_dbm = -10.0 # Target output power + _request.config.line_port = 1 # Line port number + # Add DSC group configuration group_1 = _request.config.dsc_groups.add() group_1.id.pluggable.device.device_uuid.uuid = device_uuid diff --git a/src/pytest.ini b/src/pytest.ini new file mode 100644 index 000000000..6dfc29442 --- /dev/null +++ b/src/pytest.ini @@ -0,0 +1,18 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +[pytest] +markers = + integration: tests that require real NETCONF devices or external infrastructure -- GitLab From 427ed4e3e40a46f45f0dcbd6e52eb443def094b0 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 30 Oct 2025 05:21:55 +0000 Subject: [PATCH 04/15] feat: Add Pluggables Component activation options in deployment scripts and CI configuration --- deploy/all.sh | 3 + my_deploy.sh | 3 + src/pluggables/.gitlab-ci.yml | 102 ++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/deploy/all.sh b/deploy/all.sh index bd171eb66..85e27a640 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -69,6 +69,9 @@ export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice n # Uncomment to activate E2E Orchestrator #export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" +# Uncomment to activate Pluggables Component +#export TFS_COMPONENTS="${TFS_COMPONENTS} pluggables" + # If not already set, set the tag you want to use for your images. export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"} diff --git a/my_deploy.sh b/my_deploy.sh index 86c1a86f4..b8ed95a82 100644 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -95,6 +95,9 @@ export TFS_COMPONENTS="context device pathcomp service nbi webui" # Uncomment to activate Load Generator #export TFS_COMPONENTS="${TFS_COMPONENTS} load_generator" +# Uncomment to activate Pluggables Component +#export TFS_COMPONENTS="${TFS_COMPONENTS} pluggables" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/pluggables/.gitlab-ci.yml b/src/pluggables/.gitlab-ci.yml index 7363515f0..b9e58b0e8 100644 --- a/src/pluggables/.gitlab-ci.yml +++ b/src/pluggables/.gitlab-ci.yml @@ -11,3 +11,105 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# Build, tag, and push the Docker image to the GitLab Docker registry +build pluggables: + variables: + IMAGE_NAME: 'pluggables' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker image prune --force + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker image prune --force + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + +# Apply unit test to the component +unit_test pluggables: + variables: + IMAGE_NAME: 'pluggables' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: unit_test + needs: + - build pluggables + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi + - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi + - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi + - if docker container ls | grep context; then docker rm -f context; else echo "context container is not in the system"; fi + - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi + - docker container prune -f + script: + - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker pull "$CI_REGISTRY_IMAGE/context:$IMAGE_TAG" + - docker pull "cockroachdb/cockroach:latest-v22.2" + - docker volume create crdb + - > + docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080 + --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123 + --volume "crdb:/cockroach/cockroach-data" + cockroachdb/cockroach:latest-v22.2 start-single-node + - echo "Waiting for initialization..." + - while ! docker logs crdb 2>&1 | grep -q 'finished creating default user \"tfs\"'; do sleep 1; done + - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CRDB_ADDRESS + - > + docker run --name context -d -p 1010:1010 + --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/context:$IMAGE_TAG + - docker ps -a + - CONTEXT_ADDRESS=$(docker inspect context --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}") + - echo $CONTEXT_ADDRESS + - > + docker run --name $IMAGE_NAME -d -p 30040:30040 + --env "CONTEXTSERVICE_SERVICE_HOST=${CONTEXT_ADDRESS}" + --env "CONTEXTSERVICE_SERVICE_PORT_GRPC=1010" + --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results" + --network=teraflowbridge + $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG + - docker ps -a + - sleep 5 + - docker logs $IMAGE_NAME + - > + docker exec -i $IMAGE_NAME bash -c + "coverage run -m pytest --log-level=INFO --verbose --junitxml=/opt/results/${IMAGE_NAME}_report.xml $IMAGE_NAME/tests/test_*.py" + - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing" + coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/' + after_script: + - docker rm -f $IMAGE_NAME context crdb + - docker volume rm -f crdb + - docker network rm teraflowbridge + - docker volume prune --force + - docker image prune --force + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/common/**/*.py + - proto/*.proto + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml + artifacts: + when: always + reports: + junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml -- GitLab From fd8fe619b71f90f8d622e12d2237775195d9ade7 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 30 Oct 2025 05:54:56 +0000 Subject: [PATCH 05/15] refactor: Pluggable component - Removed DSCM form pluggable ServiceEnumName - Other minor changes --- proto/pluggables.proto | 18 ++++++++- src/common/Constants.py | 4 +- src/device/service/drivers/__init__.py | 26 ++++++------- src/pluggables/README.md | 38 +++++++++---------- src/pluggables/client/PluggablesClient.py | 4 +- src/pluggables/requirements.in | 14 +++++++ src/pluggables/service/PluggablesService.py | 2 +- .../tests/PreparePluggablesTestScenario.py | 6 +-- src/pluggables/tests/test_Pluggables.py | 6 +-- 9 files changed, 73 insertions(+), 45 deletions(-) diff --git a/proto/pluggables.proto b/proto/pluggables.proto index d6abe9daf..8036a24be 100644 --- a/proto/pluggables.proto +++ b/proto/pluggables.proto @@ -1,6 +1,21 @@ +// Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + syntax = "proto3"; -package tfs.pluggables.v0; +package pluggables; import "context.proto"; @@ -13,7 +28,6 @@ service PluggablesService { } - message PluggableId { context.DeviceId device = 1; int32 pluggable_index = 2; // physical slot number in the device diff --git a/src/common/Constants.py b/src/common/Constants.py index d00599ac2..e98807c96 100644 --- a/src/common/Constants.py +++ b/src/common/Constants.py @@ -75,7 +75,7 @@ class ServiceNameEnum(Enum): ANALYTICSBACKEND = 'analytics-backend' QOSPROFILE = 'qos-profile' OSMCLIENT = 'osm-client' - DSCMPLUGGABLE = 'dscm-pluggable' + PLUGGABLES = 'dscm-pluggable' # Used for test and debugging only DLT_GATEWAY = 'dltgateway' @@ -118,7 +118,7 @@ DEFAULT_SERVICE_GRPC_PORTS = { ServiceNameEnum.ANALYTICSBACKEND .value : 30090, ServiceNameEnum.AUTOMATION .value : 30200, ServiceNameEnum.OSMCLIENT .value : 30210, - ServiceNameEnum.DSCMPLUGGABLE .value : 30220, + ServiceNameEnum.PLUGGABLES .value : 30220, # Used for test and debugging only ServiceNameEnum.DLT_GATEWAY .value : 50051, diff --git a/src/device/service/drivers/__init__.py b/src/device/service/drivers/__init__.py index 21a0ec111..5768e2310 100644 --- a/src/device/service/drivers/__init__.py +++ b/src/device/service/drivers/__init__.py @@ -98,19 +98,19 @@ if LOAD_ALL_DEVICE_DRIVERS: } ])) -# if LOAD_ALL_DEVICE_DRIVERS: -# from .gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -# DRIVERS.append( -# (GnmiOpenConfigDriver, [ -# { -# # Real Packet Router, specifying gNMI OpenConfig Driver => use GnmiOpenConfigDriver -# FilterFieldEnum.DEVICE_TYPE: [ -# DeviceTypeEnum.PACKET_POP, -# DeviceTypeEnum.PACKET_ROUTER, -# ], -# FilterFieldEnum.DRIVER : DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG, -# } -# ])) +if LOAD_ALL_DEVICE_DRIVERS: + from .gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position + DRIVERS.append( + (GnmiOpenConfigDriver, [ + { + # Real Packet Router, specifying gNMI OpenConfig Driver => use GnmiOpenConfigDriver + FilterFieldEnum.DEVICE_TYPE: [ + DeviceTypeEnum.PACKET_POP, + DeviceTypeEnum.PACKET_ROUTER, + ], + FilterFieldEnum.DRIVER : DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG, + } + ])) if LOAD_ALL_DEVICE_DRIVERS: from .gnmi_nokia_srlinux.GnmiNokiaSrLinuxDriver import GnmiNokiaSrLinuxDriver # pylint: disable=wrong-import-position diff --git a/src/pluggables/README.md b/src/pluggables/README.md index b76723ecf..c679a8c7c 100644 --- a/src/pluggables/README.md +++ b/src/pluggables/README.md @@ -1,8 +1,8 @@ -# DSCM (Digital Subcarrier Multipelxed) Service +# Pluggables Service (Digital Subcarrier Multiplexed) ## Overview -The DSCM service provides gRPC-based management for optical pluggables and their digital subcarrier groups. It enables configuration and monitoring of coherent optical transceivers with support for multi-carrier operation. +The Pluggables service provides gRPC-based management for optical pluggables and their digital subcarrier groups. It enables configuration and monitoring of coherent optical transceivers with support for multi-carrier operation. ## Key Concepts @@ -47,11 +47,11 @@ Control the level of detail in responses: ### 1. Creating a Pluggable Without Configuration ```python -from dscm.client.DscmPluggableClient import DscmPluggableClient -from dscm.tests.testmessages import create_pluggable_request +from pluggables.client.PluggablesClient import PluggablesClient +from pluggables.tests.testmessages import create_pluggable_request # Create client -client = DscmPluggableClient() +client = PluggablesClient() # Create pluggable request (auto-assign index) request = create_pluggable_request( @@ -70,7 +70,7 @@ client.close() ### 2. Creating a Pluggable With Initial Configuration ```python -from dscm.tests.testmessages import create_pluggable_request +from pluggables.tests.testmessages import create_pluggable_request # Create pluggable with optical channel configuration request = create_pluggable_request( @@ -91,8 +91,8 @@ assert dsc_group.group_capacity_gbps == 400.0 ### 3. Listing Pluggables with View Filtering ```python -from common.proto.dscm_pluggable_pb2 import View -from dscm.tests.testmessages import create_list_pluggables_request +from common.proto.pluggables_pb2 import View +from pluggables.tests.testmessages import create_list_pluggables_request # List only configuration (no state data) request = create_list_pluggables_request( @@ -108,7 +108,7 @@ for pluggable in response.pluggables: ### 4. Getting a Specific Pluggable ```python -from dscm.tests.testmessages import create_get_pluggable_request +from pluggables.tests.testmessages import create_get_pluggable_request # Get full pluggable details request = create_get_pluggable_request( @@ -126,7 +126,7 @@ print(f"DSC Groups: {len(pluggable.config.dsc_groups)}") ### 5. Configuring a Pluggable ```python -from dscm.tests.testmessages import create_configure_pluggable_request +from pluggables.tests.testmessages import create_configure_pluggable_request # Apply full configuration (reconfigure optical channels) request = create_configure_pluggable_request( @@ -148,7 +148,7 @@ print(f"Configured {len(pluggable.config.dsc_groups[0].subcarriers)} subcarriers ### 6. Deleting a Pluggable ```python -from dscm.tests.testmessages import create_delete_pluggable_request +from pluggables.tests.testmessages import create_delete_pluggable_request # Delete pluggable from management request = create_delete_pluggable_request( @@ -165,10 +165,10 @@ print("Pluggable deleted successfully") ### Complete Configuration Example ```python -from common.proto import dscm_pluggable_pb2 +from common.proto import pluggables_pb2 # Create configuration request -request = dscm_pluggable_pb2.ConfigurePluggableRequest() +request = pluggables_pb2.ConfigurePluggableRequest() # Set pluggable ID request.config.id.device.device_uuid.uuid = "550e8400-e29b-41d4-a716-446655440000" @@ -199,15 +199,15 @@ subcarrier2 = dsc_group.subcarriers.add() # ... (similar configuration for second subcarrier) # Set view level and timeout -request.view_level = dscm_pluggable_pb2.VIEW_FULL +request.view_level = pluggables_pb2.VIEW_FULL request.apply_timeout_seconds = 30 ``` ## API Reference For complete API documentation, see: -- Protocol Buffer definitions: `/home/ubuntu/tfs-ctrl/proto/dscm_pluggable.proto` -- Client implementation: `/home/ubuntu/tfs-ctrl/src/dscm/client/DscmPluggableClient.py` -- Service implementation: `/home/ubuntu/tfs-ctrl/src/dscm/service/DscmPluggableServiceServicerImpl.py` -- Test examples: `/home/ubuntu/tfs-ctrl/src/dscm/tests/test_DscmPluggables.py` -- Message helpers: `/home/ubuntu/tfs-ctrl/src/dscm/tests/testmessages.py` +- Protocol Buffer definitions: `proto/pluggables.proto` +- Client implementation: `src/pluggables/client/PluggablesClient.py` +- Service implementation: `src/pluggables/service/PluggablesServiceServicerImpl.py` +- Test examples: `src/pluggables/tests/test_Pluggables.py` +- Message helpers: `src/pluggables/tests/testmessages.py` diff --git a/src/pluggables/client/PluggablesClient.py b/src/pluggables/client/PluggablesClient.py index f8e7deadb..559619294 100644 --- a/src/pluggables/client/PluggablesClient.py +++ b/src/pluggables/client/PluggablesClient.py @@ -32,8 +32,8 @@ RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, class PluggablesClient: def __init__(self, host=None, port=None): - if not host: host = get_service_host(ServiceNameEnum.DSCMPLUGGABLE) - if not port: port = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) + if not host: host = get_service_host(ServiceNameEnum.PLUGGABLES) + if not port: port = get_service_port_grpc(ServiceNameEnum.PLUGGABLES) self.endpoint = '{:s}:{:s}'.format(str(host), str(port)) LOGGER.debug('Creating channel to {:s}...'.format(str(self.endpoint))) diff --git a/src/pluggables/requirements.in b/src/pluggables/requirements.in index e69de29bb..3ccc21c7d 100644 --- a/src/pluggables/requirements.in +++ b/src/pluggables/requirements.in @@ -0,0 +1,14 @@ +# Copyright 2022-2025 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/pluggables/service/PluggablesService.py b/src/pluggables/service/PluggablesService.py index 92e9aa6e1..65675ec3c 100644 --- a/src/pluggables/service/PluggablesService.py +++ b/src/pluggables/service/PluggablesService.py @@ -20,7 +20,7 @@ from pluggables.service.PluggablesServiceServicerImpl import PluggablesServiceSe class PluggablesService(GenericGrpcService): def __init__(self, cls_name: str = __name__) -> None: - port = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) + port = get_service_port_grpc(ServiceNameEnum.PLUGGABLES) super().__init__(port, cls_name=cls_name) self.dscmPluggableService_servicer = PluggablesServiceServicerImpl() diff --git a/src/pluggables/tests/PreparePluggablesTestScenario.py b/src/pluggables/tests/PreparePluggablesTestScenario.py index 937f09099..c6fdd62fd 100644 --- a/src/pluggables/tests/PreparePluggablesTestScenario.py +++ b/src/pluggables/tests/PreparePluggablesTestScenario.py @@ -47,14 +47,14 @@ MOCKSERVICE_PORT = 10000 # Configure service endpoints CONTEXT_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.CONTEXT) DEVICE_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.DEVICE) -DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) +DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.PLUGGABLES) os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(CONTEXT_SERVICE_PORT) os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DEVICE_SERVICE_PORT) -os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) +os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) class MockContextService(GenericGrpcService): diff --git a/src/pluggables/tests/test_Pluggables.py b/src/pluggables/tests/test_Pluggables.py index 5e12ad1a3..8528c4330 100644 --- a/src/pluggables/tests/test_Pluggables.py +++ b/src/pluggables/tests/test_Pluggables.py @@ -43,9 +43,9 @@ from pluggables.tests.testmessages import (create_pluggable_request, LOCAL_HOST = '127.0.0.1' -DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.DSCMPLUGGABLE) # type: ignore -os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.DSCMPLUGGABLE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) +DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.PLUGGABLES) # type: ignore +os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_HOST )] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) LOGGER = logging.getLogger(__name__) -- GitLab From 9034dcd37297ac2a3cb275b504d3712189a0b8e4 Mon Sep 17 00:00:00 2001 From: Waleed Akbar Date: Thu, 30 Oct 2025 08:50:45 +0000 Subject: [PATCH 06/15] feat: Minor improvements and added TODOs - Add OperationFailedException for error handling - Improve logging in PluggablesService - Clean up imports in config_translator and testmessages --- .../service/PluggablesServiceServicerImpl.py | 17 +++++++++-------- src/pluggables/service/config_translator.py | 4 ++-- src/pluggables/tests/testmessages.py | 3 +++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pluggables/service/PluggablesServiceServicerImpl.py b/src/pluggables/service/PluggablesServiceServicerImpl.py index 5509b9d66..098c4f9ac 100644 --- a/src/pluggables/service/PluggablesServiceServicerImpl.py +++ b/src/pluggables/service/PluggablesServiceServicerImpl.py @@ -20,7 +20,7 @@ from common.proto.pluggables_pb2 import ( Pluggable, CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, GetPluggableRequest, DeletePluggableRequest, ConfigurePluggableRequest) from common.method_wrappers.ServiceExceptions import ( - NotFoundException, AlreadyExistsException, InvalidArgumentException) + NotFoundException, AlreadyExistsException, InvalidArgumentException, OperationFailedException) from common.tools.object_factory.ConfigRule import json_config_rule_set from context.client.ContextClient import ContextClient from device.client.DeviceClient import DeviceClient @@ -46,7 +46,7 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): """ LOGGER.info(f"Configuring device {device_uuid}, pluggable index {pluggable_index}") - # Step 1: Get the device from Context service (to extract IP address and determine template) + # Step 1: Get the device from Context service (to extract IP address) try: device = self.context_client.GetDevice(DeviceId(device_uuid={'uuid': device_uuid})) # type: ignore LOGGER.info(f"Retrieved device from Context: {device.name}") @@ -73,7 +73,6 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): elif device_address == '10.30.7.8': template_identifier = 'leaf' else: - # Default to hub template if IP address cannot be determined LOGGER.warning(f"Cannot determine device type from IP address {device_address}, defaulting to hub template") raise InvalidArgumentException( 'Device IP address', device_address, extra_details='Unknown device IP adress') @@ -153,7 +152,8 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): LOGGER.info(f"Device {device_uuid} found in Context service: {device.name}") except grpc.RpcError as e: LOGGER.error(f"Device {device_uuid} not found in Context service: {e}") - raise NotFoundException('Device', device_uuid, extra_details='Device must exist before creating pluggable') + raise NotFoundException( + 'Device', device_uuid, extra_details='Device must exist before creating pluggable') # If initial_config is provided, push configuration to device if request.HasField('initial_config') and len(pluggable.config.dsc_groups) > 0: @@ -162,9 +162,8 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): self._push_config_to_device(device_uuid, pluggable_index, pluggable.config) except Exception as e: LOGGER.error(f"Failed to push initial config to device: {e}") - raise - # We still create the pluggable in memory, but log the error - # In production, you might want to raise the exception instead + raise OperationFailedException( + 'Push initial pluggable configuration', extra_details=str(e)) self.pluggables[pluggable_key] = pluggable @@ -214,7 +213,9 @@ class PluggablesServiceServicerImpl(PluggablesServiceServicer): LOGGER.info(f'No matching pluggable found: device={device_uuid}, index={pluggable_index}') raise NotFoundException('Pluggable', pluggable_key) - # Remove pluggable config from device via DSCM driver + # Remove pluggable config from device + # TODO: Verify deletion works with actual hub and leaf devices + # try: pluggable = self.pluggables[pluggable_key] # Create empty config to trigger deletion diff --git a/src/pluggables/service/config_translator.py b/src/pluggables/service/config_translator.py index 3405fbb66..d28e2ae01 100644 --- a/src/pluggables/service/config_translator.py +++ b/src/pluggables/service/config_translator.py @@ -13,9 +13,9 @@ # limitations under the License. import logging -from typing import Dict, Any, List, Optional +from typing import Dict, Any from common.proto.pluggables_pb2 import PluggableConfig -from common.proto.context_pb2 import ConfigRule, ConfigActionEnum +from common.proto.context_pb2 import ConfigRule LOGGER = logging.getLogger(__name__) diff --git a/src/pluggables/tests/testmessages.py b/src/pluggables/tests/testmessages.py index 9a0fb5a05..c7a7fe934 100644 --- a/src/pluggables/tests/testmessages.py +++ b/src/pluggables/tests/testmessages.py @@ -142,6 +142,9 @@ def create_get_pluggable_request( # DeletePluggableRequest ########################### +# TODO: Both leaf and hub have a same jinja template for deleting pluggable config. +# The difference lies in the component name (channel-1 for hub, channel-1/3/5 for leaf). + def create_delete_pluggable_request( device_uuid: str, pluggable_index: int -- GitLab From 332d0fb3733db4176d367c7b243391397311e9fa Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 14:57:40 +0000 Subject: [PATCH 07/15] Pre-merge code cleanup --- src/pluggables/Dockerfile | 10 +-- .../tests/PreparePluggablesTestScenario.py | 35 +++++------ ...{test_Pluggables.py => test_pluggables.py} | 63 ++++++++++--------- .../tests/test_pluggables_with_SBI.py | 1 - 4 files changed, 54 insertions(+), 55 deletions(-) rename src/pluggables/tests/{test_Pluggables.py => test_pluggables.py} (81%) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 22c1c92ae..264562a43 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -54,15 +54,15 @@ RUN rm *.proto RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; # Create component sub-folders, get specific Python packages -RUN mkdir -p /var/teraflow/dscm -WORKDIR /var/teraflow/dscm -COPY src/dscm/requirements.in requirements.in +RUN mkdir -p /var/teraflow/pluggables +WORKDIR /var/teraflow/pluggables +COPY src/pluggables/requirements.in requirements.in RUN pip-compile --quiet --output-file=requirements.txt requirements.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/dscm/. dscm/ +COPY src/pluggables/. pluggables/ # # Start the service -ENTRYPOINT [ "python", "-m", "dscm.service" ] +ENTRYPOINT [ "python", "-m", "pluggables.service" ] diff --git a/src/pluggables/tests/PreparePluggablesTestScenario.py b/src/pluggables/tests/PreparePluggablesTestScenario.py index c6fdd62fd..e70b2975a 100644 --- a/src/pluggables/tests/PreparePluggablesTestScenario.py +++ b/src/pluggables/tests/PreparePluggablesTestScenario.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy, logging, os, pytest +import logging, os, pytest from typing import Union from common.Constants import ServiceNameEnum from common.Settings import ( ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, get_service_port_grpc ) -from common.proto.context_pb2 import Device, DeviceId, Topology, Context +from common.proto.context_pb2 import DeviceId, Topology, Context from common.tools.service.GenericGrpcService import GenericGrpcService from common.tests.MockServicerImpl_Context import MockServicerImpl_Context from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server @@ -37,7 +37,7 @@ from pluggables.tests.CommonObjects import ( CONTEXT_ID, CONTEXT, TOPOLOGY_ID, TOPOLOGY, get_device_hub_with_connect_rules, get_device_leaf_with_connect_rules ) -from common.tools.object_factory.Topology import json_topology + LOGGER = logging.getLogger(__name__) @@ -47,22 +47,22 @@ MOCKSERVICE_PORT = 10000 # Configure service endpoints CONTEXT_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.CONTEXT) DEVICE_SERVICE_PORT = MOCKSERVICE_PORT + get_service_port_grpc(ServiceNameEnum.DEVICE) -DSCMPLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.PLUGGABLES) +PLUGGABLE_SERVICE_PORT = get_service_port_grpc(ServiceNameEnum.PLUGGABLES) -os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(CONTEXT_SERVICE_PORT) -os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DEVICE_SERVICE_PORT) +os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(CONTEXT_SERVICE_PORT) +os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) +os.environ[get_env_var_name(ServiceNameEnum.DEVICE, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DEVICE_SERVICE_PORT) os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_HOST)] = str(LOCAL_HOST) -os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(DSCMPLUGGABLE_SERVICE_PORT) +os.environ[get_env_var_name(ServiceNameEnum.PLUGGABLES, ENVVAR_SUFIX_SERVICE_PORT_GRPC)] = str(PLUGGABLE_SERVICE_PORT) class MockContextService(GenericGrpcService): """Mock Context Service for testing""" - + def __init__(self, bind_port: Union[str, int]) -> None: super().__init__(bind_port, LOCAL_HOST, enable_health_servicer=False, cls_name='MockContextService') - + def install_servicers(self): self.context_servicer = MockServicerImpl_Context() add_ContextServiceServicer_to_server(self.context_servicer, self.server) @@ -139,32 +139,31 @@ def test_prepare_environment( pluggables_client: PluggablesClient, device_service: DeviceService ): # pylint: disable=redefined-outer-name """Prepare test environment by adding devices to Context""" - + LOGGER.info('Preparing test environment...') - + context_client.SetContext(Context(**CONTEXT)) context_client.SetTopology(Topology(**TOPOLOGY)) LOGGER.info('Created admin Context and Topology') - + # Add Hub device with connect rules hub_device = get_device_hub_with_connect_rules() context_client.SetDevice(hub_device) LOGGER.info(f'Added Hub device: {DEVICE_HUB_UUID}') - + # Add Leaf device with connect rules leaf_device = get_device_leaf_with_connect_rules() context_client.SetDevice(leaf_device) LOGGER.info(f'Added Leaf device: {DEVICE_LEAF_UUID}') - + # Verify devices were added hub_device_retrieved = context_client.GetDevice(DeviceId(**DEVICE_HUB_ID)) assert hub_device_retrieved is not None assert hub_device_retrieved.device_id.device_uuid.uuid == DEVICE_HUB_UUID LOGGER.info(f'Verified Hub device: {hub_device_retrieved.name}') - + leaf_device_retrieved = context_client.GetDevice(DeviceId(**DEVICE_LEAF_ID)) assert leaf_device_retrieved is not None assert leaf_device_retrieved.device_id.device_uuid.uuid == DEVICE_LEAF_UUID LOGGER.info(f'Verified Leaf device: {leaf_device_retrieved.name}') - diff --git a/src/pluggables/tests/test_Pluggables.py b/src/pluggables/tests/test_pluggables.py similarity index 81% rename from src/pluggables/tests/test_Pluggables.py rename to src/pluggables/tests/test_pluggables.py index 8528c4330..9bce8c92c 100644 --- a/src/pluggables/tests/test_Pluggables.py +++ b/src/pluggables/tests/test_pluggables.py @@ -25,16 +25,17 @@ from common.Settings import ( get_env_var_name, get_service_port_grpc) from common.tests.MockServicerImpl_Context import MockServicerImpl_Context from common.proto.context_pb2_grpc import add_ContextServiceServicer_to_server - -from common.proto.pluggables_pb2 import (PluggableId, Pluggable, - CreatePluggableRequest, ListPluggablesRequest, ListPluggablesResponse, View) +from common.proto.pluggables_pb2 import ( + PluggableId, Pluggable, ListPluggablesResponse, View +) from common.tools.service.GenericGrpcService import GenericGrpcService - from pluggables.client.PluggablesClient import PluggablesClient from pluggables.service.PluggablesService import PluggablesService -from pluggables.tests.testmessages import (create_pluggable_request, - create_list_pluggables_request, create_get_pluggable_request, - create_delete_pluggable_request, create_configure_pluggable_request) +from pluggables.tests.testmessages import ( + create_pluggable_request, create_list_pluggables_request, + create_get_pluggable_request, create_delete_pluggable_request, + create_configure_pluggable_request, +) ########################### @@ -63,21 +64,21 @@ class MockContextService(GenericGrpcService): # This fixture will be requested by test cases and last during testing session @pytest.fixture(scope='session') def pluggables_service(): - LOGGER.info('Initializing DscmPluggableService...') + LOGGER.info('Initializing PluggableService...') _service = PluggablesService() _service.start() # yield the server, when test finishes, execution will resume to stop it - LOGGER.info('Yielding DscmPluggableService...') + LOGGER.info('Yielding PluggableService...') yield _service - LOGGER.info('Terminating DscmPluggableService...') + LOGGER.info('Terminating PluggableService...') _service.stop() - LOGGER.info('Terminated DscmPluggableService...') + LOGGER.info('Terminated PluggableService...') @pytest.fixture(scope='function') -def dscm_pluggable_client(pluggables_service : PluggablesService): +def pluggable_client(pluggables_service : PluggablesService): LOGGER.info('Creating PluggablesClient...') _client = PluggablesClient() @@ -104,10 +105,10 @@ def log_all_methods(request): ########################### # CreatePluggable Test without configuration -def test_CreatePluggable(dscm_pluggable_client : PluggablesClient): +def test_CreatePluggable(pluggable_client : PluggablesClient): LOGGER.info('Creating Pluggable for test...') _pluggable_request = create_pluggable_request(preferred_pluggable_index=-1) - _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + _pluggable = pluggable_client.CreatePluggable(_pluggable_request) LOGGER.info('Created Pluggable for test: %s', _pluggable) assert isinstance(_pluggable, Pluggable) assert _pluggable.id.pluggable_index == _pluggable_request.preferred_pluggable_index @@ -115,13 +116,13 @@ def test_CreatePluggable(dscm_pluggable_client : PluggablesClient): # CreatePluggable Test with configuration -def test_CreatePluggable_with_config(dscm_pluggable_client : PluggablesClient): +def test_CreatePluggable_with_config(pluggable_client : PluggablesClient): LOGGER.info('Creating Pluggable with initial configuration for test...') _pluggable_request = create_pluggable_request( device_uuid = "9bbf1937-db9e-45bc-b2c6-3214a9d42157", preferred_pluggable_index = -1, with_initial_config = True) - _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + _pluggable = pluggable_client.CreatePluggable(_pluggable_request) LOGGER.info('Created Pluggable with initial configuration for test: %s', _pluggable) assert isinstance(_pluggable, Pluggable) assert _pluggable.id.pluggable_index == _pluggable_request.preferred_pluggable_index @@ -133,26 +134,26 @@ def test_CreatePluggable_with_config(dscm_pluggable_client : PluggablesClient): assert len(dsc_group.subcarriers) == 2 # create pluggable request with pluggable key already exists error -def test_CreatePluggable_already_exists(dscm_pluggable_client : PluggablesClient): +def test_CreatePluggable_already_exists(pluggable_client : PluggablesClient): LOGGER.info('Creating Pluggable for test...') _pluggable_request = create_pluggable_request(preferred_pluggable_index=5) - _pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + _pluggable = pluggable_client.CreatePluggable(_pluggable_request) LOGGER.info('Created Pluggable for test: %s', _pluggable) assert isinstance(_pluggable, Pluggable) assert _pluggable.id.pluggable_index == _pluggable_request.preferred_pluggable_index assert _pluggable.id.device.device_uuid.uuid == _pluggable_request.device.device_uuid.uuid # Try to create the same pluggable again, should raise ALREADY_EXISTS with pytest.raises(grpc.RpcError) as e: - dscm_pluggable_client.CreatePluggable(_pluggable_request) + pluggable_client.CreatePluggable(_pluggable_request) assert e.value.code() == grpc.StatusCode.ALREADY_EXISTS # ListPluggables Test -def test_ListPluggables(dscm_pluggable_client : PluggablesClient): +def test_ListPluggables(pluggable_client : PluggablesClient): LOGGER.info('Listing Pluggables for test...') _list_request = create_list_pluggables_request( view_level = View.VIEW_CONFIG # View.VIEW_STATE ) - _pluggables = dscm_pluggable_client.ListPluggables(_list_request) + _pluggables = pluggable_client.ListPluggables(_list_request) LOGGER.info('Listed Pluggables for test: %s', _pluggables) assert isinstance(_pluggables, ListPluggablesResponse) if len(_pluggables.pluggables) != 0: @@ -164,12 +165,12 @@ def test_ListPluggables(dscm_pluggable_client : PluggablesClient): assert len(_pluggables.pluggables) == 0 # GetPluggable Test -def test_GetPluggable(dscm_pluggable_client : PluggablesClient): +def test_GetPluggable(pluggable_client : PluggablesClient): LOGGER.info('Starting GetPluggable test...') LOGGER.info('Getting Pluggable for test...') # First create a pluggable to get it later _pluggable_request = create_pluggable_request(preferred_pluggable_index=1) - _created_pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + _created_pluggable = pluggable_client.CreatePluggable(_pluggable_request) LOGGER.info('Created Pluggable for GetPluggable test: %s', _created_pluggable) _get_request = create_get_pluggable_request( @@ -177,7 +178,7 @@ def test_GetPluggable(dscm_pluggable_client : PluggablesClient): pluggable_index = _created_pluggable.id.pluggable_index, view_level = View.VIEW_FULL ) - _pluggable = dscm_pluggable_client.GetPluggable(_get_request) + _pluggable = pluggable_client.GetPluggable(_get_request) LOGGER.info('Got Pluggable for test: %s', _pluggable) assert isinstance(_pluggable, Pluggable) assert _pluggable.id.pluggable_index == _created_pluggable.id.pluggable_index @@ -185,26 +186,26 @@ def test_GetPluggable(dscm_pluggable_client : PluggablesClient): # DeletePluggable Test -def test_DeletePluggable(dscm_pluggable_client : PluggablesClient): +def test_DeletePluggable(pluggable_client : PluggablesClient): LOGGER.info('Starting DeletePluggable test...') LOGGER.info('Creating Pluggable to delete for test...') # First create a pluggable to delete it later _pluggable_request = create_pluggable_request(preferred_pluggable_index=2) - _created_pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + _created_pluggable = pluggable_client.CreatePluggable(_pluggable_request) LOGGER.info('Created Pluggable to delete for test: %s', _created_pluggable) _delete_request = create_delete_pluggable_request( device_uuid = _created_pluggable.id.device.device_uuid.uuid, pluggable_index = _created_pluggable.id.pluggable_index ) - _response = dscm_pluggable_client.DeletePluggable(_delete_request) + _response = pluggable_client.DeletePluggable(_delete_request) LOGGER.info('Deleted Pluggable for test, response: %s', _response) assert isinstance(_response, Empty) # Try to get the deleted pluggable, should raise NOT_FOUND with pytest.raises(grpc.RpcError) as e: - dscm_pluggable_client.GetPluggable( + pluggable_client.GetPluggable( create_get_pluggable_request( device_uuid = _created_pluggable.id.device.device_uuid.uuid, pluggable_index = _created_pluggable.id.pluggable_index @@ -213,20 +214,20 @@ def test_DeletePluggable(dscm_pluggable_client : PluggablesClient): assert e.value.code() == grpc.StatusCode.NOT_FOUND # ConfigurePluggable Test -def test_ConfigurePluggable(dscm_pluggable_client : PluggablesClient): +def test_ConfigurePluggable(pluggable_client : PluggablesClient): LOGGER.info('Starting ConfigurePluggable test...') LOGGER.info('Creating Pluggable to configure for test...') # First create a pluggable to configure it later _pluggable_request = create_pluggable_request(preferred_pluggable_index=3) - _created_pluggable = dscm_pluggable_client.CreatePluggable(_pluggable_request) + _created_pluggable = pluggable_client.CreatePluggable(_pluggable_request) LOGGER.info('Created Pluggable to configure for test: %s', _created_pluggable) _configure_request = create_configure_pluggable_request( device_uuid = _created_pluggable.id.device.device_uuid.uuid, pluggable_index = _created_pluggable.id.pluggable_index, ) - _pluggable = dscm_pluggable_client.ConfigurePluggable(_configure_request) + _pluggable = pluggable_client.ConfigurePluggable(_configure_request) LOGGER.info('Configured Pluggable for test: %s', _pluggable) assert isinstance(_pluggable, Pluggable) assert _pluggable.config is not None diff --git a/src/pluggables/tests/test_pluggables_with_SBI.py b/src/pluggables/tests/test_pluggables_with_SBI.py index 7e8e56037..d93142942 100644 --- a/src/pluggables/tests/test_pluggables_with_SBI.py +++ b/src/pluggables/tests/test_pluggables_with_SBI.py @@ -16,7 +16,6 @@ import grpc, logging, pytest from common.proto.context_pb2 import Empty from common.proto.pluggables_pb2 import Pluggable, View from context.client.ContextClient import ContextClient -from device.client.DeviceClient import DeviceClient from pluggables.client.PluggablesClient import PluggablesClient from pluggables.tests.testmessages import ( create_pluggable_request, create_list_pluggables_request, -- GitLab From 96c22bcc0a0fac3a7e83339aa42b105c550fb059 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 14:59:23 +0000 Subject: [PATCH 08/15] Pluggables component: - Fixed Python dependency versions --- src/pluggables/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 264562a43..36a78ab5e 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -28,9 +28,9 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ chmod +x /bin/grpc_health_probe # Get generic Python packages -RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install --upgrade setuptools wheel -RUN python3 -m pip install --upgrade pip-tools +RUN python3 -m pip install --upgrade 'pip==25.2' +RUN python3 -m pip install --upgrade 'setuptools==79.0.0' 'wheel==0.45.1' +RUN python3 -m pip install --upgrade 'pip-tools==7.3.0' # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components -- GitLab From eda18c26ca7ba258edbedbcabf2d98daaa1cf2a3 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 15:07:16 +0000 Subject: [PATCH 09/15] Context and Service component: - Fix Python dependenciy versions --- src/context/Dockerfile | 6 +++--- src/service/Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/context/Dockerfile b/src/context/Dockerfile index add63fe65..dc08840cc 100644 --- a/src/context/Dockerfile +++ b/src/context/Dockerfile @@ -28,9 +28,9 @@ ENV PYTHONUNBUFFERED=0 # chmod +x /bin/grpc_health_probe # Get generic Python packages -RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install --upgrade setuptools wheel -RUN python3 -m pip install --upgrade pip-tools +RUN python3 -m pip install --upgrade 'pip==25.2' +RUN python3 -m pip install --upgrade 'setuptools==79.0.0' 'wheel==0.45.1' +RUN python3 -m pip install --upgrade 'pip-tools==7.3.0' # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 493094769..d0315a3cb 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -28,9 +28,9 @@ ENV PYTHONUNBUFFERED=0 # chmod +x /bin/grpc_health_probe # Get generic Python packages -RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install --upgrade setuptools wheel -RUN python3 -m pip install --upgrade pip-tools +RUN python3 -m pip install --upgrade 'pip==25.2' +RUN python3 -m pip install --upgrade 'setuptools==79.0.0' 'wheel==0.45.1' +RUN python3 -m pip install --upgrade 'pip-tools==7.3.0' # Get common Python packages # Note: this step enables sharing the previous Docker build steps among all the Python components -- GitLab From 44f8098e3a855b4cc4ef6b715a0a850c16a635a1 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 15:26:10 +0000 Subject: [PATCH 10/15] Added Pluggables component to CI/CD pipeline --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2856f9fed..aa5009dc8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,6 +58,7 @@ include: - local: '/src/ztp_server/.gitlab-ci.yml' - local: '/src/osm_client/.gitlab-ci.yml' - local: '/src/simap_connector/.gitlab-ci.yml' + - local: '/src/pluggables/.gitlab-ci.yml' # This should be last one: end-to-end integration tests - local: '/src/tests/.gitlab-ci.yml' -- GitLab From 272dcada4a2b3d7bade94d3de4c2c8abae63fd02 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 15:54:00 +0000 Subject: [PATCH 11/15] Pre-merge code cleanup --- src/pluggables/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 36a78ab5e..7d9006f93 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -62,7 +62,11 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/device/__init__.py device/__init__.py +COPY src/device/client/. device/client/ COPY src/pluggables/. pluggables/ -# # Start the service -ENTRYPOINT [ "python", "-m", "pluggables.service" ] +# Start the service +ENTRYPOINT ["python", "-m", "pluggables.service"] -- GitLab From d299bc7fef39ce052aecdacebf6161b4d46c2f34 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 15:58:49 +0000 Subject: [PATCH 12/15] Pre-merge code cleanup --- src/pluggables/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 7d9006f93..8345ebdad 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -66,6 +66,7 @@ COPY src/context/__init__.py context/__init__.py COPY src/context/client/. context/client/ COPY src/device/__init__.py device/__init__.py COPY src/device/client/. device/client/ +COPY src/device/server/. device/server/ COPY src/pluggables/. pluggables/ # Start the service -- GitLab From b91d59e03f08fdd2035305b9fbd92893866d559e Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 16:01:21 +0000 Subject: [PATCH 13/15] Pre-merge code cleanup --- src/pluggables/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 8345ebdad..2f2f4bb07 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -66,7 +66,7 @@ COPY src/context/__init__.py context/__init__.py COPY src/context/client/. context/client/ COPY src/device/__init__.py device/__init__.py COPY src/device/client/. device/client/ -COPY src/device/server/. device/server/ +COPY src/device/service/. device/service/ COPY src/pluggables/. pluggables/ # Start the service -- GitLab From 93c60ebb4223b7bf456a909a7975be76063663d7 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 16:06:02 +0000 Subject: [PATCH 14/15] Pre-merge code cleanup --- src/pluggables/Dockerfile | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 2f2f4bb07..3b8810c1c 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -16,9 +16,24 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ git && \ + apt-get --yes --quiet --quiet install wget g++ git build-essential cmake libpcre2-dev python3-dev python3-cffi && \ rm -rf /var/lib/apt/lists/* +# Download, build and install libyang. Note that APT package is outdated +# - Ref: https://github.com/CESNET/libyang +# - Ref: https://github.com/CESNET/libyang-python/ +RUN mkdir -p /var/libyang +RUN git clone https://github.com/CESNET/libyang.git /var/libyang +WORKDIR /var/libyang +RUN git fetch +RUN git checkout v2.1.148 +RUN mkdir -p /var/libyang/build +WORKDIR /var/libyang/build +RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. +RUN make +RUN make install +RUN ldconfig + # Set Python to show logs as they occur ENV PYTHONUNBUFFERED=0 @@ -56,8 +71,9 @@ RUN find . -type f -exec sed -i -E 's/^(import\ .*)_pb2/from . \1_pb2/g' {} \; # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/pluggables WORKDIR /var/teraflow/pluggables -COPY src/pluggables/requirements.in requirements.in -RUN pip-compile --quiet --output-file=requirements.txt requirements.in +COPY src/device/requirements.in requirements_device.in +COPY src/pluggables/requirements.in requirements_pluggables.in +RUN pip-compile --quiet --output-file=requirements.txt requirements_device.in requirements_pluggables.in RUN python3 -m pip install -r requirements.txt # Add component files into working directory @@ -65,6 +81,7 @@ WORKDIR /var/teraflow COPY src/context/__init__.py context/__init__.py COPY src/context/client/. context/client/ COPY src/device/__init__.py device/__init__.py +COPY src/device/Config.py device/Config.py COPY src/device/client/. device/client/ COPY src/device/service/. device/service/ COPY src/pluggables/. pluggables/ -- GitLab From 19778c2ce9217eeb26113c70e2cad2f55f9f3a0b Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 4 Nov 2025 16:12:37 +0000 Subject: [PATCH 15/15] Pre-merge code cleanup --- src/pluggables/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pluggables/Dockerfile b/src/pluggables/Dockerfile index 3b8810c1c..de4d7aa62 100644 --- a/src/pluggables/Dockerfile +++ b/src/pluggables/Dockerfile @@ -84,6 +84,8 @@ COPY src/device/__init__.py device/__init__.py COPY src/device/Config.py device/Config.py COPY src/device/client/. device/client/ COPY src/device/service/. device/service/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ COPY src/pluggables/. pluggables/ # Start the service -- GitLab