From 038b74707bb20da7e71961cc9cc220a3b459802c Mon Sep 17 00:00:00 2001 From: armingol Date: Tue, 9 Jun 2026 14:09:46 +0000 Subject: [PATCH 1/3] feat: implement UpdateDeviceInventory functionality across service, client, and web UI layers --- proto/device.proto | 1 + src/device/client/DeviceClient.py | 8 +++ .../service/DeviceServiceServicerImpl.py | 51 ++++++++++++++++++- src/webui/service/device/routes.py | 15 +++++- .../service/templates/device/detail.html | 6 +++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/proto/device.proto b/proto/device.proto index 005521abf..40803e8e2 100644 --- a/proto/device.proto +++ b/proto/device.proto @@ -25,6 +25,7 @@ service DeviceService { rpc GetInitialConfig (context.DeviceId ) returns (context.DeviceConfig ) {} rpc MonitorDeviceKpi (MonitoringSettings ) returns (context.Empty ) {} rpc SSETelemetrySubscribe(monitoring.SSEMonitoringSubscriptionConfig) returns (monitoring.SSEMonitoringSubscriptionResponse ) {} + rpc UpdateDeviceInventory(context.DeviceId ) returns (context.Empty ) {} } message MonitoringSettings { diff --git a/src/device/client/DeviceClient.py b/src/device/client/DeviceClient.py index b15589ac0..549e9a5a3 100644 --- a/src/device/client/DeviceClient.py +++ b/src/device/client/DeviceClient.py @@ -113,3 +113,11 @@ class DeviceClient: response = self.stub.SSETelemetrySubscribe(request) LOGGER.debug('SSETelemetrySubscribe result: {:s}'.format(grpc_message_to_json_string(response))) return response + + @RETRY_DECORATOR + def UpdateDeviceInventory(self, request : DeviceId) -> Empty: + LOGGER.debug('UpdateDeviceInventory request: {:s}'.format(grpc_message_to_json_string(request))) + response = self.stub.UpdateDeviceInventory(request) + LOGGER.debug('UpdateDeviceInventory result: {:s}'.format(grpc_message_to_json_string(response))) + return response + diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index 13bcc18e5..b54451eed 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -21,7 +21,7 @@ from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, safe_a from common.method_wrappers.ServiceExceptions import NotFoundException, OperationFailedException from common.proto.context_pb2 import ( Device, DeviceConfig, DeviceDriverEnum, DeviceId, DeviceOperationalStatusEnum, Empty, Link, - OpticalConfig, OpticalConfigId + OpticalConfig, OpticalConfigId, ConfigActionEnum ) from common.proto.device_pb2 import MonitoringSettings from common.proto.device_pb2_grpc import DeviceServiceServicer @@ -37,7 +37,7 @@ from .Tools import ( check_connect_rules, check_no_endpoints, compute_rules_to_add_delete, configure_rules, deconfigure_rules, get_device_controller_uuid, populate_config_rules, populate_endpoint_monitoring_resources, populate_endpoints, populate_initial_config_rules, - subscribe_kpi, unsubscribe_kpi, update_endpoints + subscribe_kpi, unsubscribe_kpi, update_endpoints, _raw_config_rules_to_grpc ) LOGGER = logging.getLogger(__name__) @@ -441,3 +441,50 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): ) return SSEMonitoringSubscriptionResponse() + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def UpdateDeviceInventory(self, request : DeviceId, context : grpc.ServicerContext) -> Empty: + device_uuid = request.device_uuid.uuid + self.mutex_queues.wait_my_turn(device_uuid) + try: + context_client = ContextClient() + device = get_device( + context_client, device_uuid, rw_copy=True, include_endpoints=False, include_components=False, + include_config_rules=True) + if device is None: + raise NotFoundException('Device', device_uuid) + + driver : _Driver = get_driver(self.driver_instance_cache, device) + if driver is None: + msg = ERROR_MISSING_DRIVER.format(device_uuid=str(device_uuid)) + raise OperationFailedException('UpdateDeviceInventory', extra_details=msg) + + from device.service.driver_api._Driver import RESOURCE_INVENTORY + results_getconfig = driver.GetConfig([RESOURCE_INVENTORY]) + + # Filter out existing /inventory rules to prevent duplicates or leftovers + new_rules = [] + for rule in device.device_config.config_rules: + if rule.WhichOneof('config_rule') == 'custom': + if '/inventory' in rule.custom.resource_key: + continue + new_rules.append(rule) + del device.device_config.config_rules[:] + device.device_config.config_rules.extend(new_rules) + + # Convert new inventory rules and update device config + errors = _raw_config_rules_to_grpc( + device_uuid, device.device_config, 'Error getting inventory resource {resource_key} on device {device_uuid}: {error}', + ConfigActionEnum.CONFIGACTION_SET, results_getconfig + ) + + if len(errors) > 0: + raise OperationFailedException('UpdateDeviceInventory', extra_details=errors) + + context_client.SetDevice(device) + context_client.close() + return Empty() + except Exception as e: + LOGGER.exception('Error updating inventory of device {:s}'.format(str(device_uuid))) + raise e + + diff --git a/src/webui/service/device/routes.py b/src/webui/service/device/routes.py index 23a19278c..e1cfb000a 100644 --- a/src/webui/service/device/routes.py +++ b/src/webui/service/device/routes.py @@ -180,7 +180,20 @@ def detail(device_uuid: str): return render_template( 'device/detail.html', device=device_obj, dde=DeviceDriverEnum, dose=DeviceOperationalStatusEnum) - + +@device.route('detail//update_inventory', methods=['GET', 'POST']) +def update_inventory(device_uuid: str): + try: + device_id = DeviceId() + device_id.device_uuid.uuid = device_uuid + device_client.connect() + device_client.UpdateDeviceInventory(device_id) + device_client.close() + flash(f'Inventory of device "{device_uuid}" was successfully updated.', 'success') + except Exception as e: + flash(f'Problem updating inventory of device "{device_uuid}": {str(e)}', 'danger') + return redirect(url_for('device.detail', device_uuid=device_uuid)) + @device.route('inventory/', methods=['GET', 'POST']) def inventory(device_uuid: str): context_client.connect() diff --git a/src/webui/service/templates/device/detail.html b/src/webui/service/templates/device/detail.html index 75952ba1d..c2b633f01 100644 --- a/src/webui/service/templates/device/detail.html +++ b/src/webui/service/templates/device/detail.html @@ -39,6 +39,12 @@ Delete device +
+ + + Update Inventory + +

-- GitLab From 724ddead1169e798ab42a5c4741b5eb1b33d2758 Mon Sep 17 00:00:00 2001 From: armingol Date: Wed, 10 Jun 2026 07:05:37 +0000 Subject: [PATCH 2/3] fix: clean up device components and inventory config rules before updates and refresh endpoints during inventory sync --- src/context/service/database/Device.py | 11 +++++++++++ src/device/service/DeviceServiceServicerImpl.py | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index bff03a0a5..89bb63d6c 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -31,6 +31,7 @@ from .models.DeviceModel import DeviceModel from .models.EndPointModel import EndPointModel from .models.ComponentModel import ComponentModel from .models.TopologyModel import TopologyDeviceModel, TopologyModel +from .models.ConfigRuleModel import DeviceConfigRuleModel, ConfigRuleKindEnum from .models.enums.DeviceDriver import grpc_to_enum__device_driver from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type @@ -243,6 +244,9 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi device_topology_ids = [obj.dump_id() for obj in device_topologies] #LOGGER.warning('device_topology_ids={:s}'.format(str(device_topology_ids))) + # Delete old components of the device to make sure they are fully updated + session.query(ComponentModel).filter_by(device_uuid=device_uuid).delete() + updated_components = False if len(components_data) > 0: @@ -261,6 +265,13 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi component_updates = session.execute(stmt).fetchall() updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates]) + # Delete old /inventory config rules of the device to make sure they are fully updated + session.query(DeviceConfigRuleModel).filter( + DeviceConfigRuleModel.device_uuid == device_uuid, + DeviceConfigRuleModel.kind == ConfigRuleKindEnum.CUSTOM, + DeviceConfigRuleModel.data.contains('/inventory') + ).delete(synchronize_session=False) + changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid) return updated or updated_endpoints or updated_components or changed_config_rules, device_topology_ids diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index b54451eed..4bb33a386 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -447,9 +447,7 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): self.mutex_queues.wait_my_turn(device_uuid) try: context_client = ContextClient() - device = get_device( - context_client, device_uuid, rw_copy=True, include_endpoints=False, include_components=False, - include_config_rules=True) + device = get_device(context_client, device_uuid, rw_copy=True) if device is None: raise NotFoundException('Device', device_uuid) @@ -458,6 +456,19 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): msg = ERROR_MISSING_DRIVER.format(device_uuid=str(device_uuid)) raise OperationFailedException('UpdateDeviceInventory', extra_details=msg) + # Clear and populate endpoints again from the driver + del device.device_endpoints[:] + new_sub_devices = dict() + new_sub_links = dict() + sorted_sub_device_uuids = list() + new_optical_configs = dict() + errors = populate_endpoints( + device, driver, self.monitoring_loops, new_sub_devices, sorted_sub_device_uuids, + new_sub_links, new_optical_configs + ) + if len(errors) > 0: + raise OperationFailedException('UpdateDeviceInventory', extra_details=errors) + from device.service.driver_api._Driver import RESOURCE_INVENTORY results_getconfig = driver.GetConfig([RESOURCE_INVENTORY]) -- GitLab From 36c368e7a84ccc9bbaf628a1ef1e14a870bc28d5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 16 Jun 2026 11:10:18 +0000 Subject: [PATCH 3/3] CI end-to-end tests - ECOC22 / OFC22 / OFC24 / OFC25: - Removed slice from CI pipeline descriptor --- src/tests/ecoc22/.gitlab-ci.yml | 2 -- src/tests/ofc22/.gitlab-ci.yml | 2 -- src/tests/ofc24/.gitlab-ci.yml | 1 - src/tests/ofc25/.gitlab-ci.yml | 1 - src/tests/ofc25/deploy_all_in_one.sh | 1 - 5 files changed, 7 deletions(-) diff --git a/src/tests/ecoc22/.gitlab-ci.yml b/src/tests/ecoc22/.gitlab-ci.yml index 176301630..00b69e918 100644 --- a/src/tests/ecoc22/.gitlab-ci.yml +++ b/src/tests/ecoc22/.gitlab-ci.yml @@ -126,7 +126,6 @@ end2end_test ecoc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml - #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh @@ -163,7 +162,6 @@ end2end_test ecoc22: - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/deviceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/pathcompservice -c frontend - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/serviceservice -c server - - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/sliceservice -c server - kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/nbiservice -c server # Clean up diff --git a/src/tests/ofc22/.gitlab-ci.yml b/src/tests/ofc22/.gitlab-ci.yml index 4bd6ada0b..10e8957ce 100644 --- a/src/tests/ofc22/.gitlab-ci.yml +++ b/src/tests/ofc22/.gitlab-ci.yml @@ -126,7 +126,6 @@ end2end_test ofc22: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml - #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/monitoringservice.yaml @@ -166,7 +165,6 @@ end2end_test ofc22: - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/deviceservice -c server > src/tests/${TEST_NAME}/component_logs/deviceservice.log 2>&1 || true - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/pathcompservice -c frontend > src/tests/${TEST_NAME}/component_logs/pathcompservice-frontend.log 2>&1 || true - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/serviceservice -c server > src/tests/${TEST_NAME}/component_logs/serviceservice.log 2>&1 || true - - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/sliceservice -c server > src/tests/${TEST_NAME}/component_logs/sliceservice.log 2>&1 || true - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/nbiservice -c server > src/tests/${TEST_NAME}/component_logs/nbiservice.log 2>&1 || true - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/monitoringservice -c server > src/tests/${TEST_NAME}/component_logs/monitoringservice.log 2>&1 || true - kubectl logs --namespace $TFS_K8S_NAMESPACE deployment/ztpservice -c ztpservice > src/tests/${TEST_NAME}/component_logs/ztpservice.log 2>&1 || true diff --git a/src/tests/ofc24/.gitlab-ci.yml b/src/tests/ofc24/.gitlab-ci.yml index c32f0900d..6629416b5 100644 --- a/src/tests/ofc24/.gitlab-ci.yml +++ b/src/tests/ofc24/.gitlab-ci.yml @@ -165,7 +165,6 @@ end2end_test ofc24: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml - #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml - source src/tests/${TEST_NAME}/deploy_specs.sh diff --git a/src/tests/ofc25/.gitlab-ci.yml b/src/tests/ofc25/.gitlab-ci.yml index b05ffae4d..adf22d0d7 100644 --- a/src/tests/ofc25/.gitlab-ci.yml +++ b/src/tests/ofc25/.gitlab-ci.yml @@ -170,7 +170,6 @@ end2end_test ofc25: #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml - #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/opticalcontrollerservice.yaml #- yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/e2e_orchestratorservice.yaml diff --git a/src/tests/ofc25/deploy_all_in_one.sh b/src/tests/ofc25/deploy_all_in_one.sh index bf37a34fe..d41b085a4 100755 --- a/src/tests/ofc25/deploy_all_in_one.sh +++ b/src/tests/ofc25/deploy_all_in_one.sh @@ -41,7 +41,6 @@ kubectl get pods --all-namespaces #yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/deviceservice.yaml #yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="frontend").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/pathcompservice.yaml #yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/serviceservice.yaml -#yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/sliceservice.yaml #yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/nbiservice.yaml #yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/e2eorchestratorservice.yaml #yq -i '((select(.kind=="Deployment").spec.template.spec.containers.[] | select(.name=="server").env.[]) | select(.name=="LOG_LEVEL").value) |= "DEBUG"' manifests/vntmservice.yaml -- GitLab