From 48269f024e6afb001f33328d540a403b9c3a9e0e Mon Sep 17 00:00:00 2001 From: armingol Date: Tue, 14 Mar 2023 09:24:56 +0100 Subject: [PATCH 01/13] NBI Inventory --- src/device/service/driver_api/_Driver.py | 1 + .../drivers/openconfig/templates/Inventory.py | 86 +++++++++++++++++++ .../drivers/openconfig/templates/Namespace.py | 2 + .../drivers/openconfig/templates/__init__.py | 6 +- .../openconfig/templates/inventory/get.xml | 3 + 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 src/device/service/drivers/openconfig/templates/Inventory.py create mode 100644 src/device/service/drivers/openconfig/templates/inventory/get.xml diff --git a/src/device/service/driver_api/_Driver.py b/src/device/service/driver_api/_Driver.py index cc9f7a2c6..8f9c61759 100644 --- a/src/device/service/driver_api/_Driver.py +++ b/src/device/service/driver_api/_Driver.py @@ -23,6 +23,7 @@ RESOURCE_INTERFACES = '__interfaces__' RESOURCE_NETWORK_INSTANCES = '__network_instances__' RESOURCE_ROUTING_POLICIES = '__routing_policies__' RESOURCE_ACL = '__acl__' +RESOURCE_INVENTORY = '__inventory__' class _Driver: diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py new file mode 100644 index 000000000..78744a76d --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -0,0 +1,86 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, lxml.etree as ET +from typing import Any, Dict, List, Tuple +from .Namespace import NAMESPACES +from .Tools import add_value_from_tag + +LOGGER = logging.getLogger(__name__) + +XPATH_PORTS = "//ocp:components/ocp:component" + +def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: + response = [] + LOGGER.info("InventoryPrueba") + + for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): + #LOGGER.info('xml_component = {:s}'.format(str(ET.tostring(xml_component)))) + inventory = {} + inventory['name'] = '' + inventory['type'] = '' + inventory['datos'] = {} + component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) + if component_name is None or component_name.text is None: continue + add_value_from_tag(inventory, 'name', component_name) + + component_type = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) + component_type.text = component_type.text.replace('oc-platform-types:','') + if component_type is None: continue + add_value_from_tag(inventory, 'type', component_type) + + component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES) + if not component_present is None: + add_value_from_tag(inventory['datos'], 'present', component_present) + if 'present' in inventory['datos'] and inventory['datos']['present'] == 'NOT_PRESENT': continue + + component_vendor = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:vendor', namespaces=NAMESPACES) + if not component_vendor is None: + add_value_from_tag(inventory['datos'], 'vendor', component_vendor) + component_connector = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:connector-type', namespaces=NAMESPACES) + if not component_connector is None: + component_connector.text = component_connector.text.replace('oc-opt-types:','') + add_value_from_tag(inventory['datos'], 'connector-type', component_connector) + component_serial = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) + if not component_serial is None: + add_value_from_tag(inventory['datos'], 'serial-no', component_serial) + component_form = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:form-factor', namespaces=NAMESPACES) + if not component_form is None: + component_form.text = component_form.text.replace('oc-platform-types:','') + add_value_from_tag(inventory['datos'], 'form-factor', component_form) + component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) + + component_HW = xml_component.find('ocp:state/ocp:hardware-version', namespaces=NAMESPACES) + if not component_HW is None: + add_value_from_tag(inventory['datos'], 'hardware-version', component_HW) + component_SW = xml_component.find('ocp:state/ocp:software-version', namespaces=NAMESPACES) + if not component_SW is None: + add_value_from_tag(inventory['datos'], 'software-version', component_SW) + component_serial = xml_component.find('ocp:state/ocp:serial-no', namespaces=NAMESPACES) + if not component_serial is None: + add_value_from_tag(inventory['datos'], 'serial-no', component_serial) + component_oper_status = xml_component.find('ocp:state/ocp:oper-status', namespaces=NAMESPACES) + if not component_oper_status is None and inventory['type'] != 'CHASSIS' and inventory['type'] != 'CPU': + component_oper_status.text = component_oper_status.text.replace('oc-platform-types:','') + add_value_from_tag(inventory['datos'], 'oper-status', component_oper_status) + component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) + if not component_parent is None and inventory['type'] != 'CHASSIS': + add_value_from_tag(inventory['datos'], 'parent', component_parent) + + if 'type' not in inventory: inventory['type'] = '-' + + #LOGGER.info('Inventoty = {:s}'.format(str(inventory))) + if len(inventory) == 0: continue + response.append(('/inventory[{:s}]'.format(inventory['name']), inventory)) + return response diff --git a/src/device/service/drivers/openconfig/templates/Namespace.py b/src/device/service/drivers/openconfig/templates/Namespace.py index eede86550..b70d5c327 100644 --- a/src/device/service/drivers/openconfig/templates/Namespace.py +++ b/src/device/service/drivers/openconfig/templates/Namespace.py @@ -28,6 +28,7 @@ NAMESPACE_POLICY_TYPES = 'http://openconfig.net/yang/policy-types' NAMESPACE_POLICY_TYPES_2 = 'http://openconfig.net/yang/policy_types' NAMESPACE_ROUTING_POLICY = 'http://openconfig.net/yang/routing-policy' NAMESPACE_VLAN = 'http://openconfig.net/yang/vlan' +NAMESPACE_PLATFORM_TRANSCEIVER = 'http://openconfig.net/yang/platform/transceiver' NAMESPACES = { 'nc' : NAMESPACE_NETCONF, @@ -44,4 +45,5 @@ NAMESPACES = { 'ocpt2': NAMESPACE_POLICY_TYPES_2, 'ocrp' : NAMESPACE_ROUTING_POLICY, 'ocv' : NAMESPACE_VLAN, + 'ocptr' : NAMESPACE_PLATFORM_TRANSCEIVER, } diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index c415bfd25..a2196b973 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -16,14 +16,16 @@ import json, logging, lxml.etree as ET, re from typing import Any, Dict, Optional from jinja2 import Environment, PackageLoader, select_autoescape from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL) + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY) from .EndPoints import parse as parse_endpoints from .Interfaces import parse as parse_interfaces, parse_counters from .NetworkInstances import parse as parse_network_instances from .RoutingPolicy import parse as parse_routing_policy from .Acl import parse as parse_acl +from .Inventory import parse as parse_inventory ALL_RESOURCE_KEYS = [ + RESOURCE_INVENTORY, RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_ROUTING_POLICIES, # routing policies should come before network instances @@ -32,6 +34,7 @@ ALL_RESOURCE_KEYS = [ ] RESOURCE_KEY_MAPPINGS = { + RESOURCE_INVENTORY : 'inventory', RESOURCE_ENDPOINTS : 'component', RESOURCE_INTERFACES : 'interface', RESOURCE_NETWORK_INSTANCES: 'network_instance', @@ -40,6 +43,7 @@ RESOURCE_KEY_MAPPINGS = { } RESOURCE_PARSERS = { + 'inventory' : parse_inventory, 'component' : parse_endpoints, 'interface' : parse_interfaces, 'network_instance': parse_network_instances, diff --git a/src/device/service/drivers/openconfig/templates/inventory/get.xml b/src/device/service/drivers/openconfig/templates/inventory/get.xml new file mode 100644 index 000000000..aa25ed1e3 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/inventory/get.xml @@ -0,0 +1,3 @@ + + + -- GitLab From 3477a539685b431339745f0bb023b4b8b1e61155 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 15 Jun 2023 06:56:20 +0000 Subject: [PATCH 02/13] Changes to manage inventory and device interfaces --- .../openconfig/templates/Interfaces.py | 3 + .../openconfig/templates/Interfaces_mng.py | 100 ++++++++++ .../drivers/openconfig/templates/Inventory.py | 180 ++++++++++++++---- .../drivers/openconfig/templates/__init__.py | 6 +- .../templates/interfaces_mng/get.xml | 3 + 5 files changed, 251 insertions(+), 41 deletions(-) create mode 100644 src/device/service/drivers/openconfig/templates/Interfaces_mng.py create mode 100644 src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml diff --git a/src/device/service/drivers/openconfig/templates/Interfaces.py b/src/device/service/drivers/openconfig/templates/Interfaces.py index fbe5cfd22..a2f6f931a 100644 --- a/src/device/service/drivers/openconfig/templates/Interfaces.py +++ b/src/device/service/drivers/openconfig/templates/Interfaces.py @@ -38,6 +38,9 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: #add_value_from_tag(interface, 'type', interface_type) interface_type = xml_interface.find('oci:config/oci:type', namespaces=NAMESPACES) + if interface_type is None: + interface_type = xml_interface.find('oci:state/oci:type', namespaces=NAMESPACES) + if interface_type is None: continue interface_type.text = interface_type.text.replace('ianaift:','') add_value_from_tag(interface, 'type', interface_type) diff --git a/src/device/service/drivers/openconfig/templates/Interfaces_mng.py b/src/device/service/drivers/openconfig/templates/Interfaces_mng.py new file mode 100644 index 000000000..ced193252 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/Interfaces_mng.py @@ -0,0 +1,100 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +#Method Name: parse + +#Parameters: + + - xml_data: [ET.Element] Represents the XML data to be parsed. + +# Functionality: + +The parse function of the interfaces_mng class has the functionality to parse +an XML document represented by the xml_data parameter and extract specific +information from the XML elements, namely the active interfaces with their name, +type, ipv4 and ipv6 addresses in case they have. + +To generate the template the following steps are performed: + +1) An empty list called response is created to store the results of the analysis. + +2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS +expression. These elements represent management interfaces in the XML document. + +3) For each management interfaces: +A dictionary called interfaces_mng is initialized that will store the information extracted +from the interfaces.The values of the relevant XML elements are extracted and added to +the dictionary. + +#Return: +List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) +with the information extracted from the XML document interfaces is returned. + +""" + + +import logging, lxml.etree as ET +from typing import Any, Dict, List, Tuple +from .Namespace import NAMESPACES +from .Tools import add_value_from_tag + +LOGGER = logging.getLogger(__name__) + +XPATH_PORTS = "//oci:interfaces/oci:interface" +XPATH_SUBINTERFACES = ".//oci:subinterfaces/oci:subinterface" +XPATH_IPV4ADDRESSES = ".//ociip:ipv4/ociip:addresses/ociip:address" +XPATH_IPV6ADDRESSES = ".//ociip:ipv6/ociip:addresses/ociip:address" + +def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: + response = [] + LOGGER.debug("Interfaces_mngPrueba") + + for xml_interface_mng in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): + LOGGER.info('xml_component xml_interfaces_mng = {:s}'.format(str(ET.tostring(xml_interface_mng)))) + interfaces_mng = {} + + interface_enabled = xml_interface_mng.find('oci:config/oci:enabled', namespaces=NAMESPACES) + if interface_enabled == None: + interface_enabled = xml_interface_mng.find('oci:state/oci:enabled', namespaces=NAMESPACES) + if interface_enabled == None: continue + if 'false' in interface_enabled or 'false' in interface_enabled.text: continue + + interface_name = xml_interface_mng.find('oci:name', namespaces=NAMESPACES) + if interface_name is None or interface_name.text is None: continue + add_value_from_tag(interfaces_mng, 'name', interface_name) + + interface_type = xml_interface_mng.find('oci:config/oci:type', namespaces=NAMESPACES) + if interface_type is None: + interface_type = xml_interface_mng.find('oci:state/oci:type', namespaces=NAMESPACES) + if interface_type is None: continue + interface_type.text = interface_type.text.replace('ianaift:','') + add_value_from_tag(interfaces_mng, 'type', interface_type) + + for xml_subinterface in xml_interface_mng.xpath(XPATH_SUBINTERFACES, namespaces=NAMESPACES): + for xml_ipv4_address in xml_subinterface.xpath(XPATH_IPV4ADDRESSES, namespaces=NAMESPACES): + address_ipv4 = xml_ipv4_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES) + if not address_ipv4 is None: + add_value_from_tag(interfaces_mng, 'ipv4', address_ipv4) + + for xml_ipv6_address in xml_subinterface.xpath(XPATH_IPV6ADDRESSES, namespaces=NAMESPACES): + address_ipv6 = xml_ipv6_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES) + if not address_ipv6 is None: + add_value_from_tag(interfaces_mng, 'ipv6', address_ipv6) + if not 'ipv4' in interfaces_mng and not 'ipv6' in interfaces_mng: + if 'ip' in interfaces_mng['type'] or 'Loopback' in interfaces_mng['type']: continue + response.append(('/interfaces_mng/{:s}'.format(interfaces_mng['name']), interfaces_mng)) + + return response diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index 78744a76d..5775ac963 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -12,6 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. + +""" +#Method Name: parse + +#Parameters: + + - xml_data: [ET.Element] Represents the XML data to be parsed. + +# Functionality: + +The parse function of the inventerio class has the functionality to parse +an XML document represented by the xml_data parameter and extract specific +information from the XML elements, namely the relevant characteristics of the +components. + +To generate the template the following steps are performed: + +1) An empty list called response is created to store the results of the analysis. + +2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS +expression. These elements represent components in the XML document. + +3) For each component element: +A dictionary called inventory is initialized that will store the information extracted +from the component.The values of the relevant XML elements are extracted and added to +the dictionary. + +#Return: +List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) +with the information extracted from the XML document components is returned. + +""" + + import logging, lxml.etree as ET from typing import Any, Dict, List, Tuple from .Namespace import NAMESPACES @@ -23,64 +57,130 @@ XPATH_PORTS = "//ocp:components/ocp:component" def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] - LOGGER.info("InventoryPrueba") - + LOGGER.debug("InventoryPrueba") + parent_types = {} for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): - #LOGGER.info('xml_component = {:s}'.format(str(ET.tostring(xml_component)))) + LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) inventory = {} + inventory['parent-component-references'] = '' + inventory['uuid'] = '' inventory['name'] = '' - inventory['type'] = '' - inventory['datos'] = {} + inventory['class'] = '' + inventory['attributes'] = {} + component_reference = [] + component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) if component_name is None or component_name.text is None: continue - add_value_from_tag(inventory, 'name', component_name) + add_value_from_tag(inventory, 'uuid', component_name) + add_value_from_tag(inventory, 'name', component_name) + component_description = xml_component.find('ocp:state/ocp:description', namespaces=NAMESPACES) + if not component_description is None: + add_value_from_tag(inventory['attributes'], 'description', component_description) + #Podría estar presente en lugar del name + component_alias = xml_component.find('ocp:state/ocp:alias', namespaces=NAMESPACES) + if not component_alias is None: + add_value_from_tag(inventory['attributes'], 'alias', component_alias) + + component_location = xml_component.find('ocp:state/ocp:location', namespaces=NAMESPACES) + if not component_location is None: + add_value_from_tag(inventory['attributes'], 'location', component_location) + component_type = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) component_type.text = component_type.text.replace('oc-platform-types:','') if component_type is None: continue - add_value_from_tag(inventory, 'type', component_type) + add_value_from_tag(inventory, 'class', component_type) + if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue + + component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) + if not component_empty is None: + add_value_from_tag(inventory['attributes'], 'contained-child', component_empty) + + #añadir el parent pos + + component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) + if component_parent is None or component_parent.text is None: + add_value_from_tag(inventory, 'parent-component-references', component_type) + else: + add_value_from_tag(inventory, 'parent-component-references', component_parent) + + component_HW = xml_component.find('ocp:state/ocp:hardware-version', namespaces=NAMESPACES) + if not component_HW is None: + add_value_from_tag(inventory['attributes'], 'hardware-rev', component_HW) + + component_firmware_version = xml_component.find('ocp:state/ocp:firmware-version', namespaces=NAMESPACES) + if not component_firmware_version is None: + add_value_from_tag(inventory['attributes'], 'firmware-rev', component_firmware_version) + + component_SW = xml_component.find('ocp:state/ocp:software-version', namespaces=NAMESPACES) + if not component_SW is None: + add_value_from_tag(inventory['attributes'], 'software-rev', component_SW) + + component_serial = xml_component.find('ocp:state/ocp:serial-no', namespaces=NAMESPACES) + if not component_serial is None: + add_value_from_tag(inventory['attributes'], 'serial-num', component_serial) + + component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) + if not component_serial_t is None: + add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t) + + component_mfg_name = xml_component.find('ocp:state/ocp:mfg-name', namespaces=NAMESPACES) + if not component_mfg_name is None: + add_value_from_tag(inventory['attributes'], 'mfg-name', component_mfg_name) + + component_part_no = xml_component.find('ocp:state/ocp:part-no', namespaces=NAMESPACES) + if not component_part_no is None: + add_value_from_tag(inventory['attributes'], 'part-number', component_part_no) + + #modificar es un identificador de seguimiento de activos asignado por el usuario para el componente + component_asset_id = xml_component.find('ocp:state/ocp:asset-id', namespaces=NAMESPACES) + if not component_asset_id is None: + add_value_from_tag(inventory['attributes'], 'asset-id', component_asset_id) - component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES) - if not component_present is None: - add_value_from_tag(inventory['datos'], 'present', component_present) - if 'present' in inventory['datos'] and inventory['datos']['present'] == 'NOT_PRESENT': continue + component_removable = xml_component.find('ocp:state/ocp:removable', namespaces=NAMESPACES) + if not component_removable is None: + add_value_from_tag(inventory['attributes'], 'is-fru', component_removable) + + component_mfg_date = xml_component.find('ocp:state/ocp:mfg-date', namespaces=NAMESPACES) + if not component_mfg_date is None: + add_value_from_tag(inventory['attributes'], 'mfg-date', component_mfg_date) + + #modificar "Este nodo contiene información de identificación sobre el componente" + component_uri = xml_component.find('ocp:state/ocp:uri', namespaces=NAMESPACES) + if not component_uri is None: + add_value_from_tag(inventory['attributes'], 'uri', component_uri) + #Para transceiver + component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES) + if component_present is not None and 'NOT_PRESENT' in component_present.text: continue + component_vendor = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:vendor', namespaces=NAMESPACES) if not component_vendor is None: - add_value_from_tag(inventory['datos'], 'vendor', component_vendor) + add_value_from_tag(inventory['attributes'], 'vendor', component_vendor) component_connector = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:connector-type', namespaces=NAMESPACES) if not component_connector is None: component_connector.text = component_connector.text.replace('oc-opt-types:','') - add_value_from_tag(inventory['datos'], 'connector-type', component_connector) - component_serial = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) - if not component_serial is None: - add_value_from_tag(inventory['datos'], 'serial-no', component_serial) + add_value_from_tag(inventory['attributes'], 'connector-type', component_connector) + component_form = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:form-factor', namespaces=NAMESPACES) if not component_form is None: - component_form.text = component_form.text.replace('oc-platform-types:','') - add_value_from_tag(inventory['datos'], 'form-factor', component_form) - component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) + component_form.text = component_form.text.replace('oc-opt-types:','') + add_value_from_tag(inventory['attributes'], 'form-factor', component_form) + #añadir características del parent references + if inventory['parent-component-references'] not in parent_types: + parent_types[inventory['parent-component-references']] = len(parent_types) + 1 + + component_reference.extend([parent_types[inventory['parent-component-references']]]) - component_HW = xml_component.find('ocp:state/ocp:hardware-version', namespaces=NAMESPACES) - if not component_HW is None: - add_value_from_tag(inventory['datos'], 'hardware-version', component_HW) - component_SW = xml_component.find('ocp:state/ocp:software-version', namespaces=NAMESPACES) - if not component_SW is None: - add_value_from_tag(inventory['datos'], 'software-version', component_SW) - component_serial = xml_component.find('ocp:state/ocp:serial-no', namespaces=NAMESPACES) - if not component_serial is None: - add_value_from_tag(inventory['datos'], 'serial-no', component_serial) - component_oper_status = xml_component.find('ocp:state/ocp:oper-status', namespaces=NAMESPACES) - if not component_oper_status is None and inventory['type'] != 'CHASSIS' and inventory['type'] != 'CPU': - component_oper_status.text = component_oper_status.text.replace('oc-platform-types:','') - add_value_from_tag(inventory['datos'], 'oper-status', component_oper_status) - component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) - if not component_parent is None and inventory['type'] != 'CHASSIS': - add_value_from_tag(inventory['datos'], 'parent', component_parent) - if 'type' not in inventory: inventory['type'] = '-' + + response.append(('/inventory/{:s}'.format(inventory['name']), inventory)) + + for tupla in response: + if inventory['parent-component-references'] in tupla[0]: + component_reference.extend([tupla[1]['class']]) + component_reference.extend([tupla[1]['uuid']]) + + inventory['component-reference'] = component_reference - #LOGGER.info('Inventoty = {:s}'.format(str(inventory))) - if len(inventory) == 0: continue - response.append(('/inventory[{:s}]'.format(inventory['name']), inventory)) - return response + return response \ No newline at end of file diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index a2196b973..fd992b807 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -16,9 +16,10 @@ import json, logging, lxml.etree as ET, re from typing import Any, Dict, Optional from jinja2 import Environment, PackageLoader, select_autoescape from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY) + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_INTERFACES_MNG,RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY) from .EndPoints import parse as parse_endpoints from .Interfaces import parse as parse_interfaces, parse_counters +from .Interfaces_mng import parse as parse_interfaces_mng from .NetworkInstances import parse as parse_network_instances from .RoutingPolicy import parse as parse_routing_policy from .Acl import parse as parse_acl @@ -28,6 +29,7 @@ ALL_RESOURCE_KEYS = [ RESOURCE_INVENTORY, RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, + RESOURCE_INTERFACES_MNG, RESOURCE_ROUTING_POLICIES, # routing policies should come before network instances RESOURCE_NETWORK_INSTANCES, RESOURCE_ACL, @@ -37,6 +39,7 @@ RESOURCE_KEY_MAPPINGS = { RESOURCE_INVENTORY : 'inventory', RESOURCE_ENDPOINTS : 'component', RESOURCE_INTERFACES : 'interface', + RESOURCE_INTERFACES_MNG : 'interfaces_mng', RESOURCE_NETWORK_INSTANCES: 'network_instance', RESOURCE_ROUTING_POLICIES : 'routing_policy', RESOURCE_ACL : 'acl', @@ -46,6 +49,7 @@ RESOURCE_PARSERS = { 'inventory' : parse_inventory, 'component' : parse_endpoints, 'interface' : parse_interfaces, + 'interfaces_mng' : parse_interfaces_mng, 'network_instance': parse_network_instances, 'routing_policy' : parse_routing_policy, 'interfaces/interface/state/counters': parse_counters, diff --git a/src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml b/src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml new file mode 100644 index 000000000..1003bff15 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file -- GitLab From e17e8cc3e1cf2a08e103df87d9a492e69decc8dc Mon Sep 17 00:00:00 2001 From: Armingol Date: Mon, 18 Sep 2023 16:17:17 +0200 Subject: [PATCH 03/13] ddbb modification to store the device inventory --- proto/context.proto | 22 ++- src/context/service/database/Component.py | 164 ++++++++++++++++++ src/context/service/database/ConfigRule.py | 3 + src/context/service/database/Device.py | 25 ++- .../service/database/models/ComponentModel.py | 51 ++++++ .../service/database/models/DeviceModel.py | 5 +- src/context/service/database/models/_Base.py | 3 + .../service/database/uuids/Component.py | 45 +++++ .../openconfig/templates/Interfaces.py | 10 ++ .../openconfig/templates/Interfaces_mng.py | 100 ----------- .../drivers/openconfig/templates/__init__.py | 6 +- .../templates/interfaces_mng/get.xml | 3 - 12 files changed, 322 insertions(+), 115 deletions(-) create mode 100644 src/context/service/database/Component.py create mode 100644 src/context/service/database/models/ComponentModel.py create mode 100644 src/context/service/database/uuids/Component.py delete mode 100644 src/device/service/drivers/openconfig/templates/Interfaces_mng.py delete mode 100644 src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml diff --git a/proto/context.proto b/proto/context.proto index 55a80470d..7c89d7c97 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -174,14 +174,30 @@ message Device { DeviceOperationalStatusEnum device_operational_status = 5; repeated DeviceDriverEnum device_drivers = 6; repeated EndPoint device_endpoints = 7; - repeated Component component = 8; // Used for inventory + map components = 8; // Used for inventory DeviceId controller_id = 9; // Identifier of node controlling the actual device } -message Component { - repeated string comp_string = 1; +message Component { //Defined previously to this section - Tested OK + Uuid uuid = 1; + string name = 2; + string type = 3; + repeated string child = 4; // list[sub-component.name] + map attributes = 5; // dict[attr.name => json.dumps(attr.value)] +} + +message ComponentId { //NEW + DeviceId device_id = 1; + Uuid endpoint_uuid = 2; } +message ComponentIdList { //NEW + repeated ComponentId component_ids = 1; +} + + +// ----------------------------------------------------- + message DeviceConfig { repeated ConfigRule config_rules = 1; } diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py new file mode 100644 index 000000000..0d33bb9e0 --- /dev/null +++ b/src/context/service/database/Component.py @@ -0,0 +1,164 @@ + + + +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime, json, logging +from sqlalchemy import delete +#from sqlalchemy.dialects import postgresql +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session +from typing import Dict, List, Optional, Set +from common.proto.context_pb2 import Component +from common.proto.context_pb2 import ConfigRule +from common.tools.grpc.Tools import grpc_message_to_json_string +from .models.ComponentModel import ComponentModel +from .uuids._Builder import get_uuid_from_string +from .uuids.EndPoint import endpoint_get_uuid +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, selectinload, sessionmaker +from sqlalchemy_cockroachdb import run_transaction +from common.proto.context_pb2 import ComponentIdList +from .models.ComponentModel import ComponentModel +from .uuids.Component import component_get_uuid +from .ConfigRule import compose_config_rules_data + +LOGGER = logging.getLogger(__name__) + +def compose_components_data( + components : List[Component], now : datetime.datetime, + device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None +) -> List[Dict]: + dict_components : List[Dict] = list() + for position,component in enumerate(components): + str_kind = component.WhichOneof('config_rule') + LOGGER.info("DATA") + message = (grpc_message_to_json_string(getattr(component, str_kind, {}))) + data = json.loads(message) + resource_key = data["resource_key"] + resource_value = data["resource_value"] + if '/inventory' in resource_key: + LOGGER.info('Parametros: KEY',resource_key,'Value',resource_value) + dict_component = { + 'data' : resource_value, + 'created_at': now, + 'updated_at': now, + } + + parent_kind,parent_uuid = '',None + if device_uuid is not None: + dict_component['device_uuid'] = device_uuid + parent_kind,parent_uuid = 'device',device_uuid + elif service_uuid is not None: + dict_component['service_uuid'] = service_uuid + parent_kind,parent_uuid = 'service',service_uuid + elif slice_uuid is not None: + dict_component['slice_uuid'] = slice_uuid + parent_kind,parent_uuid = 'slice',slice_uuid + else: + MSG = 'Parent for Component({:s}) cannot be identified '+\ + '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' + str_component = grpc_message_to_json_string(component) + raise Exception(MSG.format(str_component, str(device_uuid), str(service_uuid), str(slice_uuid))) + + + componenet_name = '{:s}:{:s}:{:s}'.format(parent_kind, 'custom', component.custom.resource_key) + + + component_uuid = get_uuid_from_string(componenet_name, prefix_for_name=parent_uuid) + dict_component['component_uuid'] = component_uuid + + dict_components.append(dict_component) + else: + continue + LOGGER.info('Parametros:',dict_components) + return dict_components + +''' +def upsert_components( + session : Session, components : List[Dict], is_delete : bool = False, + device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None, +) -> bool: + + if device_uuid is not None and service_uuid is None and slice_uuid is None: + klass = ComponentModel + + else: + MSG = 'DataModel cannot be identified (device_uuid={:s})' + raise Exception(MSG.format(str(device_uuid))) + + uuids_to_upsert : Dict[str, int] = dict() + rules_to_upsert : List[Dict] = list() + for component in components: + component_uuid = component['component_uuid'] + + position = uuids_to_upsert.get(component_uuid) + if position is None: + # if not added, add it + rules_to_upsert.append(component) + uuids_to_upsert[component_uuid] = len(rules_to_upsert) - 1 + else: + # if already added, update occurrence + rules_to_upsert[position] = component + + + upsert_affected = False + if len(rules_to_upsert) > 0: + stmt = insert(klass).values(rules_to_upsert) + stmt = stmt.on_conflict_do_update( + index_elements=[klass.component_uuid], + set_=dict( + data = stmt.excluded.data, + updated_at = stmt.excluded.updated_at, + ) + ) + stmt = stmt.returning(klass.created_at, klass.updated_at) + #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) + #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt))) + components_updates = session.execute(stmt).fetchall() + upsert_affected = any([(updated_at > created_at) for created_at,updated_at in components_updates]) + + return upsert_affected + + def component_list_names(db_engine: Engine, request: ComponentIdList) -> List[Dict]: + component_uuids = { + component_get_uuid(component_id, allow_random=False)[-1] + for component_id in request.component_ids + } + + def callback(session: Session) -> List[Dict]: + obj_list: List[ComponentModel] = session.query(ComponentModel)\ + .options(selectinload(ComponentModel.device))\ + .filter(ComponentModel.component_uuid.in_(component_uuids)).all() + return [obj.dump_name() for obj in obj_list] + + return run_transaction(sessionmaker(bind=db_engine), callback) + +def compose_components_data(data: Dict[str, any]) -> List[Dict]: + filtered_data = [] + + for item in data: + for key, value in item: + if any("inventory" in key): + filtered_data.append(item) + + LOGGER.info("Filtered Data:") + LOGGER.info(filtered_data) + + + + # Return the result + return filtered_data +''' \ No newline at end of file diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index c5b259a2d..8a09e0b8c 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -34,6 +34,9 @@ def compose_config_rules_data( ) -> List[Dict]: dict_config_rules : List[Dict] = list() for position,config_rule in enumerate(config_rules): + + LOGGER.info("REQUEST") + LOGGER.info(position,config_rule) str_kind = config_rule.WhichOneof('config_rule') kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member dict_config_rule = { diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index 8560399cc..67d8e9b0f 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -26,12 +26,14 @@ from context.service.database.uuids.Topology import topology_get_uuid from .models.DeviceModel import DeviceModel from .models.EndPointModel import EndPointModel from .models.TopologyModel import TopologyDeviceModel +from .models.ComponentModel import ComponentModel from .models.enums.DeviceDriver import grpc_to_enum__device_driver from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type from .uuids.Device import device_get_uuid from .uuids.EndPoint import endpoint_get_uuid from .ConfigRule import compose_config_rules_data, upsert_config_rules +from .Component import compose_components_data LOGGER = logging.getLogger(__name__) @@ -46,6 +48,7 @@ def device_list_objs(db_engine : Engine) -> List[Dict]: obj_list : List[DeviceModel] = session.query(DeviceModel)\ .options(selectinload(DeviceModel.endpoints))\ .options(selectinload(DeviceModel.config_rules))\ + .options(selectinload(DeviceModel.components))\ .all() #.options(selectinload(DeviceModel.components))\ return [obj.dump() for obj in obj_list] @@ -133,7 +136,8 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]: }) topology_uuids.add(endpoint_topology_uuid) - config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid) + components_data = compose_components_data(request.device_config.config_rules, now, device_uuid=device_uuid) + config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid) device_data = [{ 'device_uuid' : device_uuid, @@ -187,9 +191,26 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]: index_elements=[TopologyDeviceModel.topology_uuid, TopologyDeviceModel.device_uuid] )) + updated_components = False + + LOGGER.info("HERE ERRPR DEBUG") + LOGGER.info(components_data) + if len(components_data) > 0: + stmt = insert(ComponentModel).values(components_data) + stmt = stmt.on_conflict_do_update( + index_elements=[ComponentModel.component_uuid], + set_=dict( + data = stmt.excluded.data, + updated_at = stmt.excluded.updated_at, + ) + ) + stmt = stmt.returning(ComponentModel.created_at, ComponentModel.updated_at) + component_updates = session.execute(stmt).fetchall() + updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates]) + changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid) - return updated or updated_endpoints or changed_config_rules + return updated or updated_components or changed_config_rules updated = run_transaction(sessionmaker(bind=db_engine), callback) return json_device_id(device_uuid),updated diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py new file mode 100644 index 000000000..52ebf2fb5 --- /dev/null +++ b/src/context/service/database/models/ComponentModel.py @@ -0,0 +1,51 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from sqlalchemy import Column, DateTime, ForeignKey, String +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from typing import Dict +from ._Base import _Base + +class ComponentModel(_Base): #Inherited functionality from the base class _Base + __tablename__ = 'device_component' #Name of the table in the DB associtaed with this model + + component_uuid = Column(UUID(as_uuid=False), primary_key=True) #Unique identifier that serves as a primary key for this table + device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele) + # component_name = Column(String, nullable=False) #String field that stores the name of the component + data = Column(String, nullable=False) #String field that stores data about the component + created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component + updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component + + device = relationship('DeviceModel', back_populates='components')# lazy='selectin'#Defines a relationship between ComponentModel and DeviceModel + #Represents a 1:n relationship where 1 device -> N components // The relationship is defined by the FK device_uuid + def dump_id(self) -> Dict: + return{ + 'device_id' : self.device.dump_id(), + 'component_uuid': {'uuid': self.component_uuid}, + } + + def dump(self) -> Dict: + return { + 'component_id' : self.dump_id(), + 'data' : self.data, + } + + def dump_name(self) -> Dict: + return { + 'component_id' : self.dump_id(), + 'device_name' : self.device.device_name, + 'component_name': self.name, + } diff --git a/src/context/service/database/models/DeviceModel.py b/src/context/service/database/models/DeviceModel.py index 1097d0b9a..376dc98c4 100644 --- a/src/context/service/database/models/DeviceModel.py +++ b/src/context/service/database/models/DeviceModel.py @@ -36,6 +36,7 @@ class DeviceModel(_Base): #topology_devices = relationship('TopologyDeviceModel', back_populates='device') config_rules = relationship('DeviceConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device' endpoints = relationship('EndPointModel', passive_deletes=True) # lazy='joined', back_populates='device' + components = relationship('ComponentModel', passive_deletes=True) # lazy='joined', back_populates='device' controller = relationship('DeviceModel', remote_side=[device_uuid], passive_deletes=True) # lazy='joined', back_populates='device' def dump_id(self) -> Dict: @@ -55,7 +56,7 @@ class DeviceModel(_Base): ]} def dump_components(self) -> List[Dict]: - return [] + return [component.dump() for component in self.components] def dump(self, include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True, @@ -70,5 +71,5 @@ class DeviceModel(_Base): } if include_endpoints: result['device_endpoints'] = self.dump_endpoints() if include_config_rules: result['device_config'] = self.dump_config_rules() - if include_components: result['component'] = self.dump_components() + if include_components: result['components'] = self.dump_components() return result diff --git a/src/context/service/database/models/_Base.py b/src/context/service/database/models/_Base.py index b87b9b06d..b325381f9 100644 --- a/src/context/service/database/models/_Base.py +++ b/src/context/service/database/models/_Base.py @@ -60,6 +60,9 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [ 'topology_name', 'created_at', 'updated_at' ]), + index_storing('device_component_idx', 'device_component', ['device_uuid'], [ + 'data', 'created_at', 'updated_at' + ]), ] def callback(session : Session) -> bool: for stmt in statements: session.execute(stmt) diff --git a/src/context/service/database/uuids/Component.py b/src/context/service/database/uuids/Component.py new file mode 100644 index 000000000..218cc793c --- /dev/null +++ b/src/context/service/database/uuids/Component.py @@ -0,0 +1,45 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Tuple +from common.proto.context_pb2 import ComponentId +from common.method_wrappers.ServiceExceptions import InvalidArgumentsException +from ._Builder import get_uuid_from_string, get_uuid_random +from .Device import device_get_uuid + +def component_get_uuid( + component_id: ComponentId, component_name: str = '', allow_random: bool = False +) -> str: + device_uuid = device_get_uuid(component_id.device_id, allow_random=False) + raw_component_uuid = component_id.component_uuid.uuid + + if raw_component_uuid: + prefix_for_name = f'{device_uuid}' + return get_uuid_from_string(raw_component_uuid, prefix_for_name=prefix_for_name) + + if component_name: + prefix_for_name = f'{device_uuid}' + return get_uuid_from_string(component_name, prefix_for_name=prefix_for_name) + + if allow_random: + return get_uuid_random() + + raise InvalidArgumentsException( + [ + ('component_id.component_uuid.uuid', raw_component_uuid), + ('name', component_name), + ], + extra_details=['At least one is required to produce a Component UUID'] + ) + diff --git a/src/device/service/drivers/openconfig/templates/Interfaces.py b/src/device/service/drivers/openconfig/templates/Interfaces.py index 3855db17b..8d1dcf16e 100644 --- a/src/device/service/drivers/openconfig/templates/Interfaces.py +++ b/src/device/service/drivers/openconfig/templates/Interfaces.py @@ -22,6 +22,7 @@ LOGGER = logging.getLogger(__name__) XPATH_INTERFACES = "//oci:interfaces/oci:interface" XPATH_SUBINTERFACES = ".//oci:subinterfaces/oci:subinterface" XPATH_IPV4ADDRESSES = ".//ociip:ipv4/ociip:addresses/ociip:address" +XPATH_IPV6ADDRESSES = ".//ociip:ipv6/ociip:addresses/ociip:address" def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] @@ -97,6 +98,15 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: #add_value_from_collection(subinterface, 'ipv4_addresses', ipv4_addresses) + for xml_ipv6_address in xml_subinterface.xpath(XPATH_IPV6ADDRESSES, namespaces=NAMESPACES): + #LOGGER.info('xml_ipv6_address = {:s}'.format(str(ET.tostring(xml_ipv6_address)))) + + address = xml_ipv6_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES) + add_value_from_tag(subinterface, 'address_ipv6', address) + + prefix = xml_ipv6_address.find('ociip:state/ociip:prefix-length', namespaces=NAMESPACES) + add_value_from_tag(subinterface, 'address_prefix_v6', prefix, cast=int) + if len(subinterface) == 0: continue resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) response.append((resource_key, subinterface)) diff --git a/src/device/service/drivers/openconfig/templates/Interfaces_mng.py b/src/device/service/drivers/openconfig/templates/Interfaces_mng.py deleted file mode 100644 index ced193252..000000000 --- a/src/device/service/drivers/openconfig/templates/Interfaces_mng.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -#Method Name: parse - -#Parameters: - - - xml_data: [ET.Element] Represents the XML data to be parsed. - -# Functionality: - -The parse function of the interfaces_mng class has the functionality to parse -an XML document represented by the xml_data parameter and extract specific -information from the XML elements, namely the active interfaces with their name, -type, ipv4 and ipv6 addresses in case they have. - -To generate the template the following steps are performed: - -1) An empty list called response is created to store the results of the analysis. - -2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS -expression. These elements represent management interfaces in the XML document. - -3) For each management interfaces: -A dictionary called interfaces_mng is initialized that will store the information extracted -from the interfaces.The values of the relevant XML elements are extracted and added to -the dictionary. - -#Return: -List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) -with the information extracted from the XML document interfaces is returned. - -""" - - -import logging, lxml.etree as ET -from typing import Any, Dict, List, Tuple -from .Namespace import NAMESPACES -from .Tools import add_value_from_tag - -LOGGER = logging.getLogger(__name__) - -XPATH_PORTS = "//oci:interfaces/oci:interface" -XPATH_SUBINTERFACES = ".//oci:subinterfaces/oci:subinterface" -XPATH_IPV4ADDRESSES = ".//ociip:ipv4/ociip:addresses/ociip:address" -XPATH_IPV6ADDRESSES = ".//ociip:ipv6/ociip:addresses/ociip:address" - -def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: - response = [] - LOGGER.debug("Interfaces_mngPrueba") - - for xml_interface_mng in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): - LOGGER.info('xml_component xml_interfaces_mng = {:s}'.format(str(ET.tostring(xml_interface_mng)))) - interfaces_mng = {} - - interface_enabled = xml_interface_mng.find('oci:config/oci:enabled', namespaces=NAMESPACES) - if interface_enabled == None: - interface_enabled = xml_interface_mng.find('oci:state/oci:enabled', namespaces=NAMESPACES) - if interface_enabled == None: continue - if 'false' in interface_enabled or 'false' in interface_enabled.text: continue - - interface_name = xml_interface_mng.find('oci:name', namespaces=NAMESPACES) - if interface_name is None or interface_name.text is None: continue - add_value_from_tag(interfaces_mng, 'name', interface_name) - - interface_type = xml_interface_mng.find('oci:config/oci:type', namespaces=NAMESPACES) - if interface_type is None: - interface_type = xml_interface_mng.find('oci:state/oci:type', namespaces=NAMESPACES) - if interface_type is None: continue - interface_type.text = interface_type.text.replace('ianaift:','') - add_value_from_tag(interfaces_mng, 'type', interface_type) - - for xml_subinterface in xml_interface_mng.xpath(XPATH_SUBINTERFACES, namespaces=NAMESPACES): - for xml_ipv4_address in xml_subinterface.xpath(XPATH_IPV4ADDRESSES, namespaces=NAMESPACES): - address_ipv4 = xml_ipv4_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES) - if not address_ipv4 is None: - add_value_from_tag(interfaces_mng, 'ipv4', address_ipv4) - - for xml_ipv6_address in xml_subinterface.xpath(XPATH_IPV6ADDRESSES, namespaces=NAMESPACES): - address_ipv6 = xml_ipv6_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES) - if not address_ipv6 is None: - add_value_from_tag(interfaces_mng, 'ipv6', address_ipv6) - if not 'ipv4' in interfaces_mng and not 'ipv6' in interfaces_mng: - if 'ip' in interfaces_mng['type'] or 'Loopback' in interfaces_mng['type']: continue - response.append(('/interfaces_mng/{:s}'.format(interfaces_mng['name']), interfaces_mng)) - - return response diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index 3dde94fc5..87eea1f0b 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -20,10 +20,9 @@ from jinja2 import Environment, PackageLoader, select_autoescape import paramiko from .Tools import generate_templates from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_INTERFACES_MNG,RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY) + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY) from .EndPoints import parse as parse_endpoints from .Interfaces import parse as parse_interfaces, parse_counters -from .Interfaces_mng import parse as parse_interfaces_mng from .NetworkInstances import parse as parse_network_instances from .RoutingPolicy import parse as parse_routing_policy from .Acl import parse as parse_acl @@ -34,7 +33,6 @@ ALL_RESOURCE_KEYS = [ RESOURCE_INVENTORY, RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, - RESOURCE_INTERFACES_MNG, RESOURCE_ROUTING_POLICIES, # routing policies should come before network instances RESOURCE_NETWORK_INSTANCES, RESOURCE_ACL, @@ -44,7 +42,6 @@ RESOURCE_KEY_MAPPINGS = { RESOURCE_INVENTORY : 'inventory', RESOURCE_ENDPOINTS : 'component', RESOURCE_INTERFACES : 'interface', - RESOURCE_INTERFACES_MNG : 'interfaces_mng', RESOURCE_NETWORK_INSTANCES: 'network_instance', RESOURCE_ROUTING_POLICIES : 'routing_policy', RESOURCE_ACL : 'acl', @@ -54,7 +51,6 @@ RESOURCE_PARSERS = { 'inventory' : parse_inventory, 'component' : parse_endpoints, 'interface' : parse_interfaces, - 'interfaces_mng' : parse_interfaces_mng, 'network_instance': parse_network_instances, 'routing_policy' : parse_routing_policy, 'interfaces/interface/state/counters': parse_counters, diff --git a/src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml b/src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml deleted file mode 100644 index 1003bff15..000000000 --- a/src/device/service/drivers/openconfig/templates/interfaces_mng/get.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file -- GitLab From ccdd5bd89282cb199465f70b8a4331a1743ef164 Mon Sep 17 00:00:00 2001 From: Armingol Date: Wed, 20 Sep 2023 10:13:05 +0200 Subject: [PATCH 04/13] changes to save inventory in ddbb --- proto/context.proto | 16 +-- src/context/service/database/Component.py | 124 +++--------------- src/context/service/database/ConfigRule.py | 4 +- src/context/service/database/Device.py | 7 +- .../service/database/models/ComponentModel.py | 14 +- .../service/database/models/DeviceModel.py | 2 +- src/context/service/database/models/_Base.py | 2 +- .../service/database/uuids/Component.py | 45 ------- .../drivers/openconfig/templates/Inventory.py | 43 ++---- 9 files changed, 51 insertions(+), 206 deletions(-) delete mode 100644 src/context/service/database/uuids/Component.py diff --git a/proto/context.proto b/proto/context.proto index 7c89d7c97..2bd51a56b 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -179,23 +179,13 @@ message Device { } message Component { //Defined previously to this section - Tested OK - Uuid uuid = 1; - string name = 2; - string type = 3; + Uuid component_uuid = 1; + string name = 2; + string type = 3; repeated string child = 4; // list[sub-component.name] map attributes = 5; // dict[attr.name => json.dumps(attr.value)] } -message ComponentId { //NEW - DeviceId device_id = 1; - Uuid endpoint_uuid = 2; -} - -message ComponentIdList { //NEW - repeated ComponentId component_ids = 1; -} - - // ----------------------------------------------------- message DeviceConfig { diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py index 0d33bb9e0..befaa0ef8 100644 --- a/src/context/service/database/Component.py +++ b/src/context/service/database/Component.py @@ -30,10 +30,7 @@ from .uuids.EndPoint import endpoint_get_uuid from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, selectinload, sessionmaker from sqlalchemy_cockroachdb import run_transaction -from common.proto.context_pb2 import ComponentIdList from .models.ComponentModel import ComponentModel -from .uuids.Component import component_get_uuid -from .ConfigRule import compose_config_rules_data LOGGER = logging.getLogger(__name__) @@ -44,121 +41,42 @@ def compose_components_data( dict_components : List[Dict] = list() for position,component in enumerate(components): str_kind = component.WhichOneof('config_rule') - LOGGER.info("DATA") message = (grpc_message_to_json_string(getattr(component, str_kind, {}))) data = json.loads(message) resource_key = data["resource_key"] resource_value = data["resource_value"] if '/inventory' in resource_key: - LOGGER.info('Parametros: KEY',resource_key,'Value',resource_value) + resource_value_data = json.loads(resource_value) + name = resource_value_data.get('name') + type = resource_value_data.get('class') + uuid = resource_value_data.get('uuid') + + if name is not None: + del resource_value_data['name'] + if type is not None: + del resource_value_data['class'] + if uuid is not None: + del resource_value_data['uuid'] + + attributes = resource_value_data #Store the remaining fields in 'attributes' + dict_component = { - 'data' : resource_value, + 'name' : name, + 'type' : type, + 'attributes' : attributes, 'created_at': now, 'updated_at': now, } - parent_kind,parent_uuid = '',None - if device_uuid is not None: - dict_component['device_uuid'] = device_uuid - parent_kind,parent_uuid = 'device',device_uuid - elif service_uuid is not None: - dict_component['service_uuid'] = service_uuid - parent_kind,parent_uuid = 'service',service_uuid - elif slice_uuid is not None: - dict_component['slice_uuid'] = slice_uuid - parent_kind,parent_uuid = 'slice',slice_uuid - else: - MSG = 'Parent for Component({:s}) cannot be identified '+\ - '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' - str_component = grpc_message_to_json_string(component) - raise Exception(MSG.format(str_component, str(device_uuid), str(service_uuid), str(slice_uuid))) - - - componenet_name = '{:s}:{:s}:{:s}'.format(parent_kind, 'custom', component.custom.resource_key) + dict_component['device_uuid'] = device_uuid + component_name = '{:s}:{:s}:{:s}'.format('device', 'custom', component.custom.resource_key) - component_uuid = get_uuid_from_string(componenet_name, prefix_for_name=parent_uuid) + component_uuid = get_uuid_from_string(component_name, prefix_for_name=device_uuid) dict_component['component_uuid'] = component_uuid dict_components.append(dict_component) else: continue - LOGGER.info('Parametros:',dict_components) - return dict_components - -''' -def upsert_components( - session : Session, components : List[Dict], is_delete : bool = False, - device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None, -) -> bool: - - if device_uuid is not None and service_uuid is None and slice_uuid is None: - klass = ComponentModel - - else: - MSG = 'DataModel cannot be identified (device_uuid={:s})' - raise Exception(MSG.format(str(device_uuid))) - - uuids_to_upsert : Dict[str, int] = dict() - rules_to_upsert : List[Dict] = list() - for component in components: - component_uuid = component['component_uuid'] - - position = uuids_to_upsert.get(component_uuid) - if position is None: - # if not added, add it - rules_to_upsert.append(component) - uuids_to_upsert[component_uuid] = len(rules_to_upsert) - 1 - else: - # if already added, update occurrence - rules_to_upsert[position] = component - - - upsert_affected = False - if len(rules_to_upsert) > 0: - stmt = insert(klass).values(rules_to_upsert) - stmt = stmt.on_conflict_do_update( - index_elements=[klass.component_uuid], - set_=dict( - data = stmt.excluded.data, - updated_at = stmt.excluded.updated_at, - ) - ) - stmt = stmt.returning(klass.created_at, klass.updated_at) - #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) - #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt))) - components_updates = session.execute(stmt).fetchall() - upsert_affected = any([(updated_at > created_at) for created_at,updated_at in components_updates]) - - return upsert_affected - def component_list_names(db_engine: Engine, request: ComponentIdList) -> List[Dict]: - component_uuids = { - component_get_uuid(component_id, allow_random=False)[-1] - for component_id in request.component_ids - } - - def callback(session: Session) -> List[Dict]: - obj_list: List[ComponentModel] = session.query(ComponentModel)\ - .options(selectinload(ComponentModel.device))\ - .filter(ComponentModel.component_uuid.in_(component_uuids)).all() - return [obj.dump_name() for obj in obj_list] - - return run_transaction(sessionmaker(bind=db_engine), callback) - -def compose_components_data(data: Dict[str, any]) -> List[Dict]: - filtered_data = [] - - for item in data: - for key, value in item: - if any("inventory" in key): - filtered_data.append(item) - - LOGGER.info("Filtered Data:") - LOGGER.info(filtered_data) - - - - # Return the result - return filtered_data -''' \ No newline at end of file + return dict_components diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index 8a09e0b8c..8275a3ffe 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -35,8 +35,8 @@ def compose_config_rules_data( dict_config_rules : List[Dict] = list() for position,config_rule in enumerate(config_rules): - LOGGER.info("REQUEST") - LOGGER.info(position,config_rule) + #LOGGER.info("REQUEST") + #LOGGER.info(position,config_rule) str_kind = config_rule.WhichOneof('config_rule') kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member dict_config_rule = { diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index 67d8e9b0f..0faecf70f 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -13,6 +13,7 @@ # limitations under the License. import datetime, logging +import json from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, selectinload, sessionmaker @@ -193,14 +194,14 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]: updated_components = False - LOGGER.info("HERE ERRPR DEBUG") - LOGGER.info(components_data) if len(components_data) > 0: stmt = insert(ComponentModel).values(components_data) stmt = stmt.on_conflict_do_update( index_elements=[ComponentModel.component_uuid], set_=dict( - data = stmt.excluded.data, + name = str(stmt.excluded.name), + type = str(stmt.excluded.type), + attributes = str(stmt.excluded.attributes), updated_at = stmt.excluded.updated_at, ) ) diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py index 52ebf2fb5..ac6d44ba8 100644 --- a/src/context/service/database/models/ComponentModel.py +++ b/src/context/service/database/models/ComponentModel.py @@ -24,8 +24,9 @@ class ComponentModel(_Base): #Inherit component_uuid = Column(UUID(as_uuid=False), primary_key=True) #Unique identifier that serves as a primary key for this table device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele) - # component_name = Column(String, nullable=False) #String field that stores the name of the component - data = Column(String, nullable=False) #String field that stores data about the component + name = Column(String, nullable=False) #String field that stores the name of the component + type = Column(String, nullable=False) #String field that stores the name of the component + attributes = Column(String, nullable=False) #String field that stores data about the component created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component @@ -38,10 +39,11 @@ class ComponentModel(_Base): #Inherit } def dump(self) -> Dict: - return { - 'component_id' : self.dump_id(), - 'data' : self.data, - } + data['component_uuid'] = self.component_uuid + data['name'] = self.name + data['type'] = self.type + data['attributes'] = self.attributes + return data def dump_name(self) -> Dict: return { diff --git a/src/context/service/database/models/DeviceModel.py b/src/context/service/database/models/DeviceModel.py index 376dc98c4..bd52ce76d 100644 --- a/src/context/service/database/models/DeviceModel.py +++ b/src/context/service/database/models/DeviceModel.py @@ -56,7 +56,7 @@ class DeviceModel(_Base): ]} def dump_components(self) -> List[Dict]: - return [component.dump() for component in self.components] + return {component.name:component.dump() for component in self.components} def dump(self, include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True, diff --git a/src/context/service/database/models/_Base.py b/src/context/service/database/models/_Base.py index b325381f9..52eb6b088 100644 --- a/src/context/service/database/models/_Base.py +++ b/src/context/service/database/models/_Base.py @@ -61,7 +61,7 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: 'topology_name', 'created_at', 'updated_at' ]), index_storing('device_component_idx', 'device_component', ['device_uuid'], [ - 'data', 'created_at', 'updated_at' + 'name', 'type', 'attributes', 'created_at', 'updated_at' ]), ] def callback(session : Session) -> bool: diff --git a/src/context/service/database/uuids/Component.py b/src/context/service/database/uuids/Component.py deleted file mode 100644 index 218cc793c..000000000 --- a/src/context/service/database/uuids/Component.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Tuple -from common.proto.context_pb2 import ComponentId -from common.method_wrappers.ServiceExceptions import InvalidArgumentsException -from ._Builder import get_uuid_from_string, get_uuid_random -from .Device import device_get_uuid - -def component_get_uuid( - component_id: ComponentId, component_name: str = '', allow_random: bool = False -) -> str: - device_uuid = device_get_uuid(component_id.device_id, allow_random=False) - raw_component_uuid = component_id.component_uuid.uuid - - if raw_component_uuid: - prefix_for_name = f'{device_uuid}' - return get_uuid_from_string(raw_component_uuid, prefix_for_name=prefix_for_name) - - if component_name: - prefix_for_name = f'{device_uuid}' - return get_uuid_from_string(component_name, prefix_for_name=prefix_for_name) - - if allow_random: - return get_uuid_random() - - raise InvalidArgumentsException( - [ - ('component_id.component_uuid.uuid', raw_component_uuid), - ('name', component_name), - ], - extra_details=['At least one is required to produce a Component UUID'] - ) - diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index 5775ac963..b7e7c7c12 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -63,7 +63,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) inventory = {} inventory['parent-component-references'] = '' - inventory['uuid'] = '' inventory['name'] = '' inventory['class'] = '' inventory['attributes'] = {} @@ -71,16 +70,11 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) if component_name is None or component_name.text is None: continue - add_value_from_tag(inventory, 'uuid', component_name) add_value_from_tag(inventory, 'name', component_name) component_description = xml_component.find('ocp:state/ocp:description', namespaces=NAMESPACES) if not component_description is None: add_value_from_tag(inventory['attributes'], 'description', component_description) - #Podría estar presente en lugar del name - component_alias = xml_component.find('ocp:state/ocp:alias', namespaces=NAMESPACES) - if not component_alias is None: - add_value_from_tag(inventory['attributes'], 'alias', component_alias) component_location = xml_component.find('ocp:state/ocp:location', namespaces=NAMESPACES) if not component_location is None: @@ -90,13 +84,12 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: component_type.text = component_type.text.replace('oc-platform-types:','') if component_type is None: continue add_value_from_tag(inventory, 'class', component_type) + if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: - add_value_from_tag(inventory['attributes'], 'contained-child', component_empty) - - #añadir el parent pos + add_value_from_tag(inventory['attributes'], 'empty', component_empty) component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) if component_parent is None or component_parent.text is None: @@ -120,37 +113,24 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: if not component_serial is None: add_value_from_tag(inventory['attributes'], 'serial-num', component_serial) - component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) - if not component_serial_t is None: - add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t) - component_mfg_name = xml_component.find('ocp:state/ocp:mfg-name', namespaces=NAMESPACES) if not component_mfg_name is None: - add_value_from_tag(inventory['attributes'], 'mfg-name', component_mfg_name) - - component_part_no = xml_component.find('ocp:state/ocp:part-no', namespaces=NAMESPACES) - if not component_part_no is None: - add_value_from_tag(inventory['attributes'], 'part-number', component_part_no) - - #modificar es un identificador de seguimiento de activos asignado por el usuario para el componente - component_asset_id = xml_component.find('ocp:state/ocp:asset-id', namespaces=NAMESPACES) - if not component_asset_id is None: - add_value_from_tag(inventory['attributes'], 'asset-id', component_asset_id) + add_value_from_tag(inventory['attributes'], 'manufacturer-name', component_mfg_name) component_removable = xml_component.find('ocp:state/ocp:removable', namespaces=NAMESPACES) if not component_removable is None: - add_value_from_tag(inventory['attributes'], 'is-fru', component_removable) + add_value_from_tag(inventory['attributes'], 'removable', component_removable) component_mfg_date = xml_component.find('ocp:state/ocp:mfg-date', namespaces=NAMESPACES) if not component_mfg_date is None: add_value_from_tag(inventory['attributes'], 'mfg-date', component_mfg_date) - #modificar "Este nodo contiene información de identificación sobre el componente" - component_uri = xml_component.find('ocp:state/ocp:uri', namespaces=NAMESPACES) - if not component_uri is None: - add_value_from_tag(inventory['attributes'], 'uri', component_uri) - - #Para transceiver + #Transceiver Information + + component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) + if not component_serial_t is None: + add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t) + component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES) if component_present is not None and 'NOT_PRESENT' in component_present.text: continue @@ -166,7 +146,7 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: if not component_form is None: component_form.text = component_form.text.replace('oc-opt-types:','') add_value_from_tag(inventory['attributes'], 'form-factor', component_form) - #añadir características del parent references + if inventory['parent-component-references'] not in parent_types: parent_types[inventory['parent-component-references']] = len(parent_types) + 1 @@ -179,7 +159,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: for tupla in response: if inventory['parent-component-references'] in tupla[0]: component_reference.extend([tupla[1]['class']]) - component_reference.extend([tupla[1]['uuid']]) inventory['component-reference'] = component_reference -- GitLab From d573d04161d17c630633c6c54e5e043e0217030e Mon Sep 17 00:00:00 2001 From: Armingol Date: Wed, 20 Sep 2023 13:40:51 +0200 Subject: [PATCH 05/13] changes in "device_component" logic --- proto/context.proto | 3 +- .../service/ContextServiceServicerImpl.py | 4 +- src/context/service/database/Component.py | 46 ++++++++----------- src/context/service/database/Device.py | 7 +-- .../service/database/models/ComponentModel.py | 4 +- .../service/database/models/DeviceModel.py | 2 +- 6 files changed, 31 insertions(+), 35 deletions(-) diff --git a/proto/context.proto b/proto/context.proto index 2bd51a56b..1c7518b9f 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -174,7 +174,7 @@ message Device { DeviceOperationalStatusEnum device_operational_status = 5; repeated DeviceDriverEnum device_drivers = 6; repeated EndPoint device_endpoints = 7; - map components = 8; // Used for inventory + repeated Component components = 8; // Used for inventory DeviceId controller_id = 9; // Identifier of node controlling the actual device } @@ -184,6 +184,7 @@ message Component { //Defined previously to this section string type = 3; repeated string child = 4; // list[sub-component.name] map attributes = 5; // dict[attr.name => json.dumps(attr.value)] + string parent = 6; } // ----------------------------------------------------- diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index 6d540b494..23b70be3a 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -140,7 +140,9 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ListDevices(self, request : Empty, context : grpc.ServicerContext) -> DeviceList: - return DeviceList(devices=device_list_objs(self.db_engine)) + devices = device_list_objs(self.db_engine) + LOGGER.info('DEVICES: {:s}'.format(str(devices))) + return DeviceList(devices=devices) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetDevice(self, request : ContextId, context : grpc.ServicerContext) -> Device: diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py index befaa0ef8..8774c0cbf 100644 --- a/src/context/service/database/Component.py +++ b/src/context/service/database/Component.py @@ -47,36 +47,26 @@ def compose_components_data( resource_value = data["resource_value"] if '/inventory' in resource_key: resource_value_data = json.loads(resource_value) - name = resource_value_data.get('name') - type = resource_value_data.get('class') - uuid = resource_value_data.get('uuid') - - if name is not None: - del resource_value_data['name'] - if type is not None: - del resource_value_data['class'] - if uuid is not None: - del resource_value_data['uuid'] - - attributes = resource_value_data #Store the remaining fields in 'attributes' + name = resource_value_data.pop('name', None) + type = resource_value_data.pop('class', None) + uuid = resource_value_data.pop('uuid', None) + parent = resource_value_data.pop('parent-component-references', None) + resource_value_data.pop('component-reference', None) + attributes = { + attr_name:json.dumps(attr_value) + for attr_name,attr_value in resource_value_data.items() + } + component_uuid = get_uuid_from_string(component.custom.resource_key, prefix_for_name=device_uuid) dict_component = { - 'name' : name, - 'type' : type, - 'attributes' : attributes, - 'created_at': now, - 'updated_at': now, + 'component_uuid': component_uuid, + 'device_uuid' : device_uuid, + 'name' : name, + 'type' : type, + 'attributes' : json.dumps(attributes), #Store the remaining fields in 'attributes' + 'parent' : parent, + 'created_at' : now, + 'updated_at' : now, } - - dict_component['device_uuid'] = device_uuid - component_name = '{:s}:{:s}:{:s}'.format('device', 'custom', component.custom.resource_key) - - - component_uuid = get_uuid_from_string(component_name, prefix_for_name=device_uuid) - dict_component['component_uuid'] = component_uuid - dict_components.append(dict_component) - else: - continue - return dict_components diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index 0faecf70f..0bffce8b4 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -199,9 +199,10 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]: stmt = stmt.on_conflict_do_update( index_elements=[ComponentModel.component_uuid], set_=dict( - name = str(stmt.excluded.name), - type = str(stmt.excluded.type), - attributes = str(stmt.excluded.attributes), + name = stmt.excluded.name, + type = stmt.excluded.type, + attributes = stmt.excluded.attributes, + parent = stmt.excluded.parent, updated_at = stmt.excluded.updated_at, ) ) diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py index ac6d44ba8..c74b3955d 100644 --- a/src/context/service/database/models/ComponentModel.py +++ b/src/context/service/database/models/ComponentModel.py @@ -27,6 +27,7 @@ class ComponentModel(_Base): #Inherit name = Column(String, nullable=False) #String field that stores the name of the component type = Column(String, nullable=False) #String field that stores the name of the component attributes = Column(String, nullable=False) #String field that stores data about the component + parent = Column(String, nullable=False) #String field that stores data about the component created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component @@ -39,10 +40,11 @@ class ComponentModel(_Base): #Inherit } def dump(self) -> Dict: + data = json.loads(self.attributes) data['component_uuid'] = self.component_uuid data['name'] = self.name data['type'] = self.type - data['attributes'] = self.attributes + data['parent'] = self.parent return data def dump_name(self) -> Dict: diff --git a/src/context/service/database/models/DeviceModel.py b/src/context/service/database/models/DeviceModel.py index bd52ce76d..376dc98c4 100644 --- a/src/context/service/database/models/DeviceModel.py +++ b/src/context/service/database/models/DeviceModel.py @@ -56,7 +56,7 @@ class DeviceModel(_Base): ]} def dump_components(self) -> List[Dict]: - return {component.name:component.dump() for component in self.components} + return [component.dump() for component in self.components] def dump(self, include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True, -- GitLab From 9df78fb0658f56a924265edbb46f0491691b902c Mon Sep 17 00:00:00 2001 From: Armingol Date: Thu, 21 Sep 2023 14:47:18 +0200 Subject: [PATCH 06/13] Changes in context --- proto/context.proto | 6 +++--- src/context/service/database/models/ComponentModel.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/proto/context.proto b/proto/context.proto index 1c7518b9f..0d7a66731 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -182,9 +182,9 @@ message Component { //Defined previously to this section Uuid component_uuid = 1; string name = 2; string type = 3; - repeated string child = 4; // list[sub-component.name] - map attributes = 5; // dict[attr.name => json.dumps(attr.value)] - string parent = 6; + + map attributes = 4; // dict[attr.name => json.dumps(attr.value)] + string parent = 5; } // ----------------------------------------------------- diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py index c74b3955d..964cb9bf3 100644 --- a/src/context/service/database/models/ComponentModel.py +++ b/src/context/service/database/models/ComponentModel.py @@ -41,7 +41,7 @@ class ComponentModel(_Base): #Inherit def dump(self) -> Dict: data = json.loads(self.attributes) - data['component_uuid'] = self.component_uuid + data['component_uuid'] = {'uuid': self.component_uuid}, data['name'] = self.name data['type'] = self.type data['parent'] = self.parent -- GitLab From 167a42fa22f9e6c3897916052065e9885b6114d7 Mon Sep 17 00:00:00 2001 From: Lluis Gifre Renom Date: Thu, 21 Sep 2023 14:10:38 +0000 Subject: [PATCH 07/13] Context component: - Modified database/Component to properly store attributes in database, report raw component, and dump component leftovers --- src/context/service/database/Component.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py index 8774c0cbf..17d3cb9ff 100644 --- a/src/context/service/database/Component.py +++ b/src/context/service/database/Component.py @@ -1,6 +1,5 @@ - # Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,25 +47,27 @@ def compose_components_data( if '/inventory' in resource_key: resource_value_data = json.loads(resource_value) name = resource_value_data.pop('name', None) - type = resource_value_data.pop('class', None) - uuid = resource_value_data.pop('uuid', None) + type_ = resource_value_data.pop('class', None) parent = resource_value_data.pop('parent-component-references', None) - resource_value_data.pop('component-reference', None) + attributes = resource_value_data.pop('attributes', {}) + if len(resource_value_data) > 0: + LOGGER.warning('Discarding Component Leftovers: {:s}'.format(str(resource_value_data))) attributes = { attr_name:json.dumps(attr_value) - for attr_name,attr_value in resource_value_data.items() + for attr_name,attr_value in attributes.items() } component_uuid = get_uuid_from_string(component.custom.resource_key, prefix_for_name=device_uuid) dict_component = { 'component_uuid': component_uuid, 'device_uuid' : device_uuid, 'name' : name, - 'type' : type, + 'type' : type_, 'attributes' : json.dumps(attributes), #Store the remaining fields in 'attributes' 'parent' : parent, 'created_at' : now, 'updated_at' : now, } + LOGGER.debug('dict_component={:s}'.format(str(dict_component))) dict_components.append(dict_component) return dict_components -- GitLab From 62561f0deba77742f2dd8e1f85886e3faf631217 Mon Sep 17 00:00:00 2001 From: Armingol Date: Thu, 21 Sep 2023 17:00:14 +0200 Subject: [PATCH 08/13] Fix formatting of "component". Code cleanup --- src/context/service/database/Component.py | 15 +++++++-------- src/context/service/database/ConfigRule.py | 2 -- .../service/database/models/ComponentModel.py | 17 +++++++++-------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py index 17d3cb9ff..3f2776787 100644 --- a/src/context/service/database/Component.py +++ b/src/context/service/database/Component.py @@ -40,16 +40,16 @@ def compose_components_data( dict_components : List[Dict] = list() for position,component in enumerate(components): str_kind = component.WhichOneof('config_rule') - message = (grpc_message_to_json_string(getattr(component, str_kind, {}))) - data = json.loads(message) - resource_key = data["resource_key"] + message = (grpc_message_to_json_string(getattr(component, str_kind, {}))) + data = json.loads(message) + resource_key = data["resource_key"] resource_value = data["resource_value"] if '/inventory' in resource_key: resource_value_data = json.loads(resource_value) - name = resource_value_data.pop('name', None) - type_ = resource_value_data.pop('class', None) - parent = resource_value_data.pop('parent-component-references', None) - attributes = resource_value_data.pop('attributes', {}) + name = resource_value_data.pop('name', None) + type_ = resource_value_data.pop('class', None) + parent = resource_value_data.pop('parent-component-references', None) + attributes = resource_value_data.pop('attributes', {}) if len(resource_value_data) > 0: LOGGER.warning('Discarding Component Leftovers: {:s}'.format(str(resource_value_data))) @@ -68,6 +68,5 @@ def compose_components_data( 'created_at' : now, 'updated_at' : now, } - LOGGER.debug('dict_component={:s}'.format(str(dict_component))) dict_components.append(dict_component) return dict_components diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index 8275a3ffe..1c5a2c7d5 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -35,8 +35,6 @@ def compose_config_rules_data( dict_config_rules : List[Dict] = list() for position,config_rule in enumerate(config_rules): - #LOGGER.info("REQUEST") - #LOGGER.info(position,config_rule) str_kind = config_rule.WhichOneof('config_rule') kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member dict_config_rule = { diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py index 964cb9bf3..60927e944 100644 --- a/src/context/service/database/models/ComponentModel.py +++ b/src/context/service/database/models/ComponentModel.py @@ -25,9 +25,9 @@ class ComponentModel(_Base): #Inherit component_uuid = Column(UUID(as_uuid=False), primary_key=True) #Unique identifier that serves as a primary key for this table device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele) name = Column(String, nullable=False) #String field that stores the name of the component - type = Column(String, nullable=False) #String field that stores the name of the component - attributes = Column(String, nullable=False) #String field that stores data about the component - parent = Column(String, nullable=False) #String field that stores data about the component + type = Column(String, nullable=False) #String field that stores the type of the component + attributes = Column(String, nullable=False) #String field that stores the attributes of the component + parent = Column(String, nullable=False) #String field that stores the parent of the component created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component @@ -40,11 +40,12 @@ class ComponentModel(_Base): #Inherit } def dump(self) -> Dict: - data = json.loads(self.attributes) - data['component_uuid'] = {'uuid': self.component_uuid}, - data['name'] = self.name - data['type'] = self.type - data['parent'] = self.parent + data = dict() + data['attributes'] = json.loads(self.attributes) + data['component_uuid'] = {'uuid': self.component_uuid} + data['name'] = self.name + data['type'] = self.type + data['parent'] = self.parent return data def dump_name(self) -> Dict: -- GitLab From cc8c3f8e435b708d4c21bede4341055fc9b33902 Mon Sep 17 00:00:00 2001 From: Armingol Date: Wed, 27 Sep 2023 09:26:52 +0200 Subject: [PATCH 09/13] Code cleanup --- proto/context.proto | 2 - src/context/service/database/Component.py | 5 +- src/context/service/database/Device.py | 5 +- .../service/database/models/ComponentModel.py | 27 +++++----- .../drivers/openconfig/templates/Inventory.py | 52 ++++++++----------- 5 files changed, 38 insertions(+), 53 deletions(-) diff --git a/proto/context.proto b/proto/context.proto index 0d7a66731..30fbe01c5 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -187,8 +187,6 @@ message Component { //Defined previously to this section string parent = 5; } -// ----------------------------------------------------- - message DeviceConfig { repeated ConfigRule config_rules = 1; } diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py index 3f2776787..ae873855b 100644 --- a/src/context/service/database/Component.py +++ b/src/context/service/database/Component.py @@ -1,5 +1,3 @@ - - # Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +14,6 @@ import datetime, json, logging from sqlalchemy import delete -#from sqlalchemy.dialects import postgresql from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from typing import Dict, List, Optional, Set @@ -63,7 +60,7 @@ def compose_components_data( 'device_uuid' : device_uuid, 'name' : name, 'type' : type_, - 'attributes' : json.dumps(attributes), #Store the remaining fields in 'attributes' + 'attributes' : json.dumps(attributes), 'parent' : parent, 'created_at' : now, 'updated_at' : now, diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index ed73b7f52..6da7c91bb 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime, logging -import json +import datetime, logging, json from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, selectinload, sessionmaker @@ -55,7 +54,6 @@ def device_list_objs(db_engine : Engine) -> DeviceList: .options(selectinload(DeviceModel.config_rules))\ .options(selectinload(DeviceModel.components))\ .all() - #.options(selectinload(DeviceModel.components))\ return [obj.dump() for obj in obj_list] devices = run_transaction(sessionmaker(bind=db_engine), callback) return DeviceList(devices=devices) @@ -67,7 +65,6 @@ def device_get(db_engine : Engine, request : DeviceId) -> Device: .options(selectinload(DeviceModel.endpoints))\ .options(selectinload(DeviceModel.config_rules))\ .filter_by(device_uuid=device_uuid).one_or_none() - #.options(selectinload(DeviceModel.components))\ return None if obj is None else obj.dump() obj = run_transaction(sessionmaker(bind=db_engine), callback) if obj is None: diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py index 60927e944..c9acfaeab 100644 --- a/src/context/service/database/models/ComponentModel.py +++ b/src/context/service/database/models/ComponentModel.py @@ -19,20 +19,19 @@ from sqlalchemy.orm import relationship from typing import Dict from ._Base import _Base -class ComponentModel(_Base): #Inherited functionality from the base class _Base - __tablename__ = 'device_component' #Name of the table in the DB associtaed with this model - - component_uuid = Column(UUID(as_uuid=False), primary_key=True) #Unique identifier that serves as a primary key for this table - device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele) - name = Column(String, nullable=False) #String field that stores the name of the component - type = Column(String, nullable=False) #String field that stores the type of the component - attributes = Column(String, nullable=False) #String field that stores the attributes of the component - parent = Column(String, nullable=False) #String field that stores the parent of the component - created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component - updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component - - device = relationship('DeviceModel', back_populates='components')# lazy='selectin'#Defines a relationship between ComponentModel and DeviceModel - #Represents a 1:n relationship where 1 device -> N components // The relationship is defined by the FK device_uuid +class ComponentModel(_Base): + __tablename__ = 'device_component' + + component_uuid = Column(UUID(as_uuid=False), primary_key=True) + device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) + name = Column(String, nullable=False) + type = Column(String, nullable=False) + attributes = Column(String, nullable=False) + parent = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + device = relationship('DeviceModel', back_populates='components') def dump_id(self) -> Dict: return{ 'device_id' : self.device.dump_id(), diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index b7e7c7c12..0d98d7288 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging, lxml.etree as ET +from typing import Any, Dict, List, Tuple +from .Namespace import NAMESPACES +from .Tools import add_value_from_tag + +LOGGER = logging.getLogger(__name__) + +XPATH_PORTS = "//ocp:components/ocp:component" """ #Method Name: parse @@ -22,39 +30,28 @@ # Functionality: -The parse function of the inventerio class has the functionality to parse -an XML document represented by the xml_data parameter and extract specific -information from the XML elements, namely the relevant characteristics of the -components. + The parse function of the inventerio class has the functionality to parse + an XML document represented by the xml_data parameter and extract specific + information from the XML elements, namely the relevant characteristics of the + components. -To generate the template the following steps are performed: + To generate the template the following steps are performed: -1) An empty list called response is created to store the results of the analysis. + 1) An empty list called response is created to store the results of the analysis. -2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS -expression. These elements represent components in the XML document. + 2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS + expression. These elements represent components in the XML document. -3) For each component element: -A dictionary called inventory is initialized that will store the information extracted -from the component.The values of the relevant XML elements are extracted and added to -the dictionary. + 3) For each component element: + A dictionary called inventory is initialized that will store the information extracted + from the component.The values of the relevant XML elements are extracted and added to + the dictionary. #Return: -List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) -with the information extracted from the XML document components is returned. - + List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) + with the information extracted from the XML document components is returned. """ - -import logging, lxml.etree as ET -from typing import Any, Dict, List, Tuple -from .Namespace import NAMESPACES -from .Tools import add_value_from_tag - -LOGGER = logging.getLogger(__name__) - -XPATH_PORTS = "//ocp:components/ocp:component" - def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] LOGGER.debug("InventoryPrueba") @@ -126,7 +123,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory['attributes'], 'mfg-date', component_mfg_date) #Transceiver Information - component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) if not component_serial_t is None: add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t) @@ -152,8 +148,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: component_reference.extend([parent_types[inventory['parent-component-references']]]) - - response.append(('/inventory/{:s}'.format(inventory['name']), inventory)) for tupla in response: @@ -162,4 +156,4 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: inventory['component-reference'] = component_reference - return response \ No newline at end of file + return response -- GitLab From e5d2f40702f17a4dd5d4fe6134134e4a28820f75 Mon Sep 17 00:00:00 2001 From: Armingol Date: Fri, 29 Sep 2023 14:40:57 +0200 Subject: [PATCH 10/13] 1) Code cleanup 2)Moving interface management to another branch --- src/context/service/database/ConfigRule.py | 1 - .../service/drivers/openconfig/templates/Interfaces.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index 1c5a2c7d5..c5b259a2d 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -34,7 +34,6 @@ def compose_config_rules_data( ) -> List[Dict]: dict_config_rules : List[Dict] = list() for position,config_rule in enumerate(config_rules): - str_kind = config_rule.WhichOneof('config_rule') kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member dict_config_rule = { diff --git a/src/device/service/drivers/openconfig/templates/Interfaces.py b/src/device/service/drivers/openconfig/templates/Interfaces.py index 8d1dcf16e..27048385e 100644 --- a/src/device/service/drivers/openconfig/templates/Interfaces.py +++ b/src/device/service/drivers/openconfig/templates/Interfaces.py @@ -98,15 +98,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: #add_value_from_collection(subinterface, 'ipv4_addresses', ipv4_addresses) - for xml_ipv6_address in xml_subinterface.xpath(XPATH_IPV6ADDRESSES, namespaces=NAMESPACES): - #LOGGER.info('xml_ipv6_address = {:s}'.format(str(ET.tostring(xml_ipv6_address)))) - - address = xml_ipv6_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES) - add_value_from_tag(subinterface, 'address_ipv6', address) - - prefix = xml_ipv6_address.find('ociip:state/ociip:prefix-length', namespaces=NAMESPACES) - add_value_from_tag(subinterface, 'address_prefix_v6', prefix, cast=int) - if len(subinterface) == 0: continue resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) response.append((resource_key, subinterface)) -- GitLab From e5980527b276d58bc18eb29c60e3fb14783be4db Mon Sep 17 00:00:00 2001 From: Armingol Date: Fri, 29 Sep 2023 14:42:09 +0200 Subject: [PATCH 11/13] Code cleanup --- src/device/service/drivers/openconfig/templates/Interfaces.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/device/service/drivers/openconfig/templates/Interfaces.py b/src/device/service/drivers/openconfig/templates/Interfaces.py index 27048385e..3855db17b 100644 --- a/src/device/service/drivers/openconfig/templates/Interfaces.py +++ b/src/device/service/drivers/openconfig/templates/Interfaces.py @@ -22,7 +22,6 @@ LOGGER = logging.getLogger(__name__) XPATH_INTERFACES = "//oci:interfaces/oci:interface" XPATH_SUBINTERFACES = ".//oci:subinterfaces/oci:subinterface" XPATH_IPV4ADDRESSES = ".//ociip:ipv4/ociip:addresses/ociip:address" -XPATH_IPV6ADDRESSES = ".//ociip:ipv6/ociip:addresses/ociip:address" def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] -- GitLab From d4d15c5d709c7c1932f1b555eb81fbe47921c2d6 Mon Sep 17 00:00:00 2001 From: Armingol Date: Fri, 29 Sep 2023 15:05:09 +0200 Subject: [PATCH 12/13] Code cleanup and fix bug in context/device.py --- src/context/service/ContextServiceServicerImpl.py | 4 +--- src/context/service/database/Device.py | 1 + src/device/service/drivers/openconfig/templates/Namespace.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index 6854d4ff8..93f078e75 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -124,9 +124,7 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ListDevices(self, request : Empty, context : grpc.ServicerContext) -> DeviceList: - devices = device_list_objs(self.db_engine) - LOGGER.info('DEVICES: {:s}'.format(str(devices))) - return DeviceList(devices=devices) + return device_list_objs(self.db_engine) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetDevice(self, request : ContextId, context : grpc.ServicerContext) -> Device: diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index 6da7c91bb..c4f2b3d95 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -64,6 +64,7 @@ def device_get(db_engine : Engine, request : DeviceId) -> Device: obj : Optional[DeviceModel] = session.query(DeviceModel)\ .options(selectinload(DeviceModel.endpoints))\ .options(selectinload(DeviceModel.config_rules))\ + .options(selectinload(DeviceModel.components))\ .filter_by(device_uuid=device_uuid).one_or_none() return None if obj is None else obj.dump() obj = run_transaction(sessionmaker(bind=db_engine), callback) diff --git a/src/device/service/drivers/openconfig/templates/Namespace.py b/src/device/service/drivers/openconfig/templates/Namespace.py index b70d5c327..bdc27a1ff 100644 --- a/src/device/service/drivers/openconfig/templates/Namespace.py +++ b/src/device/service/drivers/openconfig/templates/Namespace.py @@ -45,5 +45,5 @@ NAMESPACES = { 'ocpt2': NAMESPACE_POLICY_TYPES_2, 'ocrp' : NAMESPACE_ROUTING_POLICY, 'ocv' : NAMESPACE_VLAN, - 'ocptr' : NAMESPACE_PLATFORM_TRANSCEIVER, + 'ocptr': NAMESPACE_PLATFORM_TRANSCEIVER, } -- GitLab From 3e0555bb2da66ce1f644b239af28feb43604b9e5 Mon Sep 17 00:00:00 2001 From: Armingol Date: Fri, 29 Sep 2023 15:09:33 +0200 Subject: [PATCH 13/13] code cleanup --- src/context/service/database/Device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index c4f2b3d95..3aff20ade 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import datetime, logging, json +import datetime, logging from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, selectinload, sessionmaker -- GitLab