Merge branch 'feat/tid-device-inventory' into 'develop'

Resolve "(TID) Network device inventory management"

See merge request !150

proto/context.proto

@@ -174,12 +174,17 @@ message Device {
   DeviceOperationalStatusEnum device_operational_status = 5;
   repeated DeviceDriverEnum device_drivers = 6;
   repeated EndPoint device_endpoints = 7;
-  repeated Component component = 8; // Used for inventory
+  repeated Component components = 8; // Used for inventory
   DeviceId controller_id = 9; // Identifier of node controlling the actual device
 }
 
-message Component {
-  repeated string comp_string = 1;
+message Component {                         //Defined previously to this section - Tested OK
+  Uuid component_uuid   = 1;
+  string name           = 2;
+  string type           = 3;
+  
+  map<string, string> attributes = 4; // dict[attr.name => json.dumps(attr.value)]
+  string parent         = 5;
 }
 
 message DeviceConfig {

src/context/service/database/Component.py

+# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime, json, logging
+from sqlalchemy import delete
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.orm import Session
+from typing import Dict, List, Optional, Set
+from common.proto.context_pb2 import Component
+from common.proto.context_pb2 import ConfigRule
+from common.tools.grpc.Tools import grpc_message_to_json_string
+from .models.ComponentModel import ComponentModel
+from .uuids._Builder import get_uuid_from_string
+from .uuids.EndPoint import endpoint_get_uuid
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session, selectinload, sessionmaker
+from sqlalchemy_cockroachdb import run_transaction
+from .models.ComponentModel import ComponentModel
+
+LOGGER = logging.getLogger(__name__)
+
+def compose_components_data(
+    components : List[Component], now : datetime.datetime,
+    device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None
+) -> List[Dict]:
+    dict_components : List[Dict] = list()
+    for position,component in enumerate(components):
+        str_kind = component.WhichOneof('config_rule')
+        message  = (grpc_message_to_json_string(getattr(component, str_kind, {})))
+        data     = json.loads(message)
+        resource_key   = data["resource_key"]
+        resource_value = data["resource_value"]
+        if '/inventory' in resource_key:
+            resource_value_data = json.loads(resource_value)
+            name                = resource_value_data.pop('name', None)
+            type_               = resource_value_data.pop('class', None)
+            parent              = resource_value_data.pop('parent-component-references', None)
+            attributes          = resource_value_data.pop('attributes', {})
+            if len(resource_value_data) > 0:
+                LOGGER.warning('Discarding Component Leftovers: {:s}'.format(str(resource_value_data)))
+
+            attributes = {
+                attr_name:json.dumps(attr_value)
+                for attr_name,attr_value in attributes.items()
+            }
+            component_uuid = get_uuid_from_string(component.custom.resource_key, prefix_for_name=device_uuid)            
+            dict_component = {
+                'component_uuid': component_uuid,
+                'device_uuid'   : device_uuid,
+                'name'          : name,
+                'type'          : type_,
+                'attributes'    : json.dumps(attributes),
+                'parent'        : parent,
+                'created_at'    : now,
+                'updated_at'    : now,
+            }
+            dict_components.append(dict_component)
+    return dict_components

src/context/service/database/Device.py

@@ -27,6 +27,7 @@ from common.tools.object_factory.Device import json_device_id
 from context.service.database.uuids.Topology import topology_get_uuid
 from .models.DeviceModel import DeviceModel
 from .models.EndPointModel import EndPointModel
+from .models.ComponentModel import ComponentModel
 from .models.TopologyModel import TopologyDeviceModel, TopologyModel
 from .models.enums.DeviceDriver import grpc_to_enum__device_driver
 from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status
@@ -34,6 +35,7 @@ from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type
 from .uuids.Device import device_get_uuid
 from .uuids.EndPoint import endpoint_get_uuid
 from .ConfigRule import compose_config_rules_data, upsert_config_rules
+from .Component import compose_components_data
 from .Events import notify_event_context, notify_event_device, notify_event_topology
 
 LOGGER = logging.getLogger(__name__)
@@ -50,8 +52,8 @@ def device_list_objs(db_engine : Engine) -> DeviceList:
         obj_list : List[DeviceModel] = session.query(DeviceModel)\
             .options(selectinload(DeviceModel.endpoints))\
             .options(selectinload(DeviceModel.config_rules))\
+            .options(selectinload(DeviceModel.components))\
             .all()
-            #.options(selectinload(DeviceModel.components))\
         return [obj.dump() for obj in obj_list]
     devices = run_transaction(sessionmaker(bind=db_engine), callback)
     return DeviceList(devices=devices)
@@ -62,8 +64,8 @@ def device_get(db_engine : Engine, request : DeviceId) -> Device:
         obj : Optional[DeviceModel] = session.query(DeviceModel)\
             .options(selectinload(DeviceModel.endpoints))\
             .options(selectinload(DeviceModel.config_rules))\
+            .options(selectinload(DeviceModel.components))\
             .filter_by(device_uuid=device_uuid).one_or_none()
-            #.options(selectinload(DeviceModel.components))\
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)
     if obj is None:
@@ -138,7 +140,8 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi
             })
             topology_uuids.add(endpoint_topology_uuid)
 
-    config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid)
+    components_data = compose_components_data(request.device_config.config_rules, now, device_uuid=device_uuid)
+    config_rules    = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid)
 
     device_data = [{
         'device_uuid'              : device_uuid,
@@ -206,6 +209,24 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi
                 device_topology_ids = [obj.dump_id() for obj in device_topologies]
                 LOGGER.warning('device_topology_ids={:s}'.format(str(device_topology_ids)))
 
+        updated_components = False
+        
+        if len(components_data) > 0:
+            stmt = insert(ComponentModel).values(components_data)
+            stmt = stmt.on_conflict_do_update(
+                index_elements=[ComponentModel.component_uuid],
+                set_=dict(
+                    name             = stmt.excluded.name,
+                    type             = stmt.excluded.type,
+                    attributes       = stmt.excluded.attributes,
+                    parent           = stmt.excluded.parent,
+                    updated_at       = stmt.excluded.updated_at,
+                )
+            )
+            stmt = stmt.returning(ComponentModel.created_at, ComponentModel.updated_at)
+            component_updates = session.execute(stmt).fetchall()
+            updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates])
+        
         changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid)
 
         return updated or updated_endpoints or changed_config_rules, device_topology_ids

src/context/service/database/models/ComponentModel.py

+# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from sqlalchemy import Column, DateTime, ForeignKey, String
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import relationship
+from typing import Dict
+from ._Base import _Base                                                
+
+class ComponentModel(_Base):
+    __tablename__ = 'device_component'
+    
+    component_uuid  = Column(UUID(as_uuid=False), primary_key=True)
+    device_uuid     = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True)
+    name            = Column(String, nullable=False)
+    type            = Column(String, nullable=False)
+    attributes      = Column(String, nullable=False)
+    parent          = Column(String, nullable=False)
+    created_at      = Column(DateTime, nullable=False)
+    updated_at      = Column(DateTime, nullable=False)
+    
+    device           = relationship('DeviceModel', back_populates='components')
+    def dump_id(self) -> Dict:
+        return{
+            'device_id'     : self.device.dump_id(),
+            'component_uuid': {'uuid': self.component_uuid},
+        }
+
+    def dump(self) -> Dict:
+        data = dict()
+        data['attributes']     = json.loads(self.attributes)
+        data['component_uuid'] = {'uuid': self.component_uuid}
+        data['name']           = self.name
+        data['type']           = self.type
+        data['parent']         = self.parent
+        return data
+
+    def dump_name(self) -> Dict:
+        return {
+            'component_id'  : self.dump_id(),
+            'device_name'   : self.device.device_name,
+            'component_name': self.name,
+        }

src/context/service/database/models/DeviceModel.py

@@ -36,6 +36,7 @@ class DeviceModel(_Base):
     #topology_devices = relationship('TopologyDeviceModel', back_populates='device')
     config_rules = relationship('DeviceConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device'
     endpoints    = relationship('EndPointModel', passive_deletes=True) # lazy='joined', back_populates='device'
+    components   = relationship('ComponentModel', passive_deletes=True) # lazy='joined', back_populates='device' 
     controller   = relationship('DeviceModel', remote_side=[device_uuid], passive_deletes=True) # lazy='joined', back_populates='device'
 
     def dump_id(self) -> Dict:
@@ -55,7 +56,7 @@ class DeviceModel(_Base):
         ]}
 
     def dump_components(self) -> List[Dict]:
-        return []
+        return [component.dump() for component in self.components]
 
     def dump(self,
         include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True,
@@ -70,5 +71,5 @@ class DeviceModel(_Base):
         }
         if include_endpoints: result['device_endpoints'] = self.dump_endpoints()
         if include_config_rules: result['device_config'] = self.dump_config_rules()
-        if include_components: result['component'] = self.dump_components()
+        if include_components: result['components'] = self.dump_components()
         return result

src/context/service/database/models/_Base.py

@@ -60,6 +60,9 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None:
         index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [
             'topology_name', 'created_at', 'updated_at'
         ]),
+        index_storing('device_component_idx', 'device_component', ['device_uuid'], [
+            'name', 'type', 'attributes', 'created_at', 'updated_at'
+        ]),
     ]
     def callback(session : Session) -> bool:
         for stmt in statements: session.execute(stmt)

src/device/service/driver_api/_Driver.py

@@ -24,6 +24,7 @@ RESOURCE_NETWORK_INSTANCES = '__network_instances__'
 RESOURCE_ROUTING_POLICIES = '__routing_policies__'
 RESOURCE_SERVICES = '__services__'
 RESOURCE_ACL = '__acl__'
+RESOURCE_INVENTORY = '__inventory__'
 
 
 class _Driver:

src/device/service/drivers/openconfig/templates/Inventory.py

+# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging, lxml.etree as ET
+from typing import Any, Dict, List, Tuple
+from .Namespace import NAMESPACES
+from .Tools import add_value_from_tag
+
+LOGGER = logging.getLogger(__name__)
+
+XPATH_PORTS = "//ocp:components/ocp:component"
+
+"""
+#Method Name: parse
+
+#Parameters:
+    
+    - xml_data: [ET.Element] Represents the XML data to be parsed.
+
+# Functionality:
+
+    The parse function of the inventerio class has the functionality to parse
+    an XML document represented by the xml_data parameter and extract specific 
+    information from the XML elements, namely the relevant characteristics of the 
+    components.     
+
+    To generate the template the following steps are performed:
+
+    1) An empty list called response is created to store the results of the analysis.
+
+    2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS 
+    expression. These elements represent components in the XML document.
+
+    3) For each component element:
+    A dictionary called inventory is initialized that will store the information extracted 
+    from the component.The values of the relevant XML elements are extracted and added to
+    the dictionary.
+
+#Return: 
+    List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) 
+    with the information extracted from the XML document components is returned.
+"""
+
+def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
+    response = []
+    LOGGER.debug("InventoryPrueba")
+    parent_types = {}
+    for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES):
+        LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component))))
+        inventory = {}
+        inventory['parent-component-references'] = ''
+        inventory['name'] = ''    
+        inventory['class'] = ''
+        inventory['attributes'] = {}
+        component_reference = []
+
+        component_name = xml_component.find('ocp:name', namespaces=NAMESPACES)
+        if component_name is None or component_name.text is None: continue
+        add_value_from_tag(inventory, 'name', component_name)        
+
+        component_description = xml_component.find('ocp:state/ocp:description', namespaces=NAMESPACES)
+        if not component_description is None:
+            add_value_from_tag(inventory['attributes'], 'description', component_description)
+        
+        component_location = xml_component.find('ocp:state/ocp:location', namespaces=NAMESPACES)
+        if not component_location is None:
+            add_value_from_tag(inventory['attributes'], 'location', component_location)
+        
+        component_type = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES)
+        component_type.text = component_type.text.replace('oc-platform-types:','')
+        if component_type is None: continue
+        add_value_from_tag(inventory, 'class', component_type)
+        
+        if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue
+
+        component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES)
+        if not component_empty is None:
+            add_value_from_tag(inventory['attributes'], 'empty', component_empty)
+
+        component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES)
+        if component_parent is None or component_parent.text is None:
+            add_value_from_tag(inventory, 'parent-component-references', component_type)
+        else:
+            add_value_from_tag(inventory, 'parent-component-references', component_parent)
+
+        component_HW = xml_component.find('ocp:state/ocp:hardware-version', namespaces=NAMESPACES)
+        if not component_HW is None:
+            add_value_from_tag(inventory['attributes'], 'hardware-rev', component_HW)
+
+        component_firmware_version = xml_component.find('ocp:state/ocp:firmware-version', namespaces=NAMESPACES)
+        if not component_firmware_version is None:
+            add_value_from_tag(inventory['attributes'], 'firmware-rev', component_firmware_version)
+
+        component_SW = xml_component.find('ocp:state/ocp:software-version', namespaces=NAMESPACES)
+        if not component_SW is None:
+            add_value_from_tag(inventory['attributes'], 'software-rev', component_SW)
+
+        component_serial = xml_component.find('ocp:state/ocp:serial-no', namespaces=NAMESPACES)
+        if not component_serial is None:
+            add_value_from_tag(inventory['attributes'], 'serial-num', component_serial)
+
+        component_mfg_name = xml_component.find('ocp:state/ocp:mfg-name', namespaces=NAMESPACES)
+        if not component_mfg_name is None:
+            add_value_from_tag(inventory['attributes'], 'manufacturer-name', component_mfg_name)
+        
+        component_removable = xml_component.find('ocp:state/ocp:removable', namespaces=NAMESPACES)
+        if not component_removable is None:
+            add_value_from_tag(inventory['attributes'], 'removable', component_removable)
+
+        component_mfg_date = xml_component.find('ocp:state/ocp:mfg-date', namespaces=NAMESPACES)
+        if not component_mfg_date is None:
+            add_value_from_tag(inventory['attributes'], 'mfg-date', component_mfg_date)
+
+        #Transceiver Information
+        component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES)
+        if not component_serial_t is None:
+            add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t)
+            
+        component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES)
+        if component_present is not None and 'NOT_PRESENT' in component_present.text: continue
+        
+        component_vendor = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:vendor', namespaces=NAMESPACES)
+        if not component_vendor is None:
+            add_value_from_tag(inventory['attributes'], 'vendor', component_vendor)
+        component_connector = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:connector-type', namespaces=NAMESPACES)
+        if not component_connector is None:
+            component_connector.text = component_connector.text.replace('oc-opt-types:','')
+            add_value_from_tag(inventory['attributes'], 'connector-type', component_connector)
+        
+        component_form = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:form-factor', namespaces=NAMESPACES)
+        if not component_form is None:
+            component_form.text = component_form.text.replace('oc-opt-types:','')
+            add_value_from_tag(inventory['attributes'], 'form-factor', component_form)
+
+        if inventory['parent-component-references'] not in parent_types:
+            parent_types[inventory['parent-component-references']] = len(parent_types) + 1
+
+        component_reference.extend([parent_types[inventory['parent-component-references']]])
+        
+        response.append(('/inventory/{:s}'.format(inventory['name']), inventory))
+
+        for tupla in response:
+            if inventory['parent-component-references'] in tupla[0]:
+                component_reference.extend([tupla[1]['class']])
+
+        inventory['component-reference'] = component_reference
+        
+    return response

src/device/service/drivers/openconfig/templates/Namespace.py

@@ -28,6 +28,7 @@ NAMESPACE_POLICY_TYPES           = 'http://openconfig.net/yang/policy-types'
 NAMESPACE_POLICY_TYPES_2         = 'http://openconfig.net/yang/policy_types'
 NAMESPACE_ROUTING_POLICY         = 'http://openconfig.net/yang/routing-policy'
 NAMESPACE_VLAN                   = 'http://openconfig.net/yang/vlan'
+NAMESPACE_PLATFORM_TRANSCEIVER   = 'http://openconfig.net/yang/platform/transceiver'
 
 NAMESPACES = {
     'nc'   : NAMESPACE_NETCONF,
@@ -44,4 +45,5 @@ NAMESPACES = {
     'ocpt2': NAMESPACE_POLICY_TYPES_2,
     'ocrp' : NAMESPACE_ROUTING_POLICY,
     'ocv'  : NAMESPACE_VLAN,
+    'ocptr': NAMESPACE_PLATFORM_TRANSCEIVER,
 }

src/device/service/drivers/openconfig/templates/__init__.py

@@ -20,15 +20,17 @@ from jinja2 import Environment, PackageLoader, select_autoescape
 import paramiko
 from .Tools import generate_templates
 from device.service.driver_api._Driver import (
-    RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL)
+    RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY)
 from .EndPoints import parse as parse_endpoints
 from .Interfaces import parse as parse_interfaces, parse_counters
 from .NetworkInstances import parse as parse_network_instances
 from .RoutingPolicy import parse as parse_routing_policy
 from .Acl import parse as parse_acl
+from .Inventory import parse as parse_inventory
 LOGGER = logging.getLogger(__name__)
 
 ALL_RESOURCE_KEYS = [
+    RESOURCE_INVENTORY,
     RESOURCE_ENDPOINTS,
     RESOURCE_INTERFACES,
     RESOURCE_ROUTING_POLICIES,      # routing policies should come before network instances
@@ -37,6 +39,7 @@ ALL_RESOURCE_KEYS = [
 ]
 
 RESOURCE_KEY_MAPPINGS = {
+    RESOURCE_INVENTORY        : 'inventory',
     RESOURCE_ENDPOINTS        : 'component',
     RESOURCE_INTERFACES       : 'interface',
     RESOURCE_NETWORK_INSTANCES: 'network_instance',
@@ -45,6 +48,7 @@ RESOURCE_KEY_MAPPINGS = {
 }
 
 RESOURCE_PARSERS = {
+    'inventory'       : parse_inventory,
     'component'       : parse_endpoints,
     'interface'       : parse_interfaces,
     'network_instance': parse_network_instances,

src/device/service/drivers/openconfig/templates/inventory/get.xml

+<components xmlns="http://openconfig.net/yang/platform">
+    <component/>
+</components>