changes to save inventory in ddbb

proto/context.proto

@@ -179,23 +179,13 @@ message Device {
 }
 
 message Component {                         //Defined previously to this section - Tested OK
-  Uuid uuid = 1;
-  string name = 2;
-  string type = 3;
+  Uuid component_uuid   = 1;
+  string name           = 2;
+  string type           = 3;
   repeated string child = 4; // list[sub-component.name]
   map<string, string> attributes = 5; // dict[attr.name => json.dumps(attr.value)]
 }
 
-message ComponentId {                         //NEW
-  DeviceId device_id = 1;
-  Uuid endpoint_uuid = 2;
-}
-
-message ComponentIdList {                   //NEW
-  repeated ComponentId component_ids = 1;
-}
-
-
 // -----------------------------------------------------
 
 message DeviceConfig {

src/context/service/database/Component.py

@@ -30,10 +30,7 @@ from .uuids.EndPoint import endpoint_get_uuid
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
-from common.proto.context_pb2 import ComponentIdList
 from .models.ComponentModel import ComponentModel
-from .uuids.Component import component_get_uuid
-from .ConfigRule import compose_config_rules_data
 
 LOGGER = logging.getLogger(__name__)
 
@@ -44,121 +41,42 @@ def compose_components_data(
     dict_components : List[Dict] = list()
     for position,component in enumerate(components):
         str_kind = component.WhichOneof('config_rule')
-        LOGGER.info("DATA")
         message = (grpc_message_to_json_string(getattr(component, str_kind, {})))
         data = json.loads(message)
         resource_key = data["resource_key"]
         resource_value = data["resource_value"]
         if '/inventory' in resource_key:
-            LOGGER.info('Parametros: KEY',resource_key,'Value',resource_value)
+            resource_value_data = json.loads(resource_value)
+            name = resource_value_data.get('name')
+            type = resource_value_data.get('class')
+            uuid = resource_value_data.get('uuid')
+
+            if name is not None:
+                del resource_value_data['name']
+            if type is not None:
+                del resource_value_data['class']
+            if uuid is not None:
+                del resource_value_data['uuid']
+
+            attributes = resource_value_data  #Store the remaining fields in 'attributes'
+
             dict_component = {
-                'data'      : resource_value,
+                'name'      : name,
+                'type'      : type,
+                'attributes' : attributes,
                 'created_at': now,
                 'updated_at': now,
             }
 
-            parent_kind,parent_uuid = '',None
-            if device_uuid is not None:
-                dict_component['device_uuid'] = device_uuid
-                parent_kind,parent_uuid = 'device',device_uuid
-            elif service_uuid is not None:
-                dict_component['service_uuid'] = service_uuid
-                parent_kind,parent_uuid = 'service',service_uuid
-            elif slice_uuid is not None:
-                dict_component['slice_uuid'] = slice_uuid
-                parent_kind,parent_uuid = 'slice',slice_uuid
-            else:
-                MSG = 'Parent for Component({:s}) cannot be identified '+\
-                    '(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})'
-                str_component = grpc_message_to_json_string(component)
-                raise Exception(MSG.format(str_component, str(device_uuid), str(service_uuid), str(slice_uuid)))
-
-        
-            componenet_name = '{:s}:{:s}:{:s}'.format(parent_kind, 'custom', component.custom.resource_key)
+            dict_component['device_uuid'] = device_uuid
+            component_name = '{:s}:{:s}:{:s}'.format('device', 'custom', component.custom.resource_key)
             
             
-            component_uuid = get_uuid_from_string(componenet_name, prefix_for_name=parent_uuid)
+            component_uuid = get_uuid_from_string(component_name, prefix_for_name=device_uuid)
             dict_component['component_uuid'] = component_uuid
 
             dict_components.append(dict_component)
         else:
             continue
-    LOGGER.info('Parametros:',dict_components)
-    return dict_components
-
-'''
-def upsert_components(
-    session : Session, components : List[Dict], is_delete : bool = False,
-        device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None,
-) -> bool:
-    
-    if device_uuid is not None and service_uuid is None and slice_uuid is None:
-        klass = ComponentModel
-    
-    else:
-        MSG = 'DataModel cannot be identified (device_uuid={:s})'
-        raise Exception(MSG.format(str(device_uuid)))
-
-    uuids_to_upsert : Dict[str, int] = dict()
-    rules_to_upsert : List[Dict] = list()
-    for component in components:
-        component_uuid = component['component_uuid']
-        
-        position = uuids_to_upsert.get(component_uuid)
-        if position is None:
-            # if not added, add it
-            rules_to_upsert.append(component)
-            uuids_to_upsert[component_uuid] = len(rules_to_upsert) - 1
-        else:
-            # if already added, update occurrence
-            rules_to_upsert[position] = component
-        
-
-    upsert_affected = False
-    if len(rules_to_upsert) > 0:
-        stmt = insert(klass).values(rules_to_upsert)
-        stmt = stmt.on_conflict_do_update(
-            index_elements=[klass.component_uuid],
-            set_=dict(
-                data       = stmt.excluded.data,
-                updated_at = stmt.excluded.updated_at,
-            )
-        )
-        stmt = stmt.returning(klass.created_at, klass.updated_at)
-        #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})
-        #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt)))
-        components_updates = session.execute(stmt).fetchall()
-        upsert_affected = any([(updated_at > created_at) for created_at,updated_at in components_updates])
-
-    return upsert_affected
 
-    def component_list_names(db_engine: Engine, request: ComponentIdList) -> List[Dict]:
-    component_uuids = {
-        component_get_uuid(component_id, allow_random=False)[-1]
-        for component_id in request.component_ids
-    }
-
-    def callback(session: Session) -> List[Dict]:
-        obj_list: List[ComponentModel] = session.query(ComponentModel)\
-            .options(selectinload(ComponentModel.device))\
-            .filter(ComponentModel.component_uuid.in_(component_uuids)).all()
-        return [obj.dump_name() for obj in obj_list]
-
-    return run_transaction(sessionmaker(bind=db_engine), callback)
-
-def compose_components_data(data: Dict[str, any]) -> List[Dict]:
-    filtered_data = []
-
-    for item in data:
-        for key, value in item:
-            if any("inventory" in key):
-                filtered_data.append(item)
-
-    LOGGER.info("Filtered Data:")
-    LOGGER.info(filtered_data)
-
-
-
-    # Return the result
-    return filtered_data
-'''
+    return dict_components
\ No newline at end of file

src/context/service/database/ConfigRule.py

@@ -35,8 +35,8 @@ def compose_config_rules_data(
     dict_config_rules : List[Dict] = list()
     for position,config_rule in enumerate(config_rules):
         
-        LOGGER.info("REQUEST")
-        LOGGER.info(position,config_rule)
+        #LOGGER.info("REQUEST")
+        #LOGGER.info(position,config_rule)
         str_kind = config_rule.WhichOneof('config_rule')
         kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member
         dict_config_rule = {

src/context/service/database/Device.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import datetime, logging
+import json
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import Session, selectinload, sessionmaker
@@ -193,14 +194,14 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]:
 
         updated_components = False
         
-        LOGGER.info("HERE ERRPR DEBUG")
-        LOGGER.info(components_data)
         if len(components_data) > 0:
             stmt = insert(ComponentModel).values(components_data)
             stmt = stmt.on_conflict_do_update(
                 index_elements=[ComponentModel.component_uuid],
                 set_=dict(
-                    data             = stmt.excluded.data,
+                    name             = str(stmt.excluded.name),
+                    type             = str(stmt.excluded.type),
+                    attributes       = str(stmt.excluded.attributes),
                     updated_at       = stmt.excluded.updated_at,
                 )
             )

src/context/service/database/models/ComponentModel.py

@@ -24,8 +24,9 @@ class ComponentModel(_Base):                                            #Inherit
 
     component_uuid  = Column(UUID(as_uuid=False), primary_key=True)     #Unique identifier that serves as a primary key for this table
     device_uuid     = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True)  #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele)
-    # component_name  = Column(String, nullable=False)                    #String field that stores the name of the component
-    data            = Column(String, nullable=False)                    #String field that stores data about the component 
+    name            = Column(String, nullable=False)                    #String field that stores the name of the component
+    type            = Column(String, nullable=False)                    #String field that stores the name of the component
+    attributes      = Column(String, nullable=False)                    #String field that stores data about the component 
     created_at      = Column(DateTime, nullable=False)                  #Stores the creaton timestamp for the component
     updated_at      = Column(DateTime, nullable=False)                  #Stores the last upadted timestamp for the component
 
@@ -38,10 +39,11 @@ class ComponentModel(_Base):                                            #Inherit
         }
 
     def dump(self) -> Dict:
-        return {
-            'component_id'  : self.dump_id(),
-            'data'          : self.data,  
-        }
+        data['component_uuid'] = self.component_uuid
+        data['name'] = self.name
+        data['type'] = self.type
+        data['attributes'] = self.attributes
+        return data
 
     def dump_name(self) -> Dict:
         return {

src/context/service/database/models/DeviceModel.py

@@ -56,7 +56,7 @@ class DeviceModel(_Base):
         ]}
 
     def dump_components(self) -> List[Dict]:
-        return [component.dump() for component in self.components]
+        return {component.name:component.dump() for component in self.components}
 
     def dump(self,
         include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True,

src/context/service/database/models/_Base.py

@@ -61,7 +61,7 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None:
             'topology_name', 'created_at', 'updated_at'
         ]),
         index_storing('device_component_idx', 'device_component', ['device_uuid'], [
-            'data', 'created_at', 'updated_at'
+            'name', 'type', 'attributes', 'created_at', 'updated_at'
         ]),
     ]
     def callback(session : Session) -> bool:

src/context/service/database/uuids/Component.py

-# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Tuple
-from common.proto.context_pb2 import ComponentId
-from common.method_wrappers.ServiceExceptions import InvalidArgumentsException
-from ._Builder import get_uuid_from_string, get_uuid_random
-from .Device import device_get_uuid
-
-def component_get_uuid(
-    component_id: ComponentId, component_name: str = '', allow_random: bool = False
-) -> str:
-    device_uuid = device_get_uuid(component_id.device_id, allow_random=False)
-    raw_component_uuid = component_id.component_uuid.uuid
-
-    if raw_component_uuid:
-        prefix_for_name = f'{device_uuid}'
-        return get_uuid_from_string(raw_component_uuid, prefix_for_name=prefix_for_name)
-
-    if component_name:
-        prefix_for_name = f'{device_uuid}'
-        return get_uuid_from_string(component_name, prefix_for_name=prefix_for_name)
-
-    if allow_random:
-        return get_uuid_random()
-
-    raise InvalidArgumentsException(
-        [
-            ('component_id.component_uuid.uuid', raw_component_uuid),
-            ('name', component_name),
-        ],
-        extra_details=['At least one is required to produce a Component UUID']
-    )
-

src/device/service/drivers/openconfig/templates/Inventory.py

@@ -63,7 +63,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
         LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component))))
         inventory = {}
         inventory['parent-component-references'] = ''
-        inventory['uuid'] = ''
         inventory['name'] = ''    
         inventory['class'] = ''
         inventory['attributes'] = {}
@@ -71,16 +70,11 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
 
         component_name = xml_component.find('ocp:name', namespaces=NAMESPACES)
         if component_name is None or component_name.text is None: continue
-        add_value_from_tag(inventory, 'uuid', component_name)
         add_value_from_tag(inventory, 'name', component_name)        
 
         component_description = xml_component.find('ocp:state/ocp:description', namespaces=NAMESPACES)
         if not component_description is None:
             add_value_from_tag(inventory['attributes'], 'description', component_description)
-        #Podría estar presente en lugar del name
-        component_alias = xml_component.find('ocp:state/ocp:alias', namespaces=NAMESPACES)
-        if not component_alias is None:
-            add_value_from_tag(inventory['attributes'], 'alias', component_alias)
         
         component_location = xml_component.find('ocp:state/ocp:location', namespaces=NAMESPACES)
         if not component_location is None:
@@ -90,13 +84,12 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
         component_type.text = component_type.text.replace('oc-platform-types:','')
         if component_type is None: continue
         add_value_from_tag(inventory, 'class', component_type)
+        
         if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue
 
         component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES)
         if not component_empty is None:
-            add_value_from_tag(inventory['attributes'], 'contained-child', component_empty)
-
-        #añadir el parent pos
+            add_value_from_tag(inventory['attributes'], 'empty', component_empty)
 
         component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES)
         if component_parent is None or component_parent.text is None:
@@ -120,37 +113,24 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
         if not component_serial is None:
             add_value_from_tag(inventory['attributes'], 'serial-num', component_serial)
 
-        component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES)
-        if not component_serial_t is None:
-            add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t)
-
         component_mfg_name = xml_component.find('ocp:state/ocp:mfg-name', namespaces=NAMESPACES)
         if not component_mfg_name is None:
-            add_value_from_tag(inventory['attributes'], 'mfg-name', component_mfg_name)
-
-        component_part_no = xml_component.find('ocp:state/ocp:part-no', namespaces=NAMESPACES)
-        if not component_part_no is None:
-            add_value_from_tag(inventory['attributes'], 'part-number', component_part_no)
-
-        #modificar es un identificador de seguimiento de activos asignado por el usuario para el componente
-        component_asset_id = xml_component.find('ocp:state/ocp:asset-id', namespaces=NAMESPACES)
-        if not component_asset_id is None:
-            add_value_from_tag(inventory['attributes'], 'asset-id', component_asset_id)
+            add_value_from_tag(inventory['attributes'], 'manufacturer-name', component_mfg_name)
         
         component_removable = xml_component.find('ocp:state/ocp:removable', namespaces=NAMESPACES)
         if not component_removable is None:
-            add_value_from_tag(inventory['attributes'], 'is-fru', component_removable)
+            add_value_from_tag(inventory['attributes'], 'removable', component_removable)
 
         component_mfg_date = xml_component.find('ocp:state/ocp:mfg-date', namespaces=NAMESPACES)
         if not component_mfg_date is None:
             add_value_from_tag(inventory['attributes'], 'mfg-date', component_mfg_date)
 
-        #modificar "Este nodo contiene información de identificación sobre el componente"
-        component_uri = xml_component.find('ocp:state/ocp:uri', namespaces=NAMESPACES)
-        if not component_uri is None:
-            add_value_from_tag(inventory['attributes'], 'uri', component_uri)
-
-        #Para transceiver
+        #Transceiver Information
+        
+        component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES)
+        if not component_serial_t is None:
+            add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t)
+            
         component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES)
         if component_present is not None and 'NOT_PRESENT' in component_present.text: continue
         
@@ -166,7 +146,7 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
         if not component_form is None:
             component_form.text = component_form.text.replace('oc-opt-types:','')
             add_value_from_tag(inventory['attributes'], 'form-factor', component_form)
-        #añadir características del parent references
+
         if inventory['parent-component-references'] not in parent_types:
             parent_types[inventory['parent-component-references']] = len(parent_types) + 1
 
@@ -179,7 +159,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
         for tupla in response:
             if inventory['parent-component-references'] in tupla[0]:
                 component_reference.extend([tupla[1]['class']])
-                component_reference.extend([tupla[1]['uuid']])
 
         inventory['component-reference'] = component_reference