Commit eaab9562 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/tid-device-inventory' into 'develop' 

Resolve "(TID) Network device inventory management"

See merge request !150
parents 142d50f1 63150a36

proto/context.proto

+8 −3
Original line number Diff line number Diff line
@@ -174,12 +174,17 @@ message Device { 
  DeviceOperationalStatusEnum device_operational_status = 5;

  repeated DeviceDriverEnum device_drivers = 6;

  repeated EndPoint device_endpoints = 7;

  repeated Component component = 8; // Used for inventory

  repeated Component components = 8; // Used for inventory

  DeviceId controller_id = 9; // Identifier of node controlling the actual device

}



message Component {

  repeated string comp_string = 1;

message Component {                         //Defined previously to this section - Tested OK

  Uuid component_uuid   = 1;

  string name           = 2;

  string type           = 3;

  

  map<string, string> attributes = 4; // dict[attr.name => json.dumps(attr.value)]

  string parent         = 5;

}



message DeviceConfig {

src/context/service/database/Component.py

0 → 100644
+69 −0
Original line number Diff line number Diff line 
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



import datetime, json, logging

from sqlalchemy import delete

from sqlalchemy.dialects.postgresql import insert

from sqlalchemy.orm import Session

from typing import Dict, List, Optional, Set

from common.proto.context_pb2 import Component

from common.proto.context_pb2 import ConfigRule

from common.tools.grpc.Tools import grpc_message_to_json_string

from .models.ComponentModel import ComponentModel

from .uuids._Builder import get_uuid_from_string

from .uuids.EndPoint import endpoint_get_uuid

from sqlalchemy.engine import Engine

from sqlalchemy.orm import Session, selectinload, sessionmaker

from sqlalchemy_cockroachdb import run_transaction

from .models.ComponentModel import ComponentModel



LOGGER = logging.getLogger(__name__)



def compose_components_data(

    components : List[Component], now : datetime.datetime,

    device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None

) -> List[Dict]:

    dict_components : List[Dict] = list()

    for position,component in enumerate(components):

        str_kind = component.WhichOneof('config_rule')

        message  = (grpc_message_to_json_string(getattr(component, str_kind, {})))

        data     = json.loads(message)

        resource_key   = data["resource_key"]

        resource_value = data["resource_value"]

        if '/inventory' in resource_key:

            resource_value_data = json.loads(resource_value)

            name                = resource_value_data.pop('name', None)

            type_               = resource_value_data.pop('class', None)

            parent              = resource_value_data.pop('parent-component-references', None)

            attributes          = resource_value_data.pop('attributes', {})

            if len(resource_value_data) > 0:

                LOGGER.warning('Discarding Component Leftovers: {:s}'.format(str(resource_value_data)))



            attributes = {

                attr_name:json.dumps(attr_value)

                for attr_name,attr_value in attributes.items()

            }

            component_uuid = get_uuid_from_string(component.custom.resource_key, prefix_for_name=device_uuid)            

            dict_component = {

                'component_uuid': component_uuid,

                'device_uuid'   : device_uuid,

                'name'          : name,

                'type'          : type_,

                'attributes'    : json.dumps(attributes),

                'parent'        : parent,

                'created_at'    : now,

                'updated_at'    : now,

            }

            dict_components.append(dict_component)

    return dict_components

src/context/service/database/Device.py

+24 −3
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ from common.tools.object_factory.Device import json_device_id 
from context.service.database.uuids.Topology import topology_get_uuid

from .models.DeviceModel import DeviceModel

from .models.EndPointModel import EndPointModel

from .models.ComponentModel import ComponentModel

from .models.TopologyModel import TopologyDeviceModel, TopologyModel

from .models.enums.DeviceDriver import grpc_to_enum__device_driver

from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status
@@ -34,6 +35,7 @@ from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type 
from .uuids.Device import device_get_uuid

from .uuids.EndPoint import endpoint_get_uuid

from .ConfigRule import compose_config_rules_data, upsert_config_rules

from .Component import compose_components_data

from .Events import notify_event_context, notify_event_device, notify_event_topology



LOGGER = logging.getLogger(__name__)
@@ -50,8 +52,8 @@ def device_list_objs(db_engine : Engine) -> DeviceList: 
        obj_list : List[DeviceModel] = session.query(DeviceModel)\

            .options(selectinload(DeviceModel.endpoints))\

            .options(selectinload(DeviceModel.config_rules))\

            .options(selectinload(DeviceModel.components))\

            .all()

            #.options(selectinload(DeviceModel.components))\

        return [obj.dump() for obj in obj_list]

    devices = run_transaction(sessionmaker(bind=db_engine), callback)

    return DeviceList(devices=devices)
@@ -62,8 +64,8 @@ def device_get(db_engine : Engine, request : DeviceId) -> Device: 
        obj : Optional[DeviceModel] = session.query(DeviceModel)\

            .options(selectinload(DeviceModel.endpoints))\

            .options(selectinload(DeviceModel.config_rules))\

            .options(selectinload(DeviceModel.components))\

            .filter_by(device_uuid=device_uuid).one_or_none()

            #.options(selectinload(DeviceModel.components))\

        return None if obj is None else obj.dump()

    obj = run_transaction(sessionmaker(bind=db_engine), callback)

    if obj is None:
@@ -138,6 +140,7 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi 
            })

            topology_uuids.add(endpoint_topology_uuid)



    components_data = compose_components_data(request.device_config.config_rules, now, device_uuid=device_uuid)

    config_rules    = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid)



    device_data = [{
@@ -206,6 +209,24 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi 
                device_topology_ids = [obj.dump_id() for obj in device_topologies]

                LOGGER.warning('device_topology_ids={:s}'.format(str(device_topology_ids)))



        updated_components = False

        

        if len(components_data) > 0:

            stmt = insert(ComponentModel).values(components_data)

            stmt = stmt.on_conflict_do_update(

                index_elements=[ComponentModel.component_uuid],

                set_=dict(

                    name             = stmt.excluded.name,

                    type             = stmt.excluded.type,

                    attributes       = stmt.excluded.attributes,

                    parent           = stmt.excluded.parent,

                    updated_at       = stmt.excluded.updated_at,

                )

            )

            stmt = stmt.returning(ComponentModel.created_at, ComponentModel.updated_at)

            component_updates = session.execute(stmt).fetchall()

            updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates])

        

        changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid)



        return updated or updated_endpoints or changed_config_rules, device_topology_ids

src/context/service/database/models/ComponentModel.py

0 → 100644
+55 −0
Original line number Diff line number Diff line 
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



import json

from sqlalchemy import Column, DateTime, ForeignKey, String

from sqlalchemy.dialects.postgresql import UUID

from sqlalchemy.orm import relationship

from typing import Dict

from ._Base import _Base                                                



class ComponentModel(_Base):

    __tablename__ = 'device_component'

    

    component_uuid  = Column(UUID(as_uuid=False), primary_key=True)

    device_uuid     = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True)

    name            = Column(String, nullable=False)

    type            = Column(String, nullable=False)

    attributes      = Column(String, nullable=False)

    parent          = Column(String, nullable=False)

    created_at      = Column(DateTime, nullable=False)

    updated_at      = Column(DateTime, nullable=False)

    

    device           = relationship('DeviceModel', back_populates='components')

    def dump_id(self) -> Dict:

        return{

            'device_id'     : self.device.dump_id(),

            'component_uuid': {'uuid': self.component_uuid},

        }



    def dump(self) -> Dict:

        data = dict()

        data['attributes']     = json.loads(self.attributes)

        data['component_uuid'] = {'uuid': self.component_uuid}

        data['name']           = self.name

        data['type']           = self.type

        data['parent']         = self.parent

        return data



    def dump_name(self) -> Dict:

        return {

            'component_id'  : self.dump_id(),

            'device_name'   : self.device.device_name,

            'component_name': self.name,

        }

src/context/service/database/models/DeviceModel.py

+3 −2
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ class DeviceModel(_Base): 
    #topology_devices = relationship('TopologyDeviceModel', back_populates='device')

    config_rules = relationship('DeviceConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device'

    endpoints    = relationship('EndPointModel', passive_deletes=True) # lazy='joined', back_populates='device'

    components   = relationship('ComponentModel', passive_deletes=True) # lazy='joined', back_populates='device' 

    controller   = relationship('DeviceModel', remote_side=[device_uuid], passive_deletes=True) # lazy='joined', back_populates='device'



    def dump_id(self) -> Dict:
@@ -55,7 +56,7 @@ class DeviceModel(_Base): 
        ]}



    def dump_components(self) -> List[Dict]:

        return []

        return [component.dump() for component in self.components]



    def dump(self,

        include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True,
@@ -70,5 +71,5 @@ class DeviceModel(_Base): 
        }

        if include_endpoints: result['device_endpoints'] = self.dump_endpoints()

        if include_config_rules: result['device_config'] = self.dump_config_rules()

        if include_components: result['component'] = self.dump_components()

        if include_components: result['components'] = self.dump_components()

        return result