ddbb modification to store the device inventory (e17e8cc3) · Commits · TFS / controller

proto/context.proto

+19 −3

Original line number	Diff line number	Diff line
		@@ -174,14 +174,30 @@ message Device {
		DeviceOperationalStatusEnum device_operational_status = 5;
		repeated DeviceDriverEnum device_drivers = 6;
		repeated EndPoint device_endpoints = 7;
		repeated Component component = 8; // Used for inventory
		map<string, Component> components = 8; // Used for inventory
		DeviceId controller_id = 9; // Identifier of node controlling the actual device
		}

		message Component {
		repeated string comp_string = 1;
		message Component { //Defined previously to this section - Tested OK
		Uuid uuid = 1;
		string name = 2;
		string type = 3;
		repeated string child = 4; // list[sub-component.name]
		map<string, string> attributes = 5; // dict[attr.name => json.dumps(attr.value)]
		}

		message ComponentId { //NEW
		DeviceId device_id = 1;
		Uuid endpoint_uuid = 2;
		}

		message ComponentIdList { //NEW
		repeated ComponentId component_ids = 1;
		}


		// -----------------------------------------------------

		message DeviceConfig {
		repeated ConfigRule config_rules = 1;
		}

src/context/service/database/Component.py

0 → 100644

+164 −0

Original line number	Diff line number	Diff line



		# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import datetime, json, logging
		from sqlalchemy import delete
		#from sqlalchemy.dialects import postgresql
		from sqlalchemy.dialects.postgresql import insert
		from sqlalchemy.orm import Session
		from typing import Dict, List, Optional, Set
		from common.proto.context_pb2 import Component
		from common.proto.context_pb2 import ConfigRule
		from common.tools.grpc.Tools import grpc_message_to_json_string
		from .models.ComponentModel import ComponentModel
		from .uuids._Builder import get_uuid_from_string
		from .uuids.EndPoint import endpoint_get_uuid
		from sqlalchemy.engine import Engine
		from sqlalchemy.orm import Session, selectinload, sessionmaker
		from sqlalchemy_cockroachdb import run_transaction
		from common.proto.context_pb2 import ComponentIdList
		from .models.ComponentModel import ComponentModel
		from .uuids.Component import component_get_uuid
		from .ConfigRule import compose_config_rules_data

		LOGGER = logging.getLogger(__name__)

		def compose_components_data(
		components : List[Component], now : datetime.datetime,
		device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None
		) -> List[Dict]:
		dict_components : List[Dict] = list()
		for position,component in enumerate(components):
		str_kind = component.WhichOneof('config_rule')
		LOGGER.info("DATA")
		message = (grpc_message_to_json_string(getattr(component, str_kind, {})))
		data = json.loads(message)
		resource_key = data["resource_key"]
		resource_value = data["resource_value"]
		if '/inventory' in resource_key:
		LOGGER.info('Parametros: KEY',resource_key,'Value',resource_value)
		dict_component = {
		'data' : resource_value,
		'created_at': now,
		'updated_at': now,
		}

		parent_kind,parent_uuid = '',None
		if device_uuid is not None:
		dict_component['device_uuid'] = device_uuid
		parent_kind,parent_uuid = 'device',device_uuid
		elif service_uuid is not None:
		dict_component['service_uuid'] = service_uuid
		parent_kind,parent_uuid = 'service',service_uuid
		elif slice_uuid is not None:
		dict_component['slice_uuid'] = slice_uuid
		parent_kind,parent_uuid = 'slice',slice_uuid
		else:
		MSG = 'Parent for Component({:s}) cannot be identified '+\
		'(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})'
		str_component = grpc_message_to_json_string(component)
		raise Exception(MSG.format(str_component, str(device_uuid), str(service_uuid), str(slice_uuid)))


		componenet_name = '{:s}:{:s}:{:s}'.format(parent_kind, 'custom', component.custom.resource_key)


		component_uuid = get_uuid_from_string(componenet_name, prefix_for_name=parent_uuid)
		dict_component['component_uuid'] = component_uuid

		dict_components.append(dict_component)
		else:
		continue
		LOGGER.info('Parametros:',dict_components)
		return dict_components

		'''
		def upsert_components(
		session : Session, components : List[Dict], is_delete : bool = False,
		device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None,
		) -> bool:

		if device_uuid is not None and service_uuid is None and slice_uuid is None:
		klass = ComponentModel

		else:
		MSG = 'DataModel cannot be identified (device_uuid={:s})'
		raise Exception(MSG.format(str(device_uuid)))

		uuids_to_upsert : Dict[str, int] = dict()
		rules_to_upsert : List[Dict] = list()
		for component in components:
		component_uuid = component['component_uuid']

		position = uuids_to_upsert.get(component_uuid)
		if position is None:
		# if not added, add it
		rules_to_upsert.append(component)
		uuids_to_upsert[component_uuid] = len(rules_to_upsert) - 1
		else:
		# if already added, update occurrence
		rules_to_upsert[position] = component


		upsert_affected = False
		if len(rules_to_upsert) > 0:
		stmt = insert(klass).values(rules_to_upsert)
		stmt = stmt.on_conflict_do_update(
		index_elements=[klass.component_uuid],
		set_=dict(
		data = stmt.excluded.data,
		updated_at = stmt.excluded.updated_at,
		)
		)
		stmt = stmt.returning(klass.created_at, klass.updated_at)
		#str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})
		#LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt)))
		components_updates = session.execute(stmt).fetchall()
		upsert_affected = any([(updated_at > created_at) for created_at,updated_at in components_updates])

		return upsert_affected

		def component_list_names(db_engine: Engine, request: ComponentIdList) -> List[Dict]:
		component_uuids = {
		component_get_uuid(component_id, allow_random=False)[-1]
		for component_id in request.component_ids
		}

		def callback(session: Session) -> List[Dict]:
		obj_list: List[ComponentModel] = session.query(ComponentModel)\
		.options(selectinload(ComponentModel.device))\
		.filter(ComponentModel.component_uuid.in_(component_uuids)).all()
		return [obj.dump_name() for obj in obj_list]

		return run_transaction(sessionmaker(bind=db_engine), callback)

		def compose_components_data(data: Dict[str, any]) -> List[Dict]:
		filtered_data = []

		for item in data:
		for key, value in item:
		if any("inventory" in key):
		filtered_data.append(item)

		LOGGER.info("Filtered Data:")
		LOGGER.info(filtered_data)



		# Return the result
		return filtered_data
		'''
		No newline at end of file

src/context/service/database/ConfigRule.py

+3 −0

Original line number	Diff line number	Diff line
		@@ -34,6 +34,9 @@ def compose_config_rules_data(
		) -> List[Dict]:
		dict_config_rules : List[Dict] = list()
		for position,config_rule in enumerate(config_rules):

		LOGGER.info("REQUEST")
		LOGGER.info(position,config_rule)
		str_kind = config_rule.WhichOneof('config_rule')
		kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member
		dict_config_rule = {

src/context/service/database/Device.py

+23 −2

Original line number	Diff line number	Diff line
		@@ -26,12 +26,14 @@ from context.service.database.uuids.Topology import topology_get_uuid
		from .models.DeviceModel import DeviceModel
		from .models.EndPointModel import EndPointModel
		from .models.TopologyModel import TopologyDeviceModel
		from .models.ComponentModel import ComponentModel
		from .models.enums.DeviceDriver import grpc_to_enum__device_driver
		from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status
		from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type
		from .uuids.Device import device_get_uuid
		from .uuids.EndPoint import endpoint_get_uuid
		from .ConfigRule import compose_config_rules_data, upsert_config_rules
		from .Component import compose_components_data

		LOGGER = logging.getLogger(__name__)

		@@ -46,6 +48,7 @@ def device_list_objs(db_engine : Engine) -> List[Dict]:
		obj_list : List[DeviceModel] = session.query(DeviceModel)\
		.options(selectinload(DeviceModel.endpoints))\
		.options(selectinload(DeviceModel.config_rules))\
		.options(selectinload(DeviceModel.components))\
		.all()
		#.options(selectinload(DeviceModel.components))\
		return [obj.dump() for obj in obj_list]
		@@ -133,6 +136,7 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]:
		})
		topology_uuids.add(endpoint_topology_uuid)

		components_data = compose_components_data(request.device_config.config_rules, now, device_uuid=device_uuid)
		config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid)

		device_data = [{
		@@ -187,9 +191,26 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]:
		index_elements=[TopologyDeviceModel.topology_uuid, TopologyDeviceModel.device_uuid]
		))

		updated_components = False

		LOGGER.info("HERE ERRPR DEBUG")
		LOGGER.info(components_data)
		if len(components_data) > 0:
		stmt = insert(ComponentModel).values(components_data)
		stmt = stmt.on_conflict_do_update(
		index_elements=[ComponentModel.component_uuid],
		set_=dict(
		data = stmt.excluded.data,
		updated_at = stmt.excluded.updated_at,
		)
		)
		stmt = stmt.returning(ComponentModel.created_at, ComponentModel.updated_at)
		component_updates = session.execute(stmt).fetchall()
		updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates])

		changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid)

		return updated or updated_endpoints or changed_config_rules
		return updated or updated_components or changed_config_rules

		updated = run_transaction(sessionmaker(bind=db_engine), callback)
		return json_device_id(device_uuid),updated

src/context/service/database/models/ComponentModel.py

0 → 100644

+51 −0

Original line number	Diff line number	Diff line
		# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import json
		from sqlalchemy import Column, DateTime, ForeignKey, String
		from sqlalchemy.dialects.postgresql import UUID
		from sqlalchemy.orm import relationship
		from typing import Dict
		from ._Base import _Base

		class ComponentModel(_Base): #Inherited functionality from the base class _Base
		__tablename__ = 'device_component' #Name of the table in the DB associtaed with this model

		component_uuid = Column(UUID(as_uuid=False), primary_key=True) #Unique identifier that serves as a primary key for this table
		device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele)
		# component_name = Column(String, nullable=False) #String field that stores the name of the component
		data = Column(String, nullable=False) #String field that stores data about the component
		created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component
		updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component

		device = relationship('DeviceModel', back_populates='components')# lazy='selectin'#Defines a relationship between ComponentModel and DeviceModel
		#Represents a 1:n relationship where 1 device -> N components // The relationship is defined by the FK device_uuid
		def dump_id(self) -> Dict:
		return{
		'device_id' : self.device.dump_id(),
		'component_uuid': {'uuid': self.component_uuid},
		}

		def dump(self) -> Dict:
		return {
		'component_id' : self.dump_id(),
		'data' : self.data,
		}

		def dump_name(self) -> Dict:
		return {
		'component_id' : self.dump_id(),
		'device_name' : self.device.device_name,
		'component_name': self.name,
		}