diff --git a/proto/context.proto b/proto/context.proto index 3ca8d3e20c7899c6c16bed1b3c626279a9b11bd9..22e11bc68b840115a19551958ac322acb71fb9a4 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -174,12 +174,17 @@ message Device { DeviceOperationalStatusEnum device_operational_status = 5; repeated DeviceDriverEnum device_drivers = 6; repeated EndPoint device_endpoints = 7; - repeated Component component = 8; // Used for inventory + repeated Component components = 8; // Used for inventory DeviceId controller_id = 9; // Identifier of node controlling the actual device } -message Component { - repeated string comp_string = 1; +message Component { //Defined previously to this section - Tested OK + Uuid component_uuid = 1; + string name = 2; + string type = 3; + + map attributes = 4; // dict[attr.name => json.dumps(attr.value)] + string parent = 5; } message DeviceConfig { diff --git a/src/context/service/database/Component.py b/src/context/service/database/Component.py new file mode 100644 index 0000000000000000000000000000000000000000..ae873855b1c01c80c101eb4e97e3bf7688619fae --- /dev/null +++ b/src/context/service/database/Component.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime, json, logging +from sqlalchemy import delete +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.orm import Session +from typing import Dict, List, Optional, Set +from common.proto.context_pb2 import Component +from common.proto.context_pb2 import ConfigRule +from common.tools.grpc.Tools import grpc_message_to_json_string +from .models.ComponentModel import ComponentModel +from .uuids._Builder import get_uuid_from_string +from .uuids.EndPoint import endpoint_get_uuid +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, selectinload, sessionmaker +from sqlalchemy_cockroachdb import run_transaction +from .models.ComponentModel import ComponentModel + +LOGGER = logging.getLogger(__name__) + +def compose_components_data( + components : List[Component], now : datetime.datetime, + device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None +) -> List[Dict]: + dict_components : List[Dict] = list() + for position,component in enumerate(components): + str_kind = component.WhichOneof('config_rule') + message = (grpc_message_to_json_string(getattr(component, str_kind, {}))) + data = json.loads(message) + resource_key = data["resource_key"] + resource_value = data["resource_value"] + if '/inventory' in resource_key: + resource_value_data = json.loads(resource_value) + name = resource_value_data.pop('name', None) + type_ = resource_value_data.pop('class', None) + parent = resource_value_data.pop('parent-component-references', None) + attributes = resource_value_data.pop('attributes', {}) + if len(resource_value_data) > 0: + LOGGER.warning('Discarding Component Leftovers: {:s}'.format(str(resource_value_data))) + + attributes = { + attr_name:json.dumps(attr_value) + for attr_name,attr_value in attributes.items() + } + component_uuid = get_uuid_from_string(component.custom.resource_key, prefix_for_name=device_uuid) + dict_component = { + 'component_uuid': component_uuid, + 'device_uuid' : device_uuid, + 'name' : name, + 'type' : type_, + 'attributes' : json.dumps(attributes), + 'parent' : parent, + 'created_at' : now, + 'updated_at' : now, + } + dict_components.append(dict_component) + return dict_components diff --git a/src/context/service/database/Device.py b/src/context/service/database/Device.py index df57791adf7faf04cb24c372d82ce547f35b9c72..3aff20ade14532dcb7fbf8ec1033c084aaeead3c 100644 --- a/src/context/service/database/Device.py +++ b/src/context/service/database/Device.py @@ -27,6 +27,7 @@ from common.tools.object_factory.Device import json_device_id from context.service.database.uuids.Topology import topology_get_uuid from .models.DeviceModel import DeviceModel from .models.EndPointModel import EndPointModel +from .models.ComponentModel import ComponentModel from .models.TopologyModel import TopologyDeviceModel, TopologyModel from .models.enums.DeviceDriver import grpc_to_enum__device_driver from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status @@ -34,6 +35,7 @@ from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type from .uuids.Device import device_get_uuid from .uuids.EndPoint import endpoint_get_uuid from .ConfigRule import compose_config_rules_data, upsert_config_rules +from .Component import compose_components_data from .Events import notify_event_context, notify_event_device, notify_event_topology LOGGER = logging.getLogger(__name__) @@ -50,8 +52,8 @@ def device_list_objs(db_engine : Engine) -> DeviceList: obj_list : List[DeviceModel] = session.query(DeviceModel)\ .options(selectinload(DeviceModel.endpoints))\ .options(selectinload(DeviceModel.config_rules))\ + .options(selectinload(DeviceModel.components))\ .all() - #.options(selectinload(DeviceModel.components))\ return [obj.dump() for obj in obj_list] devices = run_transaction(sessionmaker(bind=db_engine), callback) return DeviceList(devices=devices) @@ -62,8 +64,8 @@ def device_get(db_engine : Engine, request : DeviceId) -> Device: obj : Optional[DeviceModel] = session.query(DeviceModel)\ .options(selectinload(DeviceModel.endpoints))\ .options(selectinload(DeviceModel.config_rules))\ + .options(selectinload(DeviceModel.components))\ .filter_by(device_uuid=device_uuid).one_or_none() - #.options(selectinload(DeviceModel.components))\ return None if obj is None else obj.dump() obj = run_transaction(sessionmaker(bind=db_engine), callback) if obj is None: @@ -138,7 +140,8 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi }) topology_uuids.add(endpoint_topology_uuid) - config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid) + components_data = compose_components_data(request.device_config.config_rules, now, device_uuid=device_uuid) + config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid) device_data = [{ 'device_uuid' : device_uuid, @@ -206,6 +209,24 @@ def device_set(db_engine : Engine, messagebroker : MessageBroker, request : Devi device_topology_ids = [obj.dump_id() for obj in device_topologies] LOGGER.warning('device_topology_ids={:s}'.format(str(device_topology_ids))) + updated_components = False + + if len(components_data) > 0: + stmt = insert(ComponentModel).values(components_data) + stmt = stmt.on_conflict_do_update( + index_elements=[ComponentModel.component_uuid], + set_=dict( + name = stmt.excluded.name, + type = stmt.excluded.type, + attributes = stmt.excluded.attributes, + parent = stmt.excluded.parent, + updated_at = stmt.excluded.updated_at, + ) + ) + stmt = stmt.returning(ComponentModel.created_at, ComponentModel.updated_at) + component_updates = session.execute(stmt).fetchall() + updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates]) + changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid) return updated or updated_endpoints or changed_config_rules, device_topology_ids diff --git a/src/context/service/database/models/ComponentModel.py b/src/context/service/database/models/ComponentModel.py new file mode 100644 index 0000000000000000000000000000000000000000..c9acfaeabe95c55fc464732d54459d99f9b5b054 --- /dev/null +++ b/src/context/service/database/models/ComponentModel.py @@ -0,0 +1,55 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from sqlalchemy import Column, DateTime, ForeignKey, String +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship +from typing import Dict +from ._Base import _Base + +class ComponentModel(_Base): + __tablename__ = 'device_component' + + component_uuid = Column(UUID(as_uuid=False), primary_key=True) + device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) + name = Column(String, nullable=False) + type = Column(String, nullable=False) + attributes = Column(String, nullable=False) + parent = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + device = relationship('DeviceModel', back_populates='components') + def dump_id(self) -> Dict: + return{ + 'device_id' : self.device.dump_id(), + 'component_uuid': {'uuid': self.component_uuid}, + } + + def dump(self) -> Dict: + data = dict() + data['attributes'] = json.loads(self.attributes) + data['component_uuid'] = {'uuid': self.component_uuid} + data['name'] = self.name + data['type'] = self.type + data['parent'] = self.parent + return data + + def dump_name(self) -> Dict: + return { + 'component_id' : self.dump_id(), + 'device_name' : self.device.device_name, + 'component_name': self.name, + } diff --git a/src/context/service/database/models/DeviceModel.py b/src/context/service/database/models/DeviceModel.py index 1097d0b9ab47a86c47ce2ad8394d067ae9f9953e..376dc98c4053f68c511a8c717117d58d9eda1cca 100644 --- a/src/context/service/database/models/DeviceModel.py +++ b/src/context/service/database/models/DeviceModel.py @@ -36,6 +36,7 @@ class DeviceModel(_Base): #topology_devices = relationship('TopologyDeviceModel', back_populates='device') config_rules = relationship('DeviceConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device' endpoints = relationship('EndPointModel', passive_deletes=True) # lazy='joined', back_populates='device' + components = relationship('ComponentModel', passive_deletes=True) # lazy='joined', back_populates='device' controller = relationship('DeviceModel', remote_side=[device_uuid], passive_deletes=True) # lazy='joined', back_populates='device' def dump_id(self) -> Dict: @@ -55,7 +56,7 @@ class DeviceModel(_Base): ]} def dump_components(self) -> List[Dict]: - return [] + return [component.dump() for component in self.components] def dump(self, include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True, @@ -70,5 +71,5 @@ class DeviceModel(_Base): } if include_endpoints: result['device_endpoints'] = self.dump_endpoints() if include_config_rules: result['device_config'] = self.dump_config_rules() - if include_components: result['component'] = self.dump_components() + if include_components: result['components'] = self.dump_components() return result diff --git a/src/context/service/database/models/_Base.py b/src/context/service/database/models/_Base.py index b87b9b06d6adc5825ab5dd84cf64347eb9c26f66..52eb6b088210b8edc8d121221458ae11ece484a5 100644 --- a/src/context/service/database/models/_Base.py +++ b/src/context/service/database/models/_Base.py @@ -60,6 +60,9 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [ 'topology_name', 'created_at', 'updated_at' ]), + index_storing('device_component_idx', 'device_component', ['device_uuid'], [ + 'name', 'type', 'attributes', 'created_at', 'updated_at' + ]), ] def callback(session : Session) -> bool: for stmt in statements: session.execute(stmt) diff --git a/src/device/service/driver_api/_Driver.py b/src/device/service/driver_api/_Driver.py index 7adaec79dc99f9b7c836acaec886b0d5bda97fb8..0aa1a6c5a8697d4c75f7044981221c6dd47e3aff 100644 --- a/src/device/service/driver_api/_Driver.py +++ b/src/device/service/driver_api/_Driver.py @@ -24,6 +24,7 @@ RESOURCE_NETWORK_INSTANCES = '__network_instances__' RESOURCE_ROUTING_POLICIES = '__routing_policies__' RESOURCE_SERVICES = '__services__' RESOURCE_ACL = '__acl__' +RESOURCE_INVENTORY = '__inventory__' class _Driver: diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py new file mode 100644 index 0000000000000000000000000000000000000000..0d98d72882695038d9d3ddbc2c9b86564d21d5b0 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -0,0 +1,159 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, lxml.etree as ET +from typing import Any, Dict, List, Tuple +from .Namespace import NAMESPACES +from .Tools import add_value_from_tag + +LOGGER = logging.getLogger(__name__) + +XPATH_PORTS = "//ocp:components/ocp:component" + +""" +#Method Name: parse + +#Parameters: + + - xml_data: [ET.Element] Represents the XML data to be parsed. + +# Functionality: + + The parse function of the inventerio class has the functionality to parse + an XML document represented by the xml_data parameter and extract specific + information from the XML elements, namely the relevant characteristics of the + components. + + To generate the template the following steps are performed: + + 1) An empty list called response is created to store the results of the analysis. + + 2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS + expression. These elements represent components in the XML document. + + 3) For each component element: + A dictionary called inventory is initialized that will store the information extracted + from the component.The values of the relevant XML elements are extracted and added to + the dictionary. + +#Return: + List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary) + with the information extracted from the XML document components is returned. +""" + +def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: + response = [] + LOGGER.debug("InventoryPrueba") + parent_types = {} + for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): + LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) + inventory = {} + inventory['parent-component-references'] = '' + inventory['name'] = '' + inventory['class'] = '' + inventory['attributes'] = {} + component_reference = [] + + component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) + if component_name is None or component_name.text is None: continue + add_value_from_tag(inventory, 'name', component_name) + + component_description = xml_component.find('ocp:state/ocp:description', namespaces=NAMESPACES) + if not component_description is None: + add_value_from_tag(inventory['attributes'], 'description', component_description) + + component_location = xml_component.find('ocp:state/ocp:location', namespaces=NAMESPACES) + if not component_location is None: + add_value_from_tag(inventory['attributes'], 'location', component_location) + + component_type = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) + component_type.text = component_type.text.replace('oc-platform-types:','') + if component_type is None: continue + add_value_from_tag(inventory, 'class', component_type) + + if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue + + component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) + if not component_empty is None: + add_value_from_tag(inventory['attributes'], 'empty', component_empty) + + component_parent = xml_component.find('ocp:state/ocp:parent', namespaces=NAMESPACES) + if component_parent is None or component_parent.text is None: + add_value_from_tag(inventory, 'parent-component-references', component_type) + else: + add_value_from_tag(inventory, 'parent-component-references', component_parent) + + component_HW = xml_component.find('ocp:state/ocp:hardware-version', namespaces=NAMESPACES) + if not component_HW is None: + add_value_from_tag(inventory['attributes'], 'hardware-rev', component_HW) + + component_firmware_version = xml_component.find('ocp:state/ocp:firmware-version', namespaces=NAMESPACES) + if not component_firmware_version is None: + add_value_from_tag(inventory['attributes'], 'firmware-rev', component_firmware_version) + + component_SW = xml_component.find('ocp:state/ocp:software-version', namespaces=NAMESPACES) + if not component_SW is None: + add_value_from_tag(inventory['attributes'], 'software-rev', component_SW) + + component_serial = xml_component.find('ocp:state/ocp:serial-no', namespaces=NAMESPACES) + if not component_serial is None: + add_value_from_tag(inventory['attributes'], 'serial-num', component_serial) + + component_mfg_name = xml_component.find('ocp:state/ocp:mfg-name', namespaces=NAMESPACES) + if not component_mfg_name is None: + add_value_from_tag(inventory['attributes'], 'manufacturer-name', component_mfg_name) + + component_removable = xml_component.find('ocp:state/ocp:removable', namespaces=NAMESPACES) + if not component_removable is None: + add_value_from_tag(inventory['attributes'], 'removable', component_removable) + + component_mfg_date = xml_component.find('ocp:state/ocp:mfg-date', namespaces=NAMESPACES) + if not component_mfg_date is None: + add_value_from_tag(inventory['attributes'], 'mfg-date', component_mfg_date) + + #Transceiver Information + component_serial_t = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:serial-no', namespaces=NAMESPACES) + if not component_serial_t is None: + add_value_from_tag(inventory['attributes'], 'serial-num', component_serial_t) + + component_present = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:present', namespaces=NAMESPACES) + if component_present is not None and 'NOT_PRESENT' in component_present.text: continue + + component_vendor = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:vendor', namespaces=NAMESPACES) + if not component_vendor is None: + add_value_from_tag(inventory['attributes'], 'vendor', component_vendor) + component_connector = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:connector-type', namespaces=NAMESPACES) + if not component_connector is None: + component_connector.text = component_connector.text.replace('oc-opt-types:','') + add_value_from_tag(inventory['attributes'], 'connector-type', component_connector) + + component_form = xml_component.find('ocptr:transceiver/ocptr:state/ocptr:form-factor', namespaces=NAMESPACES) + if not component_form is None: + component_form.text = component_form.text.replace('oc-opt-types:','') + add_value_from_tag(inventory['attributes'], 'form-factor', component_form) + + if inventory['parent-component-references'] not in parent_types: + parent_types[inventory['parent-component-references']] = len(parent_types) + 1 + + component_reference.extend([parent_types[inventory['parent-component-references']]]) + + response.append(('/inventory/{:s}'.format(inventory['name']), inventory)) + + for tupla in response: + if inventory['parent-component-references'] in tupla[0]: + component_reference.extend([tupla[1]['class']]) + + inventory['component-reference'] = component_reference + + return response diff --git a/src/device/service/drivers/openconfig/templates/Namespace.py b/src/device/service/drivers/openconfig/templates/Namespace.py index eede865502b043b7936d763c980be80a7ea817f8..bdc27a1ff30d5ac18b9233cdd420cd8493e7a419 100644 --- a/src/device/service/drivers/openconfig/templates/Namespace.py +++ b/src/device/service/drivers/openconfig/templates/Namespace.py @@ -28,6 +28,7 @@ NAMESPACE_POLICY_TYPES = 'http://openconfig.net/yang/policy-types' NAMESPACE_POLICY_TYPES_2 = 'http://openconfig.net/yang/policy_types' NAMESPACE_ROUTING_POLICY = 'http://openconfig.net/yang/routing-policy' NAMESPACE_VLAN = 'http://openconfig.net/yang/vlan' +NAMESPACE_PLATFORM_TRANSCEIVER = 'http://openconfig.net/yang/platform/transceiver' NAMESPACES = { 'nc' : NAMESPACE_NETCONF, @@ -44,4 +45,5 @@ NAMESPACES = { 'ocpt2': NAMESPACE_POLICY_TYPES_2, 'ocrp' : NAMESPACE_ROUTING_POLICY, 'ocv' : NAMESPACE_VLAN, + 'ocptr': NAMESPACE_PLATFORM_TRANSCEIVER, } diff --git a/src/device/service/drivers/openconfig/templates/__init__.py b/src/device/service/drivers/openconfig/templates/__init__.py index 1f86b719227f4def0bc18c45997925846defbc56..87eea1f0b6673c4bff3222598d81a16b383b4c3b 100644 --- a/src/device/service/drivers/openconfig/templates/__init__.py +++ b/src/device/service/drivers/openconfig/templates/__init__.py @@ -20,15 +20,17 @@ from jinja2 import Environment, PackageLoader, select_autoescape import paramiko from .Tools import generate_templates from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL) + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY) from .EndPoints import parse as parse_endpoints from .Interfaces import parse as parse_interfaces, parse_counters from .NetworkInstances import parse as parse_network_instances from .RoutingPolicy import parse as parse_routing_policy from .Acl import parse as parse_acl +from .Inventory import parse as parse_inventory LOGGER = logging.getLogger(__name__) ALL_RESOURCE_KEYS = [ + RESOURCE_INVENTORY, RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_ROUTING_POLICIES, # routing policies should come before network instances @@ -37,6 +39,7 @@ ALL_RESOURCE_KEYS = [ ] RESOURCE_KEY_MAPPINGS = { + RESOURCE_INVENTORY : 'inventory', RESOURCE_ENDPOINTS : 'component', RESOURCE_INTERFACES : 'interface', RESOURCE_NETWORK_INSTANCES: 'network_instance', @@ -45,6 +48,7 @@ RESOURCE_KEY_MAPPINGS = { } RESOURCE_PARSERS = { + 'inventory' : parse_inventory, 'component' : parse_endpoints, 'interface' : parse_interfaces, 'network_instance': parse_network_instances, diff --git a/src/device/service/drivers/openconfig/templates/inventory/get.xml b/src/device/service/drivers/openconfig/templates/inventory/get.xml new file mode 100644 index 0000000000000000000000000000000000000000..aa25ed1e3b11e0c324b361eb52d064dac87a64c5 --- /dev/null +++ b/src/device/service/drivers/openconfig/templates/inventory/get.xml @@ -0,0 +1,3 @@ + + +