Skip to content
Snippets Groups Projects
Commit e17e8cc3 authored by Pablo Armingol's avatar Pablo Armingol
Browse files

ddbb modification to store the device inventory

parent b4a2af65
No related branches found
No related tags found
2 merge requests!235Release TeraFlowSDN 3.0,!150Resolve "(TID) Network device inventory management"
Showing
with 322 additions and 115 deletions
......@@ -174,14 +174,30 @@ message Device {
DeviceOperationalStatusEnum device_operational_status = 5;
repeated DeviceDriverEnum device_drivers = 6;
repeated EndPoint device_endpoints = 7;
repeated Component component = 8; // Used for inventory
map<string, Component> components = 8; // Used for inventory
DeviceId controller_id = 9; // Identifier of node controlling the actual device
}
message Component {
repeated string comp_string = 1;
message Component { //Defined previously to this section - Tested OK
Uuid uuid = 1;
string name = 2;
string type = 3;
repeated string child = 4; // list[sub-component.name]
map<string, string> attributes = 5; // dict[attr.name => json.dumps(attr.value)]
}
message ComponentId { //NEW
DeviceId device_id = 1;
Uuid endpoint_uuid = 2;
}
message ComponentIdList { //NEW
repeated ComponentId component_ids = 1;
}
// -----------------------------------------------------
message DeviceConfig {
repeated ConfigRule config_rules = 1;
}
......
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime, json, logging
from sqlalchemy import delete
#from sqlalchemy.dialects import postgresql
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.orm import Session
from typing import Dict, List, Optional, Set
from common.proto.context_pb2 import Component
from common.proto.context_pb2 import ConfigRule
from common.tools.grpc.Tools import grpc_message_to_json_string
from .models.ComponentModel import ComponentModel
from .uuids._Builder import get_uuid_from_string
from .uuids.EndPoint import endpoint_get_uuid
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, selectinload, sessionmaker
from sqlalchemy_cockroachdb import run_transaction
from common.proto.context_pb2 import ComponentIdList
from .models.ComponentModel import ComponentModel
from .uuids.Component import component_get_uuid
from .ConfigRule import compose_config_rules_data
LOGGER = logging.getLogger(__name__)
def compose_components_data(
components : List[Component], now : datetime.datetime,
device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None
) -> List[Dict]:
dict_components : List[Dict] = list()
for position,component in enumerate(components):
str_kind = component.WhichOneof('config_rule')
LOGGER.info("DATA")
message = (grpc_message_to_json_string(getattr(component, str_kind, {})))
data = json.loads(message)
resource_key = data["resource_key"]
resource_value = data["resource_value"]
if '/inventory' in resource_key:
LOGGER.info('Parametros: KEY',resource_key,'Value',resource_value)
dict_component = {
'data' : resource_value,
'created_at': now,
'updated_at': now,
}
parent_kind,parent_uuid = '',None
if device_uuid is not None:
dict_component['device_uuid'] = device_uuid
parent_kind,parent_uuid = 'device',device_uuid
elif service_uuid is not None:
dict_component['service_uuid'] = service_uuid
parent_kind,parent_uuid = 'service',service_uuid
elif slice_uuid is not None:
dict_component['slice_uuid'] = slice_uuid
parent_kind,parent_uuid = 'slice',slice_uuid
else:
MSG = 'Parent for Component({:s}) cannot be identified '+\
'(device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})'
str_component = grpc_message_to_json_string(component)
raise Exception(MSG.format(str_component, str(device_uuid), str(service_uuid), str(slice_uuid)))
componenet_name = '{:s}:{:s}:{:s}'.format(parent_kind, 'custom', component.custom.resource_key)
component_uuid = get_uuid_from_string(componenet_name, prefix_for_name=parent_uuid)
dict_component['component_uuid'] = component_uuid
dict_components.append(dict_component)
else:
continue
LOGGER.info('Parametros:',dict_components)
return dict_components
'''
def upsert_components(
session : Session, components : List[Dict], is_delete : bool = False,
device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None,
) -> bool:
if device_uuid is not None and service_uuid is None and slice_uuid is None:
klass = ComponentModel
else:
MSG = 'DataModel cannot be identified (device_uuid={:s})'
raise Exception(MSG.format(str(device_uuid)))
uuids_to_upsert : Dict[str, int] = dict()
rules_to_upsert : List[Dict] = list()
for component in components:
component_uuid = component['component_uuid']
position = uuids_to_upsert.get(component_uuid)
if position is None:
# if not added, add it
rules_to_upsert.append(component)
uuids_to_upsert[component_uuid] = len(rules_to_upsert) - 1
else:
# if already added, update occurrence
rules_to_upsert[position] = component
upsert_affected = False
if len(rules_to_upsert) > 0:
stmt = insert(klass).values(rules_to_upsert)
stmt = stmt.on_conflict_do_update(
index_elements=[klass.component_uuid],
set_=dict(
data = stmt.excluded.data,
updated_at = stmt.excluded.updated_at,
)
)
stmt = stmt.returning(klass.created_at, klass.updated_at)
#str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})
#LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt)))
components_updates = session.execute(stmt).fetchall()
upsert_affected = any([(updated_at > created_at) for created_at,updated_at in components_updates])
return upsert_affected
def component_list_names(db_engine: Engine, request: ComponentIdList) -> List[Dict]:
component_uuids = {
component_get_uuid(component_id, allow_random=False)[-1]
for component_id in request.component_ids
}
def callback(session: Session) -> List[Dict]:
obj_list: List[ComponentModel] = session.query(ComponentModel)\
.options(selectinload(ComponentModel.device))\
.filter(ComponentModel.component_uuid.in_(component_uuids)).all()
return [obj.dump_name() for obj in obj_list]
return run_transaction(sessionmaker(bind=db_engine), callback)
def compose_components_data(data: Dict[str, any]) -> List[Dict]:
filtered_data = []
for item in data:
for key, value in item:
if any("inventory" in key):
filtered_data.append(item)
LOGGER.info("Filtered Data:")
LOGGER.info(filtered_data)
# Return the result
return filtered_data
'''
\ No newline at end of file
......@@ -34,6 +34,9 @@ def compose_config_rules_data(
) -> List[Dict]:
dict_config_rules : List[Dict] = list()
for position,config_rule in enumerate(config_rules):
LOGGER.info("REQUEST")
LOGGER.info(position,config_rule)
str_kind = config_rule.WhichOneof('config_rule')
kind = ConfigRuleKindEnum._member_map_.get(str_kind.upper()) # pylint: disable=no-member
dict_config_rule = {
......
......@@ -26,12 +26,14 @@ from context.service.database.uuids.Topology import topology_get_uuid
from .models.DeviceModel import DeviceModel
from .models.EndPointModel import EndPointModel
from .models.TopologyModel import TopologyDeviceModel
from .models.ComponentModel import ComponentModel
from .models.enums.DeviceDriver import grpc_to_enum__device_driver
from .models.enums.DeviceOperationalStatus import grpc_to_enum__device_operational_status
from .models.enums.KpiSampleType import grpc_to_enum__kpi_sample_type
from .uuids.Device import device_get_uuid
from .uuids.EndPoint import endpoint_get_uuid
from .ConfigRule import compose_config_rules_data, upsert_config_rules
from .Component import compose_components_data
LOGGER = logging.getLogger(__name__)
......@@ -46,6 +48,7 @@ def device_list_objs(db_engine : Engine) -> List[Dict]:
obj_list : List[DeviceModel] = session.query(DeviceModel)\
.options(selectinload(DeviceModel.endpoints))\
.options(selectinload(DeviceModel.config_rules))\
.options(selectinload(DeviceModel.components))\
.all()
#.options(selectinload(DeviceModel.components))\
return [obj.dump() for obj in obj_list]
......@@ -133,7 +136,8 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]:
})
topology_uuids.add(endpoint_topology_uuid)
config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid)
components_data = compose_components_data(request.device_config.config_rules, now, device_uuid=device_uuid)
config_rules = compose_config_rules_data(request.device_config.config_rules, now, device_uuid=device_uuid)
device_data = [{
'device_uuid' : device_uuid,
......@@ -187,9 +191,26 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]:
index_elements=[TopologyDeviceModel.topology_uuid, TopologyDeviceModel.device_uuid]
))
updated_components = False
LOGGER.info("HERE ERRPR DEBUG")
LOGGER.info(components_data)
if len(components_data) > 0:
stmt = insert(ComponentModel).values(components_data)
stmt = stmt.on_conflict_do_update(
index_elements=[ComponentModel.component_uuid],
set_=dict(
data = stmt.excluded.data,
updated_at = stmt.excluded.updated_at,
)
)
stmt = stmt.returning(ComponentModel.created_at, ComponentModel.updated_at)
component_updates = session.execute(stmt).fetchall()
updated_components = any([(updated_at > created_at) for created_at,updated_at in component_updates])
changed_config_rules = upsert_config_rules(session, config_rules, device_uuid=device_uuid)
return updated or updated_endpoints or changed_config_rules
return updated or updated_components or changed_config_rules
updated = run_transaction(sessionmaker(bind=db_engine), callback)
return json_device_id(device_uuid),updated
......
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from sqlalchemy import Column, DateTime, ForeignKey, String
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import relationship
from typing import Dict
from ._Base import _Base
class ComponentModel(_Base): #Inherited functionality from the base class _Base
__tablename__ = 'device_component' #Name of the table in the DB associtaed with this model
component_uuid = Column(UUID(as_uuid=False), primary_key=True) #Unique identifier that serves as a primary key for this table
device_uuid = Column(ForeignKey('device.device_uuid',ondelete='CASCADE' ), nullable=False, index=True) #Foreign Key relationship with the field device_uuid from the Device table (CASCADE' behavior for deletion, meaning when a device is deleted, its components will also be dele)
# component_name = Column(String, nullable=False) #String field that stores the name of the component
data = Column(String, nullable=False) #String field that stores data about the component
created_at = Column(DateTime, nullable=False) #Stores the creaton timestamp for the component
updated_at = Column(DateTime, nullable=False) #Stores the last upadted timestamp for the component
device = relationship('DeviceModel', back_populates='components')# lazy='selectin'#Defines a relationship between ComponentModel and DeviceModel
#Represents a 1:n relationship where 1 device -> N components // The relationship is defined by the FK device_uuid
def dump_id(self) -> Dict:
return{
'device_id' : self.device.dump_id(),
'component_uuid': {'uuid': self.component_uuid},
}
def dump(self) -> Dict:
return {
'component_id' : self.dump_id(),
'data' : self.data,
}
def dump_name(self) -> Dict:
return {
'component_id' : self.dump_id(),
'device_name' : self.device.device_name,
'component_name': self.name,
}
......@@ -36,6 +36,7 @@ class DeviceModel(_Base):
#topology_devices = relationship('TopologyDeviceModel', back_populates='device')
config_rules = relationship('DeviceConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device'
endpoints = relationship('EndPointModel', passive_deletes=True) # lazy='joined', back_populates='device'
components = relationship('ComponentModel', passive_deletes=True) # lazy='joined', back_populates='device'
controller = relationship('DeviceModel', remote_side=[device_uuid], passive_deletes=True) # lazy='joined', back_populates='device'
def dump_id(self) -> Dict:
......@@ -55,7 +56,7 @@ class DeviceModel(_Base):
]}
def dump_components(self) -> List[Dict]:
return []
return [component.dump() for component in self.components]
def dump(self,
include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True,
......@@ -70,5 +71,5 @@ class DeviceModel(_Base):
}
if include_endpoints: result['device_endpoints'] = self.dump_endpoints()
if include_config_rules: result['device_config'] = self.dump_config_rules()
if include_components: result['component'] = self.dump_components()
if include_components: result['components'] = self.dump_components()
return result
......@@ -60,6 +60,9 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None:
index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [
'topology_name', 'created_at', 'updated_at'
]),
index_storing('device_component_idx', 'device_component', ['device_uuid'], [
'data', 'created_at', 'updated_at'
]),
]
def callback(session : Session) -> bool:
for stmt in statements: session.execute(stmt)
......
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Tuple
from common.proto.context_pb2 import ComponentId
from common.method_wrappers.ServiceExceptions import InvalidArgumentsException
from ._Builder import get_uuid_from_string, get_uuid_random
from .Device import device_get_uuid
def component_get_uuid(
component_id: ComponentId, component_name: str = '', allow_random: bool = False
) -> str:
device_uuid = device_get_uuid(component_id.device_id, allow_random=False)
raw_component_uuid = component_id.component_uuid.uuid
if raw_component_uuid:
prefix_for_name = f'{device_uuid}'
return get_uuid_from_string(raw_component_uuid, prefix_for_name=prefix_for_name)
if component_name:
prefix_for_name = f'{device_uuid}'
return get_uuid_from_string(component_name, prefix_for_name=prefix_for_name)
if allow_random:
return get_uuid_random()
raise InvalidArgumentsException(
[
('component_id.component_uuid.uuid', raw_component_uuid),
('name', component_name),
],
extra_details=['At least one is required to produce a Component UUID']
)
......@@ -22,6 +22,7 @@ LOGGER = logging.getLogger(__name__)
XPATH_INTERFACES = "//oci:interfaces/oci:interface"
XPATH_SUBINTERFACES = ".//oci:subinterfaces/oci:subinterface"
XPATH_IPV4ADDRESSES = ".//ociip:ipv4/ociip:addresses/ociip:address"
XPATH_IPV6ADDRESSES = ".//ociip:ipv6/ociip:addresses/ociip:address"
def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
response = []
......@@ -97,6 +98,15 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
#add_value_from_collection(subinterface, 'ipv4_addresses', ipv4_addresses)
for xml_ipv6_address in xml_subinterface.xpath(XPATH_IPV6ADDRESSES, namespaces=NAMESPACES):
#LOGGER.info('xml_ipv6_address = {:s}'.format(str(ET.tostring(xml_ipv6_address))))
address = xml_ipv6_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES)
add_value_from_tag(subinterface, 'address_ipv6', address)
prefix = xml_ipv6_address.find('ociip:state/ociip:prefix-length', namespaces=NAMESPACES)
add_value_from_tag(subinterface, 'address_prefix_v6', prefix, cast=int)
if len(subinterface) == 0: continue
resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index']))
response.append((resource_key, subinterface))
......
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
#Method Name: parse
#Parameters:
- xml_data: [ET.Element] Represents the XML data to be parsed.
# Functionality:
The parse function of the interfaces_mng class has the functionality to parse
an XML document represented by the xml_data parameter and extract specific
information from the XML elements, namely the active interfaces with their name,
type, ipv4 and ipv6 addresses in case they have.
To generate the template the following steps are performed:
1) An empty list called response is created to store the results of the analysis.
2) Iterate over the XML elements that match the pattern specified by the XPATH_PORTS
expression. These elements represent management interfaces in the XML document.
3) For each management interfaces:
A dictionary called interfaces_mng is initialized that will store the information extracted
from the interfaces.The values of the relevant XML elements are extracted and added to
the dictionary.
#Return:
List[Tuple[str, Dict[str, Any]]] The response list containing the tuples (path, dictionary)
with the information extracted from the XML document interfaces is returned.
"""
import logging, lxml.etree as ET
from typing import Any, Dict, List, Tuple
from .Namespace import NAMESPACES
from .Tools import add_value_from_tag
LOGGER = logging.getLogger(__name__)
XPATH_PORTS = "//oci:interfaces/oci:interface"
XPATH_SUBINTERFACES = ".//oci:subinterfaces/oci:subinterface"
XPATH_IPV4ADDRESSES = ".//ociip:ipv4/ociip:addresses/ociip:address"
XPATH_IPV6ADDRESSES = ".//ociip:ipv6/ociip:addresses/ociip:address"
def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]:
response = []
LOGGER.debug("Interfaces_mngPrueba")
for xml_interface_mng in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES):
LOGGER.info('xml_component xml_interfaces_mng = {:s}'.format(str(ET.tostring(xml_interface_mng))))
interfaces_mng = {}
interface_enabled = xml_interface_mng.find('oci:config/oci:enabled', namespaces=NAMESPACES)
if interface_enabled == None:
interface_enabled = xml_interface_mng.find('oci:state/oci:enabled', namespaces=NAMESPACES)
if interface_enabled == None: continue
if 'false' in interface_enabled or 'false' in interface_enabled.text: continue
interface_name = xml_interface_mng.find('oci:name', namespaces=NAMESPACES)
if interface_name is None or interface_name.text is None: continue
add_value_from_tag(interfaces_mng, 'name', interface_name)
interface_type = xml_interface_mng.find('oci:config/oci:type', namespaces=NAMESPACES)
if interface_type is None:
interface_type = xml_interface_mng.find('oci:state/oci:type', namespaces=NAMESPACES)
if interface_type is None: continue
interface_type.text = interface_type.text.replace('ianaift:','')
add_value_from_tag(interfaces_mng, 'type', interface_type)
for xml_subinterface in xml_interface_mng.xpath(XPATH_SUBINTERFACES, namespaces=NAMESPACES):
for xml_ipv4_address in xml_subinterface.xpath(XPATH_IPV4ADDRESSES, namespaces=NAMESPACES):
address_ipv4 = xml_ipv4_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES)
if not address_ipv4 is None:
add_value_from_tag(interfaces_mng, 'ipv4', address_ipv4)
for xml_ipv6_address in xml_subinterface.xpath(XPATH_IPV6ADDRESSES, namespaces=NAMESPACES):
address_ipv6 = xml_ipv6_address.find('ociip:state/ociip:ip', namespaces=NAMESPACES)
if not address_ipv6 is None:
add_value_from_tag(interfaces_mng, 'ipv6', address_ipv6)
if not 'ipv4' in interfaces_mng and not 'ipv6' in interfaces_mng:
if 'ip' in interfaces_mng['type'] or 'Loopback' in interfaces_mng['type']: continue
response.append(('/interfaces_mng/{:s}'.format(interfaces_mng['name']), interfaces_mng))
return response
......@@ -20,10 +20,9 @@ from jinja2 import Environment, PackageLoader, select_autoescape
import paramiko
from .Tools import generate_templates
from device.service.driver_api._Driver import (
RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_INTERFACES_MNG,RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY)
RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_ACL, RESOURCE_INVENTORY)
from .EndPoints import parse as parse_endpoints
from .Interfaces import parse as parse_interfaces, parse_counters
from .Interfaces_mng import parse as parse_interfaces_mng
from .NetworkInstances import parse as parse_network_instances
from .RoutingPolicy import parse as parse_routing_policy
from .Acl import parse as parse_acl
......@@ -34,7 +33,6 @@ ALL_RESOURCE_KEYS = [
RESOURCE_INVENTORY,
RESOURCE_ENDPOINTS,
RESOURCE_INTERFACES,
RESOURCE_INTERFACES_MNG,
RESOURCE_ROUTING_POLICIES, # routing policies should come before network instances
RESOURCE_NETWORK_INSTANCES,
RESOURCE_ACL,
......@@ -44,7 +42,6 @@ RESOURCE_KEY_MAPPINGS = {
RESOURCE_INVENTORY : 'inventory',
RESOURCE_ENDPOINTS : 'component',
RESOURCE_INTERFACES : 'interface',
RESOURCE_INTERFACES_MNG : 'interfaces_mng',
RESOURCE_NETWORK_INSTANCES: 'network_instance',
RESOURCE_ROUTING_POLICIES : 'routing_policy',
RESOURCE_ACL : 'acl',
......@@ -54,7 +51,6 @@ RESOURCE_PARSERS = {
'inventory' : parse_inventory,
'component' : parse_endpoints,
'interface' : parse_interfaces,
'interfaces_mng' : parse_interfaces_mng,
'network_instance': parse_network_instances,
'routing_policy' : parse_routing_policy,
'interfaces/interface/state/counters': parse_counters,
......
<interfaces xmlns="http://openconfig.net/yang/interfaces">
<interface/>
</interfaces>
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment