From fe5b4e32b75c8c1b300ffe1c1efa1d171324682e Mon Sep 17 00:00:00 2001 From: gifrerenom <lluis.gifre@cttc.es> Date: Sun, 18 Dec 2022 02:02:24 +0000 Subject: [PATCH] Tool - dlt-load-gen: - new tool (under implementation) to trigger recording of entities in DLT and evaluate performance --- src/tests/tools/dlt_load_gen/__init__.py | 14 ++ src/tests/tools/dlt_load_gen/__main__.py | 82 +++++++ src/tests/tools/dlt_load_gen/deploy_specs.sh | 26 ++ src/tests/tools/dlt_load_gen/descriptors.json | 229 ++++++++++++++++++ src/tests/tools/dlt_load_gen/run.sh | 17 ++ 5 files changed, 368 insertions(+) create mode 100644 src/tests/tools/dlt_load_gen/__init__.py create mode 100644 src/tests/tools/dlt_load_gen/__main__.py create mode 100644 src/tests/tools/dlt_load_gen/deploy_specs.sh create mode 100644 src/tests/tools/dlt_load_gen/descriptors.json create mode 100755 src/tests/tools/dlt_load_gen/run.sh diff --git a/src/tests/tools/dlt_load_gen/__init__.py b/src/tests/tools/dlt_load_gen/__init__.py new file mode 100644 index 000000000..70a332512 --- /dev/null +++ b/src/tests/tools/dlt_load_gen/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/tests/tools/dlt_load_gen/__main__.py b/src/tests/tools/dlt_load_gen/__main__.py new file mode 100644 index 000000000..cdad0662c --- /dev/null +++ b/src/tests/tools/dlt_load_gen/__main__.py @@ -0,0 +1,82 @@ +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, sys +from common.proto.context_pb2 import DeviceId, Empty, LinkId, ServiceId, SliceId, TopologyId +from common.proto.dlt_connector_pb2 import DltDeviceId, DltLinkId, DltServiceId, DltSliceId +from common.tools.object_factory.Topology import json_topology_id +from context.client.ContextClient import ContextClient +from dlt.connector.client.DltConnectorClient import DltConnectorClient + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + +def record_device(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, device_id : DeviceId): + dlt_device_id = DltDeviceId() + dlt_device_id.topology_id.CopyFrom(domain_id) # pylint: disable=no-member + dlt_device_id.device_id.CopyFrom(device_id) # pylint: disable=no-member + dlt_connector_client.RecordDevice(dlt_device_id) + +def record_link(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, link_id : LinkId): + dlt_link_id = DltLinkId() + dlt_link_id.topology_id.CopyFrom(domain_id) # pylint: disable=no-member + dlt_link_id.link_id.CopyFrom(link_id) # pylint: disable=no-member + dlt_connector_client.RecordLink(dlt_link_id) + +def record_service(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, service_id : ServiceId): + dlt_service_id = DltServiceId() + dlt_service_id.topology_id.CopyFrom(domain_id) # pylint: disable=no-member + dlt_service_id.service_id.CopyFrom(service_id) # pylint: disable=no-member + dlt_connector_client.RecordService(dlt_service_id) + +def record_slice(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, slice_id : SliceId): + dlt_slice_id = DltSliceId() + dlt_slice_id.topology_id.CopyFrom(domain_id) # pylint: disable=no-member + dlt_slice_id.slice_id.CopyFrom(slice_id) # pylint: disable=no-member + dlt_connector_client.RecordSlice(dlt_slice_id) + +def main(): + LOGGER.info('Starting...') + + context_client = ContextClient() + dlt_connector_client = DltConnectorClient() + + domain_id = TopologyId(**json_topology_id('dlt-perf-eval')) + + device_ids = context_client.ListDeviceIds(Empty()) + for device_id in device_ids.device_ids: + record_device(dlt_connector_client, domain_id, device_id) + + link_ids = context_client.ListLinkIds(Empty()) + for link_id in link_ids.link_ids: + record_link(dlt_connector_client, domain_id, link_id) + + context_ids = context_client.ListContextIds(Empty()) + for context_id in context_ids.context_ids: + service_ids = context_client.ListServiceIds(context_id) + for service_id in service_ids.service_ids: + record_service(dlt_connector_client, domain_id, service_id) + + slice_ids = context_client.ListSliceIds(context_id) + for slice_id in slice_ids.slice_ids: + record_slice(dlt_connector_client, domain_id, slice_id) + + LOGGER.info('Done!') + return 0 + +if __name__ == '__main__': + sys.exit(main()) + + + diff --git a/src/tests/tools/dlt_load_gen/deploy_specs.sh b/src/tests/tools/dlt_load_gen/deploy_specs.sh new file mode 100644 index 000000000..1982ef227 --- /dev/null +++ b/src/tests/tools/dlt_load_gen/deploy_specs.sh @@ -0,0 +1,26 @@ +# Set the URL of your local Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +# Supported components are: +# context device automation policy service compute monitoring webui +# interdomain slice pathcomp dlt +# dbscanserving opticalattackmitigator opticalattackdetector +# l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector +export TFS_COMPONENTS="context device pathcomp service slice webui dlt" # automation monitoring compute + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml manifests/servicemonitors.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# If not already set, disable skip-build flag. +# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used. +export TFS_SKIP_BUILD="NO" #${TFS_SKIP_BUILD:-"YES"} diff --git a/src/tests/tools/dlt_load_gen/descriptors.json b/src/tests/tools/dlt_load_gen/descriptors.json new file mode 100644 index 000000000..5fb0c0867 --- /dev/null +++ b/src/tests/tools/dlt_load_gen/descriptors.json @@ -0,0 +1,229 @@ +{ + "contexts": [ + { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "topology_ids": [], "service_ids": [] + } + ], + "topologies": [ + { + "topology_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "topology_uuid": {"uuid": "admin"} + }, + "device_ids": [ + {"device_uuid": {"uuid": "R1"}}, + {"device_uuid": {"uuid": "R2"}}, + {"device_uuid": {"uuid": "R3"}}, + {"device_uuid": {"uuid": "R4"}}, + {"device_uuid": {"uuid": "R5"}}, + {"device_uuid": {"uuid": "R6"}}, + {"device_uuid": {"uuid": "R7"}} + ], + "link_ids": [ + {"link_uuid": {"uuid": "R1==R2"}}, + {"link_uuid": {"uuid": "R2==R3"}}, + {"link_uuid": {"uuid": "R3==R4"}}, + {"link_uuid": {"uuid": "R4==R5"}}, + {"link_uuid": {"uuid": "R5==R6"}}, + {"link_uuid": {"uuid": "R6==R1"}}, + {"link_uuid": {"uuid": "R1==R7"}}, + {"link_uuid": {"uuid": "R3==R7"}}, + {"link_uuid": {"uuid": "R5==R7"}} + ] + } + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "R1"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "1/5"}, + {"sample_types": [], "type": "copper", "uuid": "1/6"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R2"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "1/5"}, + {"sample_types": [], "type": "copper", "uuid": "1/6"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R3"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "1/5"}, + {"sample_types": [], "type": "copper", "uuid": "1/6"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R4"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "1/5"}, + {"sample_types": [], "type": "copper", "uuid": "1/6"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R5"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "1/5"}, + {"sample_types": [], "type": "copper", "uuid": "1/6"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R6"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "1/4"}, + {"sample_types": [], "type": "copper", "uuid": "1/5"}, + {"sample_types": [], "type": "copper", "uuid": "1/6"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "R7"}}, "device_type": "emu-packet-router", "device_drivers": [0], + "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"sample_types": [], "type": "copper", "uuid": "1/1"}, + {"sample_types": [], "type": "copper", "uuid": "1/2"}, + {"sample_types": [], "type": "copper", "uuid": "1/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/1"}, + {"sample_types": [], "type": "copper", "uuid": "2/2"}, + {"sample_types": [], "type": "copper", "uuid": "2/3"}, + {"sample_types": [], "type": "copper", "uuid": "2/4"}, + {"sample_types": [], "type": "copper", "uuid": "2/5"}, + {"sample_types": [], "type": "copper", "uuid": "2/6"} + ]}}} + ]} + } + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "R1==R2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R2==R3"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "R3"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R3==R4"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R3"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "R4"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R4==R5"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R4"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "R5"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R5==R6"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R5"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "R6"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R6==R1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R6"}}, "endpoint_uuid": {"uuid": "2/1"}}, + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R1==R7"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2/3"}}, + {"device_id": {"device_uuid": {"uuid": "R7"}}, "endpoint_uuid": {"uuid": "2/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R3==R7"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R3"}}, "endpoint_uuid": {"uuid": "2/3"}}, + {"device_id": {"device_uuid": {"uuid": "R7"}}, "endpoint_uuid": {"uuid": "2/3"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "R5==R7"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "R5"}}, "endpoint_uuid": {"uuid": "2/3"}}, + {"device_id": {"device_uuid": {"uuid": "R7"}}, "endpoint_uuid": {"uuid": "2/5"}} + ] + } + ] +} \ No newline at end of file diff --git a/src/tests/tools/dlt_load_gen/run.sh b/src/tests/tools/dlt_load_gen/run.sh new file mode 100755 index 000000000..6329b5434 --- /dev/null +++ b/src/tests/tools/dlt_load_gen/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source tfs_runtime_env_vars.sh +python -m tests.tools.dlt_load_gen -- GitLab