From fe5b4e32b75c8c1b300ffe1c1efa1d171324682e Mon Sep 17 00:00:00 2001
From: gifrerenom <lluis.gifre@cttc.es>
Date: Sun, 18 Dec 2022 02:02:24 +0000
Subject: [PATCH] Tool - dlt-load-gen:

- new tool (under implementation) to trigger recording of entities in DLT and evaluate performance
---
 src/tests/tools/dlt_load_gen/__init__.py      |  14 ++
 src/tests/tools/dlt_load_gen/__main__.py      |  82 +++++++
 src/tests/tools/dlt_load_gen/deploy_specs.sh  |  26 ++
 src/tests/tools/dlt_load_gen/descriptors.json | 229 ++++++++++++++++++
 src/tests/tools/dlt_load_gen/run.sh           |  17 ++
 5 files changed, 368 insertions(+)
 create mode 100644 src/tests/tools/dlt_load_gen/__init__.py
 create mode 100644 src/tests/tools/dlt_load_gen/__main__.py
 create mode 100644 src/tests/tools/dlt_load_gen/deploy_specs.sh
 create mode 100644 src/tests/tools/dlt_load_gen/descriptors.json
 create mode 100755 src/tests/tools/dlt_load_gen/run.sh

diff --git a/src/tests/tools/dlt_load_gen/__init__.py b/src/tests/tools/dlt_load_gen/__init__.py
new file mode 100644
index 000000000..70a332512
--- /dev/null
+++ b/src/tests/tools/dlt_load_gen/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/src/tests/tools/dlt_load_gen/__main__.py b/src/tests/tools/dlt_load_gen/__main__.py
new file mode 100644
index 000000000..cdad0662c
--- /dev/null
+++ b/src/tests/tools/dlt_load_gen/__main__.py
@@ -0,0 +1,82 @@
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging, sys
+from common.proto.context_pb2 import DeviceId, Empty, LinkId, ServiceId, SliceId, TopologyId
+from common.proto.dlt_connector_pb2 import DltDeviceId, DltLinkId, DltServiceId, DltSliceId
+from common.tools.object_factory.Topology import json_topology_id
+from context.client.ContextClient import ContextClient
+from dlt.connector.client.DltConnectorClient import DltConnectorClient
+
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger(__name__)
+
+def record_device(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, device_id : DeviceId):
+    dlt_device_id = DltDeviceId()
+    dlt_device_id.topology_id.CopyFrom(domain_id)       # pylint: disable=no-member
+    dlt_device_id.device_id.CopyFrom(device_id)         # pylint: disable=no-member
+    dlt_connector_client.RecordDevice(dlt_device_id)
+
+def record_link(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, link_id : LinkId):
+    dlt_link_id = DltLinkId()
+    dlt_link_id.topology_id.CopyFrom(domain_id)         # pylint: disable=no-member
+    dlt_link_id.link_id.CopyFrom(link_id)               # pylint: disable=no-member
+    dlt_connector_client.RecordLink(dlt_link_id)
+
+def record_service(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, service_id : ServiceId):
+    dlt_service_id = DltServiceId()
+    dlt_service_id.topology_id.CopyFrom(domain_id)      # pylint: disable=no-member
+    dlt_service_id.service_id.CopyFrom(service_id)      # pylint: disable=no-member
+    dlt_connector_client.RecordService(dlt_service_id)
+
+def record_slice(dlt_connector_client : DltConnectorClient, domain_id : TopologyId, slice_id : SliceId):
+    dlt_slice_id = DltSliceId()
+    dlt_slice_id.topology_id.CopyFrom(domain_id)        # pylint: disable=no-member
+    dlt_slice_id.slice_id.CopyFrom(slice_id)            # pylint: disable=no-member
+    dlt_connector_client.RecordSlice(dlt_slice_id)
+
+def main():
+    LOGGER.info('Starting...')
+
+    context_client = ContextClient()
+    dlt_connector_client = DltConnectorClient()
+
+    domain_id = TopologyId(**json_topology_id('dlt-perf-eval'))
+
+    device_ids = context_client.ListDeviceIds(Empty())
+    for device_id in device_ids.device_ids:
+        record_device(dlt_connector_client, domain_id, device_id)
+
+    link_ids = context_client.ListLinkIds(Empty())
+    for link_id in link_ids.link_ids:
+        record_link(dlt_connector_client, domain_id, link_id)
+
+    context_ids = context_client.ListContextIds(Empty())
+    for context_id in context_ids.context_ids:
+        service_ids = context_client.ListServiceIds(context_id)
+        for service_id in service_ids.service_ids:
+            record_service(dlt_connector_client, domain_id, service_id)
+
+        slice_ids = context_client.ListSliceIds(context_id)
+        for slice_id in slice_ids.slice_ids:
+            record_slice(dlt_connector_client, domain_id, slice_id)
+
+    LOGGER.info('Done!')
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())
+
+
+
diff --git a/src/tests/tools/dlt_load_gen/deploy_specs.sh b/src/tests/tools/dlt_load_gen/deploy_specs.sh
new file mode 100644
index 000000000..1982ef227
--- /dev/null
+++ b/src/tests/tools/dlt_load_gen/deploy_specs.sh
@@ -0,0 +1,26 @@
+# Set the URL of your local Docker registry where the images will be uploaded to.
+export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/"
+
+# Set the list of components, separated by spaces, you want to build images for, and deploy.
+# Supported components are:
+#   context device automation policy service compute monitoring webui
+#   interdomain slice pathcomp dlt
+#   dbscanserving opticalattackmitigator opticalattackdetector
+#   l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector
+export TFS_COMPONENTS="context device pathcomp service slice webui dlt" # automation monitoring compute
+
+# Set the tag you want to use for your images.
+export TFS_IMAGE_TAG="dev"
+
+# Set the name of the Kubernetes namespace to deploy to.
+export TFS_K8S_NAMESPACE="tfs"
+
+# Set additional manifest files to be applied after the deployment
+export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml manifests/servicemonitors.yaml"
+
+# Set the new Grafana admin password
+export TFS_GRAFANA_PASSWORD="admin123+"
+
+# If not already set, disable skip-build flag.
+# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.
+export TFS_SKIP_BUILD="NO" #${TFS_SKIP_BUILD:-"YES"}
diff --git a/src/tests/tools/dlt_load_gen/descriptors.json b/src/tests/tools/dlt_load_gen/descriptors.json
new file mode 100644
index 000000000..5fb0c0867
--- /dev/null
+++ b/src/tests/tools/dlt_load_gen/descriptors.json
@@ -0,0 +1,229 @@
+{
+    "contexts": [
+        {
+            "context_id": {"context_uuid": {"uuid": "admin"}},
+            "topology_ids": [], "service_ids": []
+        }
+    ],
+    "topologies": [
+        {
+            "topology_id": {
+                "context_id": {"context_uuid": {"uuid": "admin"}},
+                "topology_uuid": {"uuid": "admin"}
+            },
+            "device_ids": [
+                {"device_uuid": {"uuid": "R1"}},
+                {"device_uuid": {"uuid": "R2"}},
+                {"device_uuid": {"uuid": "R3"}},
+                {"device_uuid": {"uuid": "R4"}},
+                {"device_uuid": {"uuid": "R5"}},
+                {"device_uuid": {"uuid": "R6"}},
+                {"device_uuid": {"uuid": "R7"}}
+            ],
+            "link_ids": [
+                {"link_uuid": {"uuid": "R1==R2"}},
+                {"link_uuid": {"uuid": "R2==R3"}},
+                {"link_uuid": {"uuid": "R3==R4"}},
+                {"link_uuid": {"uuid": "R4==R5"}},
+                {"link_uuid": {"uuid": "R5==R6"}},
+                {"link_uuid": {"uuid": "R6==R1"}},
+                {"link_uuid": {"uuid": "R1==R7"}},
+                {"link_uuid": {"uuid": "R3==R7"}},
+                {"link_uuid": {"uuid": "R5==R7"}}
+            ]
+        }
+    ],
+    "devices": [
+        {
+            "device_id": {"device_uuid": {"uuid": "R1"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/6"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"}
+                ]}}}
+            ]}
+        },
+        {
+            "device_id": {"device_uuid": {"uuid": "R2"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/6"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"}
+                ]}}}
+            ]}
+        },
+        {
+            "device_id": {"device_uuid": {"uuid": "R3"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/6"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"}
+                ]}}}
+            ]}
+        },
+        {
+            "device_id": {"device_uuid": {"uuid": "R4"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/6"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"}
+                ]}}}
+            ]}
+        },
+        {
+            "device_id": {"device_uuid": {"uuid": "R5"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/6"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"}
+                ]}}}
+            ]}
+        },
+        {
+            "device_id": {"device_uuid": {"uuid": "R6"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/6"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"}
+                ]}}}
+            ]}
+        },
+        {
+            "device_id": {"device_uuid": {"uuid": "R7"}}, "device_type": "emu-packet-router", "device_drivers": [0],
+            "device_endpoints": [], "device_operational_status": 1, "device_config": {"config_rules": [
+                {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}},
+                {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}},
+                {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [
+                    {"sample_types": [], "type": "copper", "uuid": "1/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "1/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/1"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/2"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/3"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/4"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/5"},
+                    {"sample_types": [], "type": "copper", "uuid": "2/6"}
+                ]}}}
+            ]}
+        }
+    ],
+    "links": [
+        {
+            "link_id": {"link_uuid": {"uuid": "R1==R2"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2/1"}},
+                {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "2/2"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R2==R3"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R2"}}, "endpoint_uuid": {"uuid": "2/1"}},
+                {"device_id": {"device_uuid": {"uuid": "R3"}}, "endpoint_uuid": {"uuid": "2/2"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R3==R4"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R3"}}, "endpoint_uuid": {"uuid": "2/1"}},
+                {"device_id": {"device_uuid": {"uuid": "R4"}}, "endpoint_uuid": {"uuid": "2/2"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R4==R5"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R4"}}, "endpoint_uuid": {"uuid": "2/1"}},
+                {"device_id": {"device_uuid": {"uuid": "R5"}}, "endpoint_uuid": {"uuid": "2/2"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R5==R6"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R5"}}, "endpoint_uuid": {"uuid": "2/1"}},
+                {"device_id": {"device_uuid": {"uuid": "R6"}}, "endpoint_uuid": {"uuid": "2/2"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R6==R1"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R6"}}, "endpoint_uuid": {"uuid": "2/1"}},
+                {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2/2"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R1==R7"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R1"}}, "endpoint_uuid": {"uuid": "2/3"}},
+                {"device_id": {"device_uuid": {"uuid": "R7"}}, "endpoint_uuid": {"uuid": "2/1"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R3==R7"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R3"}}, "endpoint_uuid": {"uuid": "2/3"}},
+                {"device_id": {"device_uuid": {"uuid": "R7"}}, "endpoint_uuid": {"uuid": "2/3"}}
+            ]
+        },
+        {
+            "link_id": {"link_uuid": {"uuid": "R5==R7"}},
+            "link_endpoint_ids": [
+                {"device_id": {"device_uuid": {"uuid": "R5"}}, "endpoint_uuid": {"uuid": "2/3"}},
+                {"device_id": {"device_uuid": {"uuid": "R7"}}, "endpoint_uuid": {"uuid": "2/5"}}
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/src/tests/tools/dlt_load_gen/run.sh b/src/tests/tools/dlt_load_gen/run.sh
new file mode 100755
index 000000000..6329b5434
--- /dev/null
+++ b/src/tests/tools/dlt_load_gen/run.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source tfs_runtime_env_vars.sh
+python -m tests.tools.dlt_load_gen
-- 
GitLab