From 52bab918c4ba712cb1d0925375d6309fa01a1341 Mon Sep 17 00:00:00 2001 From: Jose Luis Carcel Date: Wed, 26 Jul 2023 07:42:54 +0000 Subject: [PATCH 001/941] Update file context.proto --- proto/context.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proto/context.proto b/proto/context.proto index 55a80470d..87680a5b9 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -448,9 +448,10 @@ message EndPointId { message EndPoint { EndPointId endpoint_id = 1; string name = 2; - string endpoint_type = 3; + string endpoint_type = 3; // == 'smartnics' repeated kpi_sample_types.KpiSampleType kpi_sample_types = 4; Location endpoint_location = 5; + map capabilities = 6; } message EndPointName { -- GitLab From ab9a0dcd86da5bcd9e2230b9b9bbd18b3c696797 Mon Sep 17 00:00:00 2001 From: Jose Luis Carcel Date: Wed, 26 Jul 2023 08:18:08 +0000 Subject: [PATCH 002/941] Create Context Smartnics --- proto/context-ext-smartnics.proto | 100 ++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 proto/context-ext-smartnics.proto diff --git a/proto/context-ext-smartnics.proto b/proto/context-ext-smartnics.proto new file mode 100644 index 000000000..f4783da73 --- /dev/null +++ b/proto/context-ext-smartnics.proto @@ -0,0 +1,100 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// References: +// https://www.nvidia.com/content/dam/en-zz/Solutions/data-center/products/a30-gpu/pdf/a30-datasheet.pdf +// https://www.nvidia.com/content/dam/en-zz/Solutions/Data-Center/documents/datasheet-nvidia-bluefield-2-dpu.pdf +// https://www.nvidia.com/content/dam/en-zz/Solutions/networking/ethernet-adapters/connectX-6-dx-datasheet.pdf +// converged accel: https://www.nvidia.com/content/dam/en-zz/Solutions/gtcf21/converged-accelerator/pdf/datasheet.pdf + +// bluefield-2 = connectX-6 + DPUs +// conv_accel = bluefield-2 + GPU + +syntax = "proto3"; +package context-ext-smartnics; + +import context; +import "kpi_sample_types.proto"; + +message SmartnicsCapabilities { + string vendor = 1; + string model = 2; + string serial_number = 3; + repeated Transceiver transceivers = 4; + repeated DPU dpus = 5; + repeated GPU gpus = 6; +} + +enum TransceiverPortTypeEnum { + TRANSCEIVER_PORT_TYPE_UNDEFINED = 0; + TRANSCEIVER_PORT_TYPE_SFP = 1; + TRANSCEIVER_PORT_TYPE_SFP_PLUS = 2; + TRANSCEIVER_PORT_TYPE_QSFP_28 = 3; + TRANSCEIVER_PORT_TYPE_QSFP_56 = 4; + TRANSCEIVER_PORT_TYPE_QSFP_DD = 5; +} + +enum TransceiverPortSpeedEnum { + TRANSCEIVER_PORT_SPEED_UNDEFINED = 0; + TRANSCEIVER_PORT_SPEED_1G = 1; + TRANSCEIVER_PORT_SPEED_10G = 2; + TRANSCEIVER_PORT_SPEED_25G = 3; + TRANSCEIVER_PORT_SPEED_40G = 4; + TRANSCEIVER_PORT_SPEED_100G = 5; + TRANSCEIVER_PORT_SPEED_200G = 6; + TRANSCEIVER_PORT_SPEED_400G = 7; + TRANSCEIVER_PORT_SPEED_800G = 8; +} + +message Transceiver { + TransceiverPortTypeEnum port_type = 1; + repeated TransceiverPortSpeedEnum port_speeds = 2; + repeated kpi_sample_types.KpiSampleType kpi_sample_types = 3; +} + +message DPU { + uint32 num_cores = 1; + repeated DPU_Core cores = 2; + repeated DPU_Memory memories = 3; +} + +enum DpuCoreArchitectureEnum { + DPU_CORE_ARCHITECTURE_UNDEFINED = 0; + DPU_CORE_ARCHITECTURE_32BIT = 1; + DPU_CORE_ARCHITECTURE_64BIT = 2; +} + +message DPU_Core { + string model = 1; // Armv8 A72 + enum DpuCoreArchitectureEnum architecture = 2; // 64-bit + // define cache: + // -- 1MB L2 cache per 2 cores + // -- 6MB L3 cache with plurality of eviction policies +} + +message DPU_Memory { + // define RAM and eMMC + // -- DDR4 DIMM Support + // -- > Single DDR4 DRAM controller + // -- > 16GB / 32GB of on-board DDR4 + // -- > ECC error protection support +} + +message GPU { + enum Architecture architecture = 1; + enum ComputeCapabilities compute_capabilities = 2; + uint64 memory_size_mb = 3; + uint32 num_cores = 4; + // complete with specs of GPU +} -- GitLab From baf109068617e6002665b0a3b07888c160e37da9 Mon Sep 17 00:00:00 2001 From: Jose Luis Carcel Date: Wed, 26 Jul 2023 08:58:00 +0000 Subject: [PATCH 003/941] Add new directory --- src/device/service/drivers/smartnic_probes/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/device/service/drivers/smartnic_probes/.gitkeep diff --git a/src/device/service/drivers/smartnic_probes/.gitkeep b/src/device/service/drivers/smartnic_probes/.gitkeep new file mode 100644 index 000000000..e69de29bb -- GitLab From 1e17fbc1da0fb88c5dead7d83de277ea118d7323 Mon Sep 17 00:00:00 2001 From: Jose Luis Carcel Date: Wed, 26 Jul 2023 08:58:54 +0000 Subject: [PATCH 004/941] Added Probes-Agent YANG --- .../drivers/smartnic_probes/probes-agent.yang | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/device/service/drivers/smartnic_probes/probes-agent.yang diff --git a/src/device/service/drivers/smartnic_probes/probes-agent.yang b/src/device/service/drivers/smartnic_probes/probes-agent.yang new file mode 100644 index 000000000..71e1337c4 --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/probes-agent.yang @@ -0,0 +1,44 @@ +probes-agent.yang + +# augment de openconfig probes + +augment /probes/probe { + leaf probe_type { + type enumeration { + enum plain_traffic; + enum morpheus_pipeline; + } + } +} + +augment /probes/probe/tests/test/config { + uses morpheus_pipelines; +} + +grouping morpheus_pipelines { + list morpheus_pipelines { + key name; + uses morpheus_pipeline; + } +} + +grouping morpheus_pipeline { + leaf name { type string; } + leaf num_threads { type uint16; } + // other settings + // https://github.com/nv-morpheus/Morpheus/blob/branch-23.11/examples/abp_pcap_detection/run.py + + list stages { + key name; + uses morpheus_pipeline_stage; + } +} + +grouping morpheus_pipeline_stage { + leaf name { type string; } + // enum stage type + // stage parameters + // soportar custom stages + // https://github.com/nv-morpheus/Morpheus/blob/branch-23.11/examples/abp_pcap_detection/run.py + +} -- GitLab From af997d4865cd625ae2ed9feb06f0f35a84325df6 Mon Sep 17 00:00:00 2001 From: Ricard Vilalta Date: Thu, 27 Jul 2023 07:57:31 +0000 Subject: [PATCH 005/941] Add new directory --- .../drivers/smartnic_probes/smartnics_probes_agent/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/device/service/drivers/smartnic_probes/smartnics_probes_agent/.gitkeep diff --git a/src/device/service/drivers/smartnic_probes/smartnics_probes_agent/.gitkeep b/src/device/service/drivers/smartnic_probes/smartnics_probes_agent/.gitkeep new file mode 100644 index 000000000..e69de29bb -- GitLab From e6505c21208113eff2b8e5dbe7fed0b8ed2f1f7e Mon Sep 17 00:00:00 2001 From: carcel Date: Thu, 3 Aug 2023 12:14:12 +0200 Subject: [PATCH 006/941] Update Context SmartNICs --- proto/context-ext-smartnics.proto | 56 ++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/proto/context-ext-smartnics.proto b/proto/context-ext-smartnics.proto index f4783da73..2a3fe6138 100644 --- a/proto/context-ext-smartnics.proto +++ b/proto/context-ext-smartnics.proto @@ -13,8 +13,6 @@ // limitations under the License. // References: -// https://www.nvidia.com/content/dam/en-zz/Solutions/data-center/products/a30-gpu/pdf/a30-datasheet.pdf -// https://www.nvidia.com/content/dam/en-zz/Solutions/Data-Center/documents/datasheet-nvidia-bluefield-2-dpu.pdf // https://www.nvidia.com/content/dam/en-zz/Solutions/networking/ethernet-adapters/connectX-6-dx-datasheet.pdf // converged accel: https://www.nvidia.com/content/dam/en-zz/Solutions/gtcf21/converged-accelerator/pdf/datasheet.pdf @@ -77,24 +75,50 @@ enum DpuCoreArchitectureEnum { message DPU_Core { string model = 1; // Armv8 A72 - enum DpuCoreArchitectureEnum architecture = 2; // 64-bit - // define cache: - // -- 1MB L2 cache per 2 cores - // -- 6MB L3 cache with plurality of eviction policies + enum DpuCoreArchitectureEnum architecture = 2; + uint64 l2_cache_size_mb = 3; + uint64 l3_cache_size_mb = 4; } message DPU_Memory { - // define RAM and eMMC - // -- DDR4 DIMM Support - // -- > Single DDR4 DRAM controller - // -- > 16GB / 32GB of on-board DDR4 - // -- > ECC error protection support + string RamMemoryType = 1; //On-Board DDR4 + string eMMCMemoryType = 2; //eMMC + enum DpuRamMemorySizeGB RamMemorySizeGB = 3; + enum DpueMMCMemorySizeGB eMMCMemorySizeGB = 4; +} + +enum DpuRamMemorySizeGB { + DPU_MEMORY_DDR4_RAM_UNDEFINED = 0; + DPU_MEMORY_DDR4_RAM_16GB = 1; + DPU_MEMORY_DDR4_RAM_32GB = 2; +} + +enum DpueMMCMemorySizeGB { + DPU_MEMORY_eMMC_UNDEFINED = 0; + DPU_MEMORY_eMMC_32GB = 1; + DPU_MEMORY_eMMC_64GB = 2; + DPU_MEMORY_eMMC_128GB = 3; } message GPU { - enum Architecture architecture = 1; - enum ComputeCapabilities compute_capabilities = 2; - uint64 memory_size_mb = 3; - uint32 num_cores = 4; - // complete with specs of GPU + enum Architecture architecture = 1; //AMPERE? + enum ComputeCapabilities compute_capabilities = 2; //MIG? + uint64 memory_size_gb = 3; //24 GB HBM2 + uint32 num_CUDA_cores = 4; //3804 + uint32 num_Tensor_cores = 5; //224 + uint32 peak_fp_32 = 6; //10.3TF + uint32 peak_fp_64 = 7; //5.2TF + uint32 peak fp_64_tensor_core = 8; //10.3TF +} + +enum Architecture { + ARCH_UNDEFINED = 0; + ARCH_AMPERE = 1; +} + +enum ComputeCapabilities { + MIG_UNDEFINED = 0; + MIG_4_AT_6GB = 1; + MIG_2_AT_12GB = 2; + MIG_1_AT_24GB = 3; } -- GitLab From 6d77d5e611716a23a7a23148167ce3285714dd81 Mon Sep 17 00:00:00 2001 From: carcel Date: Thu, 3 Aug 2023 12:32:01 +0200 Subject: [PATCH 007/941] Update Context SmartNICs --- proto/context-ext-smartnics.proto | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/proto/context-ext-smartnics.proto b/proto/context-ext-smartnics.proto index 2a3fe6138..f54535e7b 100644 --- a/proto/context-ext-smartnics.proto +++ b/proto/context-ext-smartnics.proto @@ -15,14 +15,13 @@ // References: // https://www.nvidia.com/content/dam/en-zz/Solutions/networking/ethernet-adapters/connectX-6-dx-datasheet.pdf // converged accel: https://www.nvidia.com/content/dam/en-zz/Solutions/gtcf21/converged-accelerator/pdf/datasheet.pdf - // bluefield-2 = connectX-6 + DPUs // conv_accel = bluefield-2 + GPU syntax = "proto3"; package context-ext-smartnics; -import context; +import "context"; import "kpi_sample_types.proto"; message SmartnicsCapabilities { @@ -75,7 +74,7 @@ enum DpuCoreArchitectureEnum { message DPU_Core { string model = 1; // Armv8 A72 - enum DpuCoreArchitectureEnum architecture = 2; + DpuCoreArchitectureEnum architecture = 2; uint64 l2_cache_size_mb = 3; uint64 l3_cache_size_mb = 4; } @@ -83,8 +82,8 @@ message DPU_Core { message DPU_Memory { string RamMemoryType = 1; //On-Board DDR4 string eMMCMemoryType = 2; //eMMC - enum DpuRamMemorySizeGB RamMemorySizeGB = 3; - enum DpueMMCMemorySizeGB eMMCMemorySizeGB = 4; + DpuRamMemorySizeGB RamMemorySizeGB = 3; + DpueMMCMemorySizeGB eMMCMemorySizeGB = 4; } enum DpuRamMemorySizeGB { @@ -100,17 +99,6 @@ enum DpueMMCMemorySizeGB { DPU_MEMORY_eMMC_128GB = 3; } -message GPU { - enum Architecture architecture = 1; //AMPERE? - enum ComputeCapabilities compute_capabilities = 2; //MIG? - uint64 memory_size_gb = 3; //24 GB HBM2 - uint32 num_CUDA_cores = 4; //3804 - uint32 num_Tensor_cores = 5; //224 - uint32 peak_fp_32 = 6; //10.3TF - uint32 peak_fp_64 = 7; //5.2TF - uint32 peak fp_64_tensor_core = 8; //10.3TF -} - enum Architecture { ARCH_UNDEFINED = 0; ARCH_AMPERE = 1; @@ -122,3 +110,16 @@ enum ComputeCapabilities { MIG_2_AT_12GB = 2; MIG_1_AT_24GB = 3; } + + +message GPU { + Architecture architecture = 1; //AMPERE? + ComputeCapabilities compute_capabilities = 2; //MIG? + uint64 memory_size_gb = 3; //24 GB HBM2 + uint32 num_CUDA_cores = 4; //3804 + uint32 num_Tensor_cores = 5; //224 + uint32 peak_fp_32 = 6; //10.3TF + uint32 peak_fp_64 = 7; //5.2TF + uint32 peak_fp_64_tensor_core = 8; //10.3TF +} + -- GitLab From 69d1887348454e7ab1aed261b60e3dc7cd9afffb Mon Sep 17 00:00:00 2001 From: carcel Date: Thu, 21 Sep 2023 16:42:23 +0200 Subject: [PATCH 008/941] SmartNIC, NOS and NS Data Models --- proto/context-ext-smartnics.proto | 31 +++---- proto/nfv_client.proto | 44 +++++++++ .../drivers/smartnic_probes/probes-agent.yang | 89 ++++++++++++++++--- ztp_device.yang | 43 +++++++++ 4 files changed, 180 insertions(+), 27 deletions(-) create mode 100644 proto/nfv_client.proto create mode 100644 ztp_device.yang diff --git a/proto/context-ext-smartnics.proto b/proto/context-ext-smartnics.proto index f54535e7b..953b27e86 100644 --- a/proto/context-ext-smartnics.proto +++ b/proto/context-ext-smartnics.proto @@ -16,7 +16,7 @@ // https://www.nvidia.com/content/dam/en-zz/Solutions/networking/ethernet-adapters/connectX-6-dx-datasheet.pdf // converged accel: https://www.nvidia.com/content/dam/en-zz/Solutions/gtcf21/converged-accelerator/pdf/datasheet.pdf // bluefield-2 = connectX-6 + DPUs -// conv_accel = bluefield-2 + GPU +// conv_accel = bluefield-2 + GPU (A30) syntax = "proto3"; package context-ext-smartnics; @@ -73,15 +73,15 @@ enum DpuCoreArchitectureEnum { } message DPU_Core { - string model = 1; // Armv8 A72 + string model = 1; DpuCoreArchitectureEnum architecture = 2; uint64 l2_cache_size_mb = 3; uint64 l3_cache_size_mb = 4; } message DPU_Memory { - string RamMemoryType = 1; //On-Board DDR4 - string eMMCMemoryType = 2; //eMMC + string RamMemoryType = 1; + string eMMCMemoryType = 2; DpuRamMemorySizeGB RamMemorySizeGB = 3; DpueMMCMemorySizeGB eMMCMemorySizeGB = 4; } @@ -99,27 +99,28 @@ enum DpueMMCMemorySizeGB { DPU_MEMORY_eMMC_128GB = 3; } -enum Architecture { +enum GPUArchitecture { ARCH_UNDEFINED = 0; ARCH_AMPERE = 1; } -enum ComputeCapabilities { +enum GPUComputeCapabilities { MIG_UNDEFINED = 0; - MIG_4_AT_6GB = 1; + MIG_4_AT_6GB = 1; MIG_2_AT_12GB = 2; MIG_1_AT_24GB = 3; } message GPU { - Architecture architecture = 1; //AMPERE? - ComputeCapabilities compute_capabilities = 2; //MIG? - uint64 memory_size_gb = 3; //24 GB HBM2 - uint32 num_CUDA_cores = 4; //3804 - uint32 num_Tensor_cores = 5; //224 - uint32 peak_fp_32 = 6; //10.3TF - uint32 peak_fp_64 = 7; //5.2TF - uint32 peak_fp_64_tensor_core = 8; //10.3TF + string model = 1; + GPUArchitecture architecture = 2; + GPUComputeCapabilities compute_capabilities = 3; + uint64 memory_size_gb = 4; + uint32 num_CUDA_cores = 5; + uint32 num_Tensor_cores = 6; + uint32 peak_fp_32 = 7; + uint32 peak_fp_64 = 8; + uint32 peak_fp_64_tensor_core = 9; } diff --git a/proto/nfv_client.proto b/proto/nfv_client.proto new file mode 100644 index 000000000..d880a394c --- /dev/null +++ b/proto/nfv_client.proto @@ -0,0 +1,44 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package nfv_client; + +import "context.proto"; + +message Nsd { + string nsd_name = 1; +} + +message Ns { + string ns_id=1; + string ns_name=2; + string nsd_name=3; + string vim_account=4; +} + +message NsList { + repeated Ns ns = 1; +} + +message NsdList { + repeated Nsd nsd = 1; +} + +service nfv_client { + rpc CreateNs (ns) returns (ns_id) {} // Stable not final + rpc DeleteNs (ns) returns (context.Empty) {} +} + + diff --git a/src/device/service/drivers/smartnic_probes/probes-agent.yang b/src/device/service/drivers/smartnic_probes/probes-agent.yang index 71e1337c4..6029cd204 100644 --- a/src/device/service/drivers/smartnic_probes/probes-agent.yang +++ b/src/device/service/drivers/smartnic_probes/probes-agent.yang @@ -1,8 +1,18 @@ -probes-agent.yang +module probes-agent { -# augment de openconfig probes +namespace "urn:probes-agent"; +prefix "probes-agent"; -augment /probes/probe { +import openconfig-probes { prefix oc-probes; } + +organization "EVIDEN"; +contact "jose.carcel@eviden.com"; +description "Basic example of data model for configuring SmartNICs through Morpheus API"; +revision "2023-08-03" { + description "Basic example of data model for configuring SmartNICs through Morpheus API"; +} + +augment '/oc-probes:probes/oc-probes:probe' { leaf probe_type { type enumeration { enum plain_traffic; @@ -11,7 +21,7 @@ augment /probes/probe { } } -augment /probes/probe/tests/test/config { +augment '/oc-probes:probes/oc-probes:probe/oc-probes:tests/oc-probes:test/oc-probes:config' { uses morpheus_pipelines; } @@ -25,20 +35,75 @@ grouping morpheus_pipelines { grouping morpheus_pipeline { leaf name { type string; } leaf num_threads { type uint16; } - // other settings - // https://github.com/nv-morpheus/Morpheus/blob/branch-23.11/examples/abp_pcap_detection/run.py - + leaf pipeline_batch_size {type uint64; } + leaf model_max_batch_size {type uint64; } + leaf input_file {type string; } + leaf output_file {type string; } + leaf model_fea_length {type uint16; } + leaf model_name {type string; } + leaf iterative {type boolean; } + leaf server_url {type string; } + leaf file_type {type files; } list stages { key name; uses morpheus_pipeline_stage; } } +typedef files { + type enumeration { + enum "auto"; + enum "csv"; + enum "json"; + } +} + +typedef stage { + type enumeration { + enum "FileSourceStage"; + enum "DeserializeStage"; + enum "AbpPcapPreprocessingStage"; + enum "MonitorStage"; + enum "TritonInferenceStage"; + enum "AddClassificationsStage"; + enum "SerializeStage"; + enum "WriteToFileStage"; + } +} + +grouping prob { + leaf value { + type string; + description "probs"; + } +} + +grouping conf { + leaf mode {type string; } + leaf num_threads { type uint16; } + leaf pipeline_batch_size { type uint64; } + leaf model_max_batch_size { type uint64; } + leaf model_fea_length { type uint64; } + list class_labels { + key "value"; + uses prob; } +} + grouping morpheus_pipeline_stage { leaf name { type string; } - // enum stage type - // stage parameters - // soportar custom stages - // https://github.com/nv-morpheus/Morpheus/blob/branch-23.11/examples/abp_pcap_detection/run.py - + leaf stage_type {type stage; } + list configuration {key "mode"; uses conf;} + leaf filename {type string;} + leaf iterative {type boolean;} + leaf file_type {type files;} + leaf filter_null {type boolean;} + leaf descriptions {type string;} + leaf model_name {type string;} + leaf server_url {type string;} + leaf force_convert_inputs {type boolean;} + leaf unit {type string;} + list labels {key "value"; uses prob;} + leaf kwargs {type empty;} + leaf overwrite {type boolean;} } +} \ No newline at end of file diff --git a/ztp_device.yang b/ztp_device.yang new file mode 100644 index 000000000..748bc54c3 --- /dev/null +++ b/ztp_device.yang @@ -0,0 +1,43 @@ +module ztp_device { + + yang-version "1"; + + namespace "urn:ztp_nos"; + prefix "ztp_nos"; + organization "EVIDEN"; + contact "jose.carcel@eviden.com"; + description "Basic example of data model for modelling NOS ZTP Device"; + + revision "2023-08-03" { + description "Basic example of data model for modelling NOS ZTP Device"; + reference ""; + } + + grouping config{ + leaf ztp_device_sw_url{ + type string; + } + leaf config_script_url{ + type string; + } + } + + typedef state { + type enumeration{ + enum "UNDEF"; + enum "INIT_HW"; + enum "INIT_SW"; + enum "CONFIGURED"; + enum "RUNNING"; + enum "UPDATING"; + enum "MIGRATING"; + enum "DELETED"; + } + } + + grouping state { + leaf ztp_device_state { + type state; + } + } +} \ No newline at end of file -- GitLab From 432b8193084f06f381afad5bf2dac269de747d84 Mon Sep 17 00:00:00 2001 From: carcel Date: Tue, 3 Oct 2023 17:26:37 +0200 Subject: [PATCH 009/941] Update SmartNIC, NOS and NS Data Models --- proto/any.proto | 162 +++++ proto/context.proto | 3 +- ...nics.proto => context_ext_smartnics.proto} | 37 +- proto/nfv_client.proto | 11 +- proto/nos_client.proto | 33 + .../automation/src/ztp-agent/ztp_device.yang | 3 + .../smartnic_probes/ietf-yang-types.yang | 480 +++++++++++++++ .../openconfig-extensions.yang | 206 +++++++ .../openconfig-inet-types.yang | 478 +++++++++++++++ .../openconfig-probes-types.yang | 86 +++ .../smartnic_probes/openconfig-probes.yang | 575 ++++++++++++++++++ .../smartnic_probes/openconfig-types.yang | 471 ++++++++++++++ .../drivers/smartnic_probes/probes-agent.yang | 150 +++-- .../references_probes_libraries.txt | 6 + 14 files changed, 2611 insertions(+), 90 deletions(-) create mode 100644 proto/any.proto rename proto/{context-ext-smartnics.proto => context_ext_smartnics.proto} (79%) create mode 100644 proto/nos_client.proto rename ztp_device.yang => src/automation/src/ztp-agent/ztp_device.yang (91%) create mode 100644 src/device/service/drivers/smartnic_probes/ietf-yang-types.yang create mode 100644 src/device/service/drivers/smartnic_probes/openconfig-extensions.yang create mode 100644 src/device/service/drivers/smartnic_probes/openconfig-inet-types.yang create mode 100644 src/device/service/drivers/smartnic_probes/openconfig-probes-types.yang create mode 100644 src/device/service/drivers/smartnic_probes/openconfig-probes.yang create mode 100644 src/device/service/drivers/smartnic_probes/openconfig-types.yang create mode 100644 src/device/service/drivers/smartnic_probes/references_probes_libraries.txt diff --git a/proto/any.proto b/proto/any.proto new file mode 100644 index 000000000..eff44e509 --- /dev/null +++ b/proto/any.proto @@ -0,0 +1,162 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// // or ... +// if (any.isSameTypeAs(Foo.getDefaultInstance())) { +// foo = any.unpack(Foo.getDefaultInstance()); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// JSON +// ==== +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. As of May 2023, there are no widely used type server + // implementations and no plans to implement one. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/proto/context.proto b/proto/context.proto index 87680a5b9..856d6f71d 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -15,6 +15,7 @@ syntax = "proto3"; package context; +import "any.proto"; import "acl.proto"; import "kpi_sample_types.proto"; @@ -451,7 +452,7 @@ message EndPoint { string endpoint_type = 3; // == 'smartnics' repeated kpi_sample_types.KpiSampleType kpi_sample_types = 4; Location endpoint_location = 5; - map capabilities = 6; + map capabilities = 6; } message EndPointName { diff --git a/proto/context-ext-smartnics.proto b/proto/context_ext_smartnics.proto similarity index 79% rename from proto/context-ext-smartnics.proto rename to proto/context_ext_smartnics.proto index 953b27e86..fe834b493 100644 --- a/proto/context-ext-smartnics.proto +++ b/proto/context_ext_smartnics.proto @@ -19,9 +19,8 @@ // conv_accel = bluefield-2 + GPU (A30) syntax = "proto3"; -package context-ext-smartnics; +package context_ext_smartnics; -import "context"; import "kpi_sample_types.proto"; message SmartnicsCapabilities { @@ -82,40 +81,14 @@ message DPU_Core { message DPU_Memory { string RamMemoryType = 1; string eMMCMemoryType = 2; - DpuRamMemorySizeGB RamMemorySizeGB = 3; - DpueMMCMemorySizeGB eMMCMemorySizeGB = 4; + uint64 RamMemorySizeGB = 3; + uint64 eMMCMemorySizeGB = 4; } -enum DpuRamMemorySizeGB { - DPU_MEMORY_DDR4_RAM_UNDEFINED = 0; - DPU_MEMORY_DDR4_RAM_16GB = 1; - DPU_MEMORY_DDR4_RAM_32GB = 2; -} - -enum DpueMMCMemorySizeGB { - DPU_MEMORY_eMMC_UNDEFINED = 0; - DPU_MEMORY_eMMC_32GB = 1; - DPU_MEMORY_eMMC_64GB = 2; - DPU_MEMORY_eMMC_128GB = 3; -} - -enum GPUArchitecture { - ARCH_UNDEFINED = 0; - ARCH_AMPERE = 1; -} - -enum GPUComputeCapabilities { - MIG_UNDEFINED = 0; - MIG_4_AT_6GB = 1; - MIG_2_AT_12GB = 2; - MIG_1_AT_24GB = 3; -} - - message GPU { string model = 1; - GPUArchitecture architecture = 2; - GPUComputeCapabilities compute_capabilities = 3; + string architecture = 2; + string compute_capabilities = 3; uint64 memory_size_gb = 4; uint32 num_CUDA_cores = 5; uint32 num_Tensor_cores = 6; diff --git a/proto/nfv_client.proto b/proto/nfv_client.proto index d880a394c..d903e7907 100644 --- a/proto/nfv_client.proto +++ b/proto/nfv_client.proto @@ -19,6 +19,7 @@ import "context.proto"; message Nsd { string nsd_name = 1; + string config_params=2; } message Ns { @@ -26,6 +27,9 @@ message Ns { string ns_name=2; string nsd_name=3; string vim_account=4; + string config_params=5; + string status = 6; + string status_message = 7; } message NsList { @@ -37,8 +41,11 @@ message NsdList { } service nfv_client { - rpc CreateNs (ns) returns (ns_id) {} // Stable not final - rpc DeleteNs (ns) returns (context.Empty) {} + rpc GetNsList (context.Empty) returns (NsList) {} + rpc GetNsdList (context.Empty) returns (NsdList) {} + rpc CreateNs (Ns) returns (Ns) {} + rpc UpdateNs (Ns) returns (Ns) {} + rpc DeleteNs (Ns) returns (context.Empty) {} } diff --git a/proto/nos_client.proto b/proto/nos_client.proto new file mode 100644 index 000000000..e0e35a648 --- /dev/null +++ b/proto/nos_client.proto @@ -0,0 +1,33 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package nos_client; + +message NOS_SW { + string ztp_device_sw_url = 1; + bytes ztp_device_sw_file = 2; +} + +message Config_Script { + string config_script_url = 1; + bytes config_script_file = 2; +} + +service nos_client { + rpc GetNOSFile (NOS_SW) returns (NOS_SW) {} + rpc GetConfigScriptFile (Config_Script) returns (Config_Script) {} +} + + diff --git a/ztp_device.yang b/src/automation/src/ztp-agent/ztp_device.yang similarity index 91% rename from ztp_device.yang rename to src/automation/src/ztp-agent/ztp_device.yang index 748bc54c3..f506d6413 100644 --- a/ztp_device.yang +++ b/src/automation/src/ztp-agent/ztp_device.yang @@ -20,6 +20,9 @@ module ztp_device { leaf config_script_url{ type string; } + leaf ip_range { + type string; // check IP regular expression + } } typedef state { diff --git a/src/device/service/drivers/smartnic_probes/ietf-yang-types.yang b/src/device/service/drivers/smartnic_probes/ietf-yang-types.yang new file mode 100644 index 000000000..371a091d1 --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/ietf-yang-types.yang @@ -0,0 +1,480 @@ +module ietf-yang-types { + + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-types"; + prefix "yang"; + + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; + + contact + "WG Web: + WG List: + + WG Chair: David Kessens + + + WG Chair: Juergen Schoenwaelder + + + Editor: Juergen Schoenwaelder + "; + + description + "This module contains a collection of generally useful derived + YANG data types. + + Copyright (c) 2013 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Simplified BSD License + set forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 6991; see + the RFC itself for full legal notices."; + + revision 2013-07-15 { + description + "This revision adds the following new data types: + - yang-identifier + - hex-string + - uuid + - dotted-quad"; + reference + "RFC 6991: Common YANG Data Types"; + } + + revision 2010-09-24 { + description + "Initial revision."; + reference + "RFC 6021: Common YANG Data Types"; + } + + /*** collection of counter and gauge types ***/ + + typedef counter32 { + type uint32; + description + "The counter32 type represents a non-negative integer + that monotonically increases until it reaches a + maximum value of 2^32-1 (4294967295 decimal), when it + wraps around and starts increasing again from zero. + + Counters have no defined 'initial' value, and thus, a + single value of a counter has (in general) no information + content. Discontinuities in the monotonically increasing + value normally occur at re-initialization of the + management system, and at other times as specified in the + description of a schema node using this type. If such + other times can occur, for example, the creation of + a schema node of type counter32 at times other than + re-initialization, then a corresponding schema node + should be defined, with an appropriate type, to indicate + the last discontinuity. + + The counter32 type should not be used for configuration + schema nodes. A default statement SHOULD NOT be used in + combination with the type counter32. + + In the value set and its semantics, this type is equivalent + to the Counter32 type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef zero-based-counter32 { + type yang:counter32; + default "0"; + description + "The zero-based-counter32 type represents a counter32 + that has the defined 'initial' value zero. + + A schema node of this type will be set to zero (0) on creation + and will thereafter increase monotonically until it reaches + a maximum value of 2^32-1 (4294967295 decimal), when it + wraps around and starts increasing again from zero. + + Provided that an application discovers a new schema node + of this type within the minimum time to wrap, it can use the + 'initial' value as a delta. It is important for a management + station to be aware of this minimum time and the actual time + between polls, and to discard data if the actual time is too + long or there is no defined minimum time. + + In the value set and its semantics, this type is equivalent + to the ZeroBasedCounter32 textual convention of the SMIv2."; + reference + "RFC 4502: Remote Network Monitoring Management Information + Base Version 2"; + } + + typedef counter64 { + type uint64; + description + "The counter64 type represents a non-negative integer + that monotonically increases until it reaches a + maximum value of 2^64-1 (18446744073709551615 decimal), + when it wraps around and starts increasing again from zero. + + Counters have no defined 'initial' value, and thus, a + single value of a counter has (in general) no information + content. Discontinuities in the monotonically increasing + value normally occur at re-initialization of the + management system, and at other times as specified in the + description of a schema node using this type. If such + other times can occur, for example, the creation of + a schema node of type counter64 at times other than + re-initialization, then a corresponding schema node + should be defined, with an appropriate type, to indicate + the last discontinuity. + + The counter64 type should not be used for configuration + schema nodes. A default statement SHOULD NOT be used in + combination with the type counter64. + + In the value set and its semantics, this type is equivalent + to the Counter64 type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef zero-based-counter64 { + type yang:counter64; + default "0"; + description + "The zero-based-counter64 type represents a counter64 that + has the defined 'initial' value zero. + + + + + A schema node of this type will be set to zero (0) on creation + and will thereafter increase monotonically until it reaches + a maximum value of 2^64-1 (18446744073709551615 decimal), + when it wraps around and starts increasing again from zero. + + Provided that an application discovers a new schema node + of this type within the minimum time to wrap, it can use the + 'initial' value as a delta. It is important for a management + station to be aware of this minimum time and the actual time + between polls, and to discard data if the actual time is too + long or there is no defined minimum time. + + In the value set and its semantics, this type is equivalent + to the ZeroBasedCounter64 textual convention of the SMIv2."; + reference + "RFC 2856: Textual Conventions for Additional High Capacity + Data Types"; + } + + typedef gauge32 { + type uint32; + description + "The gauge32 type represents a non-negative integer, which + may increase or decrease, but shall never exceed a maximum + value, nor fall below a minimum value. The maximum value + cannot be greater than 2^32-1 (4294967295 decimal), and + the minimum value cannot be smaller than 0. The value of + a gauge32 has its maximum value whenever the information + being modeled is greater than or equal to its maximum + value, and has its minimum value whenever the information + being modeled is smaller than or equal to its minimum value. + If the information being modeled subsequently decreases + below (increases above) the maximum (minimum) value, the + gauge32 also decreases (increases). + + In the value set and its semantics, this type is equivalent + to the Gauge32 type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef gauge64 { + type uint64; + description + "The gauge64 type represents a non-negative integer, which + may increase or decrease, but shall never exceed a maximum + value, nor fall below a minimum value. The maximum value + cannot be greater than 2^64-1 (18446744073709551615), and + the minimum value cannot be smaller than 0. The value of + a gauge64 has its maximum value whenever the information + being modeled is greater than or equal to its maximum + value, and has its minimum value whenever the information + being modeled is smaller than or equal to its minimum value. + If the information being modeled subsequently decreases + below (increases above) the maximum (minimum) value, the + gauge64 also decreases (increases). + + In the value set and its semantics, this type is equivalent + to the CounterBasedGauge64 SMIv2 textual convention defined + in RFC 2856"; + reference + "RFC 2856: Textual Conventions for Additional High Capacity + Data Types"; + } + + /*** collection of identifier-related types ***/ + + typedef object-identifier { + type string { + pattern '(([0-1](\.[1-3]?[0-9]))|(2\.(0|([1-9]\d*))))' + + '(\.(0|([1-9]\d*)))*'; + } + description + "The object-identifier type represents administratively + assigned names in a registration-hierarchical-name tree. + + Values of this type are denoted as a sequence of numerical + non-negative sub-identifier values. Each sub-identifier + value MUST NOT exceed 2^32-1 (4294967295). Sub-identifiers + are separated by single dots and without any intermediate + whitespace. + + The ASN.1 standard restricts the value space of the first + sub-identifier to 0, 1, or 2. Furthermore, the value space + of the second sub-identifier is restricted to the range + 0 to 39 if the first sub-identifier is 0 or 1. Finally, + the ASN.1 standard requires that an object identifier + has always at least two sub-identifiers. The pattern + captures these restrictions. + + Although the number of sub-identifiers is not limited, + module designers should realize that there may be + implementations that stick with the SMIv2 limit of 128 + sub-identifiers. + + This type is a superset of the SMIv2 OBJECT IDENTIFIER type + since it is not restricted to 128 sub-identifiers. Hence, + this type SHOULD NOT be used to represent the SMIv2 OBJECT + IDENTIFIER type; the object-identifier-128 type SHOULD be + used instead."; + reference + "ISO9834-1: Information technology -- Open Systems + Interconnection -- Procedures for the operation of OSI + Registration Authorities: General procedures and top + arcs of the ASN.1 Object Identifier tree"; + } + + typedef object-identifier-128 { + type object-identifier { + pattern '\d*(\.\d*){1,127}'; + } + description + "This type represents object-identifiers restricted to 128 + sub-identifiers. + + In the value set and its semantics, this type is equivalent + to the OBJECT IDENTIFIER type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef yang-identifier { + type string { + length "1..max"; + pattern '[a-zA-Z_][a-zA-Z0-9\-_.]*'; + pattern '.|..|[^xX].*|.[^mM].*|..[^lL].*'; + } + description + "A YANG identifier string as defined by the 'identifier' + rule in Section 12 of RFC 6020. An identifier must + start with an alphabetic character or an underscore + followed by an arbitrary sequence of alphabetic or + numeric characters, underscores, hyphens, or dots. + + A YANG identifier MUST NOT start with any possible + combination of the lowercase or uppercase character + sequence 'xml'."; + reference + "RFC 6020: YANG - A Data Modeling Language for the Network + Configuration Protocol (NETCONF)"; + } + + /*** collection of types related to date and time***/ + + typedef date-and-time { + type string { + pattern '\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?' + + '(Z|[\+\-]\d{2}:\d{2})'; + } + description + "The date-and-time type is a profile of the ISO 8601 + standard for representation of dates and times using the + Gregorian calendar. The profile is defined by the + date-time production in Section 5.6 of RFC 3339. + + The date-and-time type is compatible with the dateTime XML + schema type with the following notable exceptions: + + (a) The date-and-time type does not allow negative years. + + (b) The date-and-time time-offset -00:00 indicates an unknown + time zone (see RFC 3339) while -00:00 and +00:00 and Z + all represent the same time zone in dateTime. + + (c) The canonical format (see below) of data-and-time values + differs from the canonical format used by the dateTime XML + schema type, which requires all times to be in UTC using + the time-offset 'Z'. + + This type is not equivalent to the DateAndTime textual + convention of the SMIv2 since RFC 3339 uses a different + separator between full-date and full-time and provides + higher resolution of time-secfrac. + + The canonical format for date-and-time values with a known time + zone uses a numeric time zone offset that is calculated using + the device's configured known offset to UTC time. A change of + the device's offset to UTC time will cause date-and-time values + to change accordingly. Such changes might happen periodically + in case a server follows automatically daylight saving time + (DST) time zone offset changes. The canonical format for + date-and-time values with an unknown time zone (usually + referring to the notion of local time) uses the time-offset + -00:00."; + reference + "RFC 3339: Date and Time on the Internet: Timestamps + RFC 2579: Textual Conventions for SMIv2 + XSD-TYPES: XML Schema Part 2: Datatypes Second Edition"; + } + + typedef timeticks { + type uint32; + description + "The timeticks type represents a non-negative integer that + represents the time, modulo 2^32 (4294967296 decimal), in + hundredths of a second between two epochs. When a schema + node is defined that uses this type, the description of + the schema node identifies both of the reference epochs. + + In the value set and its semantics, this type is equivalent + to the TimeTicks type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef timestamp { + type yang:timeticks; + description + "The timestamp type represents the value of an associated + timeticks schema node at which a specific occurrence + happened. The specific occurrence must be defined in the + description of any schema node defined using this type. When + the specific occurrence occurred prior to the last time the + associated timeticks attribute was zero, then the timestamp + value is zero. Note that this requires all timestamp values + to be reset to zero when the value of the associated timeticks + attribute reaches 497+ days and wraps around to zero. + + The associated timeticks schema node must be specified + in the description of any schema node using this type. + + In the value set and its semantics, this type is equivalent + to the TimeStamp textual convention of the SMIv2."; + reference + "RFC 2579: Textual Conventions for SMIv2"; + } + + /*** collection of generic address types ***/ + + typedef phys-address { + type string { + pattern '([0-9a-fA-F]{2}(:[0-9a-fA-F]{2})*)?'; + } + + + + + description + "Represents media- or physical-level addresses represented + as a sequence octets, each octet represented by two hexadecimal + numbers. Octets are separated by colons. The canonical + representation uses lowercase characters. + + In the value set and its semantics, this type is equivalent + to the PhysAddress textual convention of the SMIv2."; + reference + "RFC 2579: Textual Conventions for SMIv2"; + } + + typedef mac-address { + type string { + pattern '[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}'; + } + description + "The mac-address type represents an IEEE 802 MAC address. + The canonical representation uses lowercase characters. + + In the value set and its semantics, this type is equivalent + to the MacAddress textual convention of the SMIv2."; + reference + "IEEE 802: IEEE Standard for Local and Metropolitan Area + Networks: Overview and Architecture + RFC 2579: Textual Conventions for SMIv2"; + } + + /*** collection of XML-specific types ***/ + + typedef xpath1.0 { + type string; + description + "This type represents an XPATH 1.0 expression. + + When a schema node is defined that uses this type, the + description of the schema node MUST specify the XPath + context in which the XPath expression is evaluated."; + reference + "XPATH: XML Path Language (XPath) Version 1.0"; + } + + /*** collection of string types ***/ + + typedef hex-string { + type string { + pattern '([0-9a-fA-F]{2}(:[0-9a-fA-F]{2})*)?'; + } + description + "A hexadecimal string with octets represented as hex digits + separated by colons. The canonical representation uses + lowercase characters."; + } + + typedef uuid { + type string { + pattern '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-' + + '[0-9a-fA-F]{4}-[0-9a-fA-F]{12}'; + } + description + "A Universally Unique IDentifier in the string representation + defined in RFC 4122. The canonical representation uses + lowercase characters. + + The following is an example of a UUID in string representation: + f81d4fae-7dec-11d0-a765-00a0c91e6bf6 + "; + reference + "RFC 4122: A Universally Unique IDentifier (UUID) URN + Namespace"; + } + + typedef dotted-quad { + type string { + pattern + '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'; + } + description + "An unsigned 32-bit number expressed in the dotted-quad + notation, i.e., four octets written as decimal numbers + and separated with the '.' (full stop) character."; + } +} diff --git a/src/device/service/drivers/smartnic_probes/openconfig-extensions.yang b/src/device/service/drivers/smartnic_probes/openconfig-extensions.yang new file mode 100644 index 000000000..2e0fd9f07 --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/openconfig-extensions.yang @@ -0,0 +1,206 @@ +module openconfig-extensions { + + yang-version "1"; + + // namespace + namespace "http://openconfig.net/yang/openconfig-ext"; + + prefix "oc-ext"; + + // meta + organization "OpenConfig working group"; + + contact + "OpenConfig working group + www.openconfig.net"; + + description + "This module provides extensions to the YANG language to allow + OpenConfig specific functionality and meta-data to be defined."; + + oc-ext:openconfig-version "0.5.1"; + + revision "2022-10-05" { + description + "Add missing version statement."; + reference "0.5.1"; + } + + revision "2020-06-16" { + description + "Add extension for POSIX pattern statements."; + reference "0.5.0"; + } + + revision "2018-10-17" { + description + "Add extension for regular expression type."; + reference "0.4.0"; + } + + revision "2017-04-11" { + description + "rename password type to 'hashed' and clarify description"; + reference "0.3.0"; + } + + revision "2017-01-29" { + description + "Added extension for annotating encrypted values."; + reference "0.2.0"; + } + + revision "2015-10-09" { + description + "Initial OpenConfig public release"; + reference "0.1.0"; + } + + + // extension statements + extension openconfig-version { + argument "semver" { + yin-element false; + } + description + "The OpenConfig version number for the module. This is + expressed as a semantic version number of the form: + x.y.z + where: + * x corresponds to the major version, + * y corresponds to a minor version, + * z corresponds to a patch version. + This version corresponds to the model file within which it is + defined, and does not cover the whole set of OpenConfig models. + + Individual YANG modules are versioned independently -- the + semantic version is generally incremented only when there is a + change in the corresponding file. Submodules should always + have the same semantic version as their parent modules. + + A major version number of 0 indicates that this model is still + in development (whether within OpenConfig or with industry + partners), and is potentially subject to change. + + Following a release of major version 1, all modules will + increment major revision number where backwards incompatible + changes to the model are made. + + The minor version is changed when features are added to the + model that do not impact current clients use of the model. + + The patch-level version is incremented when non-feature changes + (such as bugfixes or clarifications to human-readable + descriptions that do not impact model functionality) are made + that maintain backwards compatibility. + + The version number is stored in the module meta-data."; + } + + extension openconfig-hashed-value { + description + "This extension provides an annotation on schema nodes to + indicate that the corresponding value should be stored and + reported in hashed form. + + Hash algorithms are by definition not reversible. Clients + reading the configuration or applied configuration for the node + should expect to receive only the hashed value. Values written + in cleartext will be hashed. This annotation may be used on + nodes such as secure passwords in which the device never reports + a cleartext value, even if the input is provided as cleartext."; + } + + extension regexp-posix { + description + "This extension indicates that the regular expressions included + within the YANG module specified are conformant with the POSIX + regular expression format rather than the W3C standard that is + specified by RFC6020 and RFC7950."; + } + + extension posix-pattern { + argument "pattern" { + yin-element false; + } + description + "Provides a POSIX ERE regular expression pattern statement as an + alternative to YANG regular expresssions based on XML Schema Datatypes. + It is used the same way as the standard YANG pattern statement defined in + RFC6020 and RFC7950, but takes an argument that is a POSIX ERE regular + expression string."; + reference + "POSIX Extended Regular Expressions (ERE) Specification: + https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04"; + } + + extension telemetry-on-change { + description + "The telemetry-on-change annotation is specified in the context + of a particular subtree (container, or list) or leaf within the + YANG schema. Where specified, it indicates that the value stored + by the nodes within the context change their value only in response + to an event occurring. The event may be local to the target, for + example - a configuration change, or external - such as the failure + of a link. + + When a telemetry subscription allows the target to determine whether + to export the value of a leaf in a periodic or event-based fashion + (e.g., TARGET_DEFINED mode in gNMI), leaves marked as + telemetry-on-change should only be exported when they change, + i.e., event-based."; + } + + extension telemetry-atomic { + description + "The telemetry-atomic annotation is specified in the context of + a subtree (containre, or list), and indicates that all nodes + within the subtree are always updated together within the data + model. For example, all elements under the subtree may be updated + as a result of a new alarm being raised, or the arrival of a new + protocol message. + + Transport protocols may use the atomic specification to determine + optimisations for sending or storing the corresponding data."; + } + + extension operational { + description + "The operational annotation is specified in the context of a + grouping, leaf, or leaf-list within a YANG module. It indicates + that the nodes within the context are derived state on the device. + + OpenConfig data models divide nodes into the following three categories: + + - intended configuration - these are leaves within a container named + 'config', and are the writable configuration of a target. + - applied configuration - these are leaves within a container named + 'state' and are the currently running value of the intended configuration. + - derived state - these are the values within the 'state' container which + are not part of the applied configuration of the device. Typically, they + represent state values reflecting underlying operational counters, or + protocol statuses."; + } + + extension catalog-organization { + argument "org" { + yin-element false; + } + description + "This extension specifies the organization name that should be used within + the module catalogue on the device for the specified YANG module. It stores + a pithy string where the YANG organization statement may contain more + details."; + } + + extension origin { + argument "origin" { + yin-element false; + } + description + "This extension specifies the name of the origin that the YANG module + falls within. This allows multiple overlapping schema trees to be used + on a single network element without requiring module based prefixing + of paths."; + } +} diff --git a/src/device/service/drivers/smartnic_probes/openconfig-inet-types.yang b/src/device/service/drivers/smartnic_probes/openconfig-inet-types.yang new file mode 100644 index 000000000..3d3ed425e --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/openconfig-inet-types.yang @@ -0,0 +1,478 @@ +module openconfig-inet-types { + + yang-version "1"; + namespace "http://openconfig.net/yang/types/inet"; + prefix "oc-inet"; + + import openconfig-extensions { prefix "oc-ext"; } + + organization + "OpenConfig working group"; + + contact + "OpenConfig working group + www.openconfig.net"; + + description + "This module contains a set of Internet address related + types for use in OpenConfig modules. + + Portions of this code were derived from IETF RFC 6021. + Please reproduce this note if possible. + + IETF code is subject to the following copyright and license: + Copyright (c) IETF Trust and the persons identified as authors of + the code. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, is permitted pursuant to, and subject to the license + terms contained in, the Simplified BSD License set forth in + Section 4.c of the IETF Trust's Legal Provisions Relating + to IETF Documents (http://trustee.ietf.org/license-info)."; + + oc-ext:openconfig-version "0.6.0"; + + revision "2023-02-06" { + description + "Add ipv6-link-local and ipv6-address-type"; + reference "0.6.0"; + } + + revision "2021-08-17" { + description + "Add ip-address-zoned typedef as a union between ipv4-address-zoned + and ipv6-address-zoned types."; + reference "0.5.0"; + } + + revision "2021-07-14" { + description + "Use auto-generated regex for ipv4 pattern statements: + - ipv4-address + - ipv4-address-zoned + - ipv4-prefix"; + reference "0.4.1"; + } + + revision "2021-01-07" { + description + "Remove module extension oc-ext:regexp-posix by making pattern regexes + conform to RFC7950. + + Types impacted: + - ipv4-address + - ipv4-address-zoned + - ipv6-address + - domain-name"; + reference "0.4.0"; + } + + revision "2020-10-12" { + description + "Fix anchors for domain-name pattern."; + reference "0.3.5"; + } + + revision "2020-06-30" { + description + "Add OpenConfig POSIX pattern extensions and add anchors for domain-name + pattern."; + reference "0.3.4"; + } + + revision "2019-04-25" { + description + "Fix regex bug for ipv6-prefix type"; + reference "0.3.3"; + } + + revision "2018-11-21" { + description + "Add OpenConfig module metadata extensions."; + reference "0.3.2"; + } + + revision 2017-08-24 { + description + "Minor formatting fixes."; + reference "0.3.1"; + } + + revision 2017-07-06 { + description + "Add domain-name and host typedefs"; + reference "0.3.0"; + } + + revision 2017-04-03 { + description + "Add ip-version typedef."; + reference "0.2.0"; + } + + revision 2017-04-03 { + description + "Update copyright notice."; + reference "0.1.1"; + } + + revision 2017-01-26 { + description + "Initial module for inet types"; + reference "0.1.0"; + } + + // OpenConfig specific extensions for module metadata. + oc-ext:catalog-organization "openconfig"; + oc-ext:origin "openconfig"; + + // IPv4 and IPv6 types. + + typedef ipv4-address { + type string { + pattern + '([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\.([0-9]|' + + '[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3}'; + oc-ext:posix-pattern + '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\.([0-9]|' + + '[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3})$'; + } + description + "An IPv4 address in dotted quad notation using the default + zone."; + } + + typedef ipv4-address-zoned { + type string { + pattern + '([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\.([0-9]|' + + '[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3}(%[a-zA-Z0-9_]+)'; + oc-ext:posix-pattern + '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\.([0-9]|' + + '[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3}(%[a-zA-Z0-9_]+))$'; + } + description + "An IPv4 address in dotted quad notation. This type allows + specification of a zone index to disambiguate identical + address values. For link-local addresses, the index is + typically the interface index or interface name."; + } + + typedef ipv6-address { + type string { + pattern + // Must support compression through different lengths + // therefore this regexp is complex. + '(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,7}:|' + + '([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|' + + '([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' + + '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|' + + '([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' + + '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|' + + ':((:[0-9a-fA-F]{1,4}){1,7}|:)' + + ')'; + oc-ext:posix-pattern + // Must support compression through different lengths + // therefore this regexp is complex. + '^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,7}:|' + + '([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|' + + '([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' + + '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|' + + '([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' + + '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|' + + ':((:[0-9a-fA-F]{1,4}){1,7}|:)' + + ')$'; + } + description + "An IPv6 address represented as either a full address; shortened + or mixed-shortened formats, using the default zone."; + } + + typedef ipv6-address-zoned { + type string { + pattern + // Must support compression through different lengths + // therefore this regexp is complex. + '^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,7}:|' + + '([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|' + + '([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' + + '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|' + + '([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' + + '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|' + + ':((:[0-9a-fA-F]{1,4}){1,7}|:)' + + ')(%[a-zA-Z0-9_]+)$'; + oc-ext:posix-pattern + // Must support compression through different lengths + // therefore this regexp is complex. + '^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,7}:|' + + '([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|' + + '([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' + + '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|' + + '([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' + + '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|' + + ':((:[0-9a-fA-F]{1,4}){1,7}|:)' + + ')(%[a-zA-Z0-9_]+)$'; + } + description + "An IPv6 address represented as either a full address; shortened + or mixed-shortened formats. This type allows specification of + a zone index to disambiguate identical address values. For + link-local addresses, the index is typically the interface + index or interface name."; + } + + typedef ipv4-prefix { + type string { + pattern + '([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\.([0-9]|' + + '[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3}/([0-9]|[12][0-9]|' + + '3[0-2])'; + oc-ext:posix-pattern + '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\.([0-9]|' + + '[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])){3}/([0-9]|[12][0-9]|' + + '3[0-2]))$'; + } + description + "An IPv4 prefix represented in dotted quad notation followed by + a slash and a CIDR mask (0 <= mask <= 32)."; + } + + typedef ipv6-prefix { + type string { + pattern + '(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,7}:|' + + '([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|' + + '([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' + + '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|' + + '([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' + + '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|' + + ':((:[0-9a-fA-F]{1,4}){1,7}|:)' + + ')/(12[0-8]|1[0-1][0-9]|[1-9][0-9]|[0-9])'; + oc-ext:posix-pattern + '^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,7}:|' + + '([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' + + '([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|' + + '([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' + + '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|' + + '([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' + + '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|' + + ':((:[0-9a-fA-F]{1,4}){1,7}|:)' + + ')/(12[0-8]|1[0-1][0-9]|[1-9][0-9]|[0-9])$'; + } + description + "An IPv6 prefix represented in full, shortened, or mixed + shortened format followed by a slash and CIDR mask + (0 <= mask <= 128)."; + } + + typedef ip-address { + type union { + type ipv4-address; + type ipv6-address; + } + description + "An IPv4 or IPv6 address with no prefix specified."; + } + + typedef ip-address-zoned { + type union { + type ipv4-address-zoned; + type ipv6-address-zoned; + } + description + "An IPv4 or IPv6 address with no prefix specified and an optional + zone index."; + } + + typedef ip-prefix { + type union { + type ipv4-prefix; + type ipv6-prefix; + } + description + "An IPv4 or IPv6 prefix."; + } + + typedef ip-version { + type enumeration { + enum UNKNOWN { + value 0; + description + "An unknown or unspecified version of the Internet + protocol."; + } + enum IPV4 { + value 4; + description + "The IPv4 protocol as defined in RFC 791."; + } + enum IPV6 { + value 6; + description + "The IPv6 protocol as defined in RFC 2460."; + } + } + description + "This value represents the version of the IP protocol. + Note that integer representation of the enumerated values + are not specified, and are not required to follow the + InetVersion textual convention in SMIv2."; + reference + "RFC 791: Internet Protocol + RFC 2460: Internet Protocol, Version 6 (IPv6) Specification + RFC 4001: Textual Conventions for Internet Network Addresses"; + } + + typedef ipv6-address-type { + type enumeration { + enum GLOBAL_UNICAST { + description + "The IPv6 address is a global unicast address type and must be in + the format defined in RFC 4291 section 2.4."; + } + enum LINK_LOCAL_UNICAST { + description + "The IPv6 address is a Link-Local unicast address type and must be + in the format defined in RFC 4291 section 2.4."; + } + } + description + "The value represents the type of IPv6 address"; + reference + "RFC 4291: IP Version 6 Addressing Architecture + section 2.5"; + } + + typedef domain-name { + type string { + length "1..253"; + pattern + '(((([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.)*' + + '([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.?)' + + '|\.)'; + oc-ext:posix-pattern + '^(((([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.)*' + + '([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.?)' + + '|\.)$'; + } + description + "The domain-name type represents a DNS domain name. + Fully quallified left to the models which utilize this type. + + Internet domain names are only loosely specified. Section + 3.5 of RFC 1034 recommends a syntax (modified in Section + 2.1 of RFC 1123). The pattern above is intended to allow + for current practice in domain name use, and some possible + future expansion. It is designed to hold various types of + domain names, including names used for A or AAAA records + (host names) and other records, such as SRV records. Note + that Internet host names have a stricter syntax (described + in RFC 952) than the DNS recommendations in RFCs 1034 and + 1123, and that systems that want to store host names in + schema nodes using the domain-name type are recommended to + adhere to this stricter standard to ensure interoperability. + + The encoding of DNS names in the DNS protocol is limited + to 255 characters. Since the encoding consists of labels + prefixed by a length bytes and there is a trailing NULL + byte, only 253 characters can appear in the textual dotted + notation. + + Domain-name values use the US-ASCII encoding. Their canonical + format uses lowercase US-ASCII characters. Internationalized + domain names MUST be encoded in punycode as described in RFC + 3492"; + } + + typedef host { + type union { + type ip-address; + type domain-name; + } + description + "The host type represents either an unzoned IP address or a DNS + domain name."; + } + + typedef as-number { + type uint32; + description + "A numeric identifier for an autonomous system (AS). An AS is a + single domain, under common administrative control, which forms + a unit of routing policy. Autonomous systems can be assigned a + 2-byte identifier, or a 4-byte identifier which may have public + or private scope. Private ASNs are assigned from dedicated + ranges. Public ASNs are assigned from ranges allocated by IANA + to the regional internet registries (RIRs)."; + reference + "RFC 1930 Guidelines for creation, selection, and registration + of an Autonomous System (AS) + RFC 4271 A Border Gateway Protocol 4 (BGP-4)"; + } + + typedef dscp { + type uint8 { + range "0..63"; + } + description + "A differentiated services code point (DSCP) marking within the + IP header."; + reference + "RFC 2474 Definition of the Differentiated Services Field + (DS Field) in the IPv4 and IPv6 Headers"; + } + + typedef ipv6-flow-label { + type uint32 { + range "0..1048575"; + } + description + "The IPv6 flow-label is a 20-bit value within the IPv6 header + which is optionally used by the source of the IPv6 packet to + label sets of packets for which special handling may be + required."; + reference + "RFC 2460 Internet Protocol, Version 6 (IPv6) Specification"; + } + + typedef port-number { + type uint16; + description + "A 16-bit port number used by a transport protocol such as TCP + or UDP."; + reference + "RFC 768 User Datagram Protocol + RFC 793 Transmission Control Protocol"; + } + + typedef uri { + type string; + description + "An ASCII-encoded Uniform Resource Identifier (URI) as defined + in RFC 3986."; + reference + "RFC 3986 Uniform Resource Identifier (URI): Generic Syntax"; + } + + typedef url { + type string; + description + "An ASCII-encoded Uniform Resource Locator (URL) as defined + in RFC 3986, section 1.1.3"; + reference + "RFC 3986, paragraph 1.1.3"; + } + +} diff --git a/src/device/service/drivers/smartnic_probes/openconfig-probes-types.yang b/src/device/service/drivers/smartnic_probes/openconfig-probes-types.yang new file mode 100644 index 000000000..c5e13f370 --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/openconfig-probes-types.yang @@ -0,0 +1,86 @@ +module openconfig-probes-types { + + yang-version "1"; + + // namespace + namespace "http://openconfig.net/yang/probes/types"; + + prefix "oc-probes-types"; + + // import some basic types + import openconfig-extensions { prefix oc-ext; } + + // meta + organization "OpenConfig working group"; + + contact + "OpenConfig working group + www.openconfig.net"; + + description + "This module defines types related to the probes."; + + oc-ext:openconfig-version "0.1.1"; + + revision "2018-11-21" { + description + "Add OpenConfig module metadata extensions."; + reference "0.1.1"; + } + + revision "2017-09-05" { + description + "Initial public revision"; + reference "0.1.0"; + } + + // OpenConfig specific extensions for module metadata. + oc-ext:regexp-posix; + oc-ext:catalog-organization "openconfig"; + oc-ext:origin "openconfig"; + + typedef test-type { + type enumeration { + enum ICMP { + description + "Send ICMP echo requests."; + } + enum ICMP6 { + description + "Send ICMP6 echo requests."; + } + enum ICMP_TIMESTAMP { + description + "Send ICMP timestamp requests."; + } + enum ICMP6_TIMESTAMP { + description + "Sedn ICMP6 timestamp requests."; + } + enum TCP { + description + "Send TPC packets."; + } + enum UDP { + description + "Send UDP packets."; + } + enum UDP_TIMESTAMP { + description + "Send UDP packets with timestamp."; + } + enum HTTP_GET { + description + "Execute HTTP GET requests."; + } + enum HTTP_GET_META { + description + "Execute HTTP GET requests of metadata."; + } + } + description + "Type definition with enumerations describing the basis of + the probe test type identifier"; + } + +} \ No newline at end of file diff --git a/src/device/service/drivers/smartnic_probes/openconfig-probes.yang b/src/device/service/drivers/smartnic_probes/openconfig-probes.yang new file mode 100644 index 000000000..27b7e4c0d --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/openconfig-probes.yang @@ -0,0 +1,575 @@ +module openconfig-probes { + + // namespace + namespace "http://openconfig.net/yang/probes"; + + prefix "oc-probes"; + + import ietf-yang-types { prefix yang; } + import openconfig-types { prefix oc-types; } + import openconfig-extensions { prefix oc-ext; } + import openconfig-inet-types { prefix oc-inet; } + import openconfig-probes-types { prefix oc-probes-types; } + + organization "OpenConfig working group"; + + contact + "OpenConfig working group + www.openconfig.net"; + + description + "This module defines configuration and operational state data + for the probes. + A probe consists on a group of tests, each test being a + source-destination pair to poll. The destination can be either + IP Address (and eventually port) or URL, depending on the + nature of the test. The test can send ICMP, UDP, TCP, or HTTP + requests. + Each test groups a list of test items, the test results + being an overall view or average of the items list. + However, the test preserves only a limited set of history + items, whose length can be controlled using the history-size."; + + oc-ext:openconfig-version "0.0.2"; + + revision "2018-11-21" { + description + "Add OpenConfig module metadata extensions."; + reference "0.0.2"; + } + + revision "2017-09-05" { + description + "Initial public revision"; + reference "0.0.1"; + } + + // OpenConfig specific extensions for module metadata. + oc-ext:regexp-posix; + oc-ext:catalog-organization "openconfig"; + oc-ext:origin "openconfig"; + + grouping test-target { + description + "Groups the config and state containers + for an individual test."; + + container target { + description + "The target configuration of the test. + The nature of the target depends on the probe type: + for HTTP probes we need to provide an URL to poll, + while ICMP probes require an IP address to monitor."; + + container config { + description + "Configuration data for the test target."; + + uses test-target-base; + } + + container state { + config false; + + description + "Operational data for the test target."; + + uses test-target-base; + } + } + } + + grouping test-target-base { + description + "Targe types for the probe test."; + + leaf address { + type oc-inet:ip-address; + description + "IP address of the target, either IPv4 or IPv6."; + } + + leaf port { + type oc-inet:port-number; + description + "Destination port."; + } + + leaf url { + type oc-inet:url; + description + "Target URL to probe."; + } + } + + grouping probe-test-config-base { + description + "Definition of test details."; + + leaf test-type { + type oc-probes-types:test-type; + description + "The type of the probe test."; + mandatory true; + } + + leaf count { + type yang:counter64; + description + "The number of probes per test."; + } + + leaf interval { + type yang:counter64; + description + "Time between two consecutive probes."; + } + + leaf source { + type oc-inet:ip-address; + description + "Source address used when probing, either IPv4 or IPv6."; + } + + leaf history-size { + type yang:counter64; + description + "The number of history entries stored."; + } + + leaf source-port { + type oc-inet:port-number; + description + "Source number used."; + } + + leaf dscp { + type oc-inet:dscp; + description + "DSCP code points"; + } + + } + + grouping probe-test-state-history-item-base { + description + "The test item results counters and statistics. + An item presents the results of a single execution + of the test. + The results of the test depend on the values + of the total items, or an average over a certain + period of time."; + + leaf id { + type yang:counter64; + description + "The test item ID."; + } + + leaf timestamp { + type oc-types:timeticks64; + description + "The test timestamp. + This is not the timestamp when the test + was actually executed nither when it finished. + Should be the timestamp when the test has been scheduled. + It may not be the same with start-timestamp."; + } + + leaf start-timestamp { + type oc-types:timeticks64; + description + "The timestamp when the test started."; + } + + leaf end-timestamp { + type oc-types:timeticks64; + description + "The timestamp when the test finished."; + } + + leaf test-duration { + type yang:counter64; + description + "The duration of the test, in microseconds."; + } + + leaf failed { + type boolean; + description + "Whether the test failed or succeeded."; + } + + leaf probes-sent { + type yang:counter64; + description + "Number of test probes sent."; + } + + leaf probes-received { + type yang:counter64; + description + "Number of test probes received."; + } + + leaf loss-percentage { + type oc-types:percentage; + description + "The loss percentage."; + } + + leaf jitter { + type yang:counter64; + description + "The round trip jitter, in microseconds."; + } + + leaf min-delay { + type yang:counter64; + description + "The minimum delay recorded during the test, in microseconds."; + } + + leaf max-delay { + type yang:counter64; + description + "The maximum delay recorded during the test, in microseconds."; + } + + leaf avg-delay { + type yang:counter64; + description + "The average delay recorded during the test, in microseconds."; + } + + leaf stddev-delay { + type yang:counter64; + description + "The standard deviation of the delay of the test."; + } + + } + + grouping probe-test-state-history-item { + description + "A history item of the probe results."; + + container state { + + config false; + + description + "A history item of the probe results: operational data only."; + + uses probe-test-state-history-item-base; + } + + } + + grouping probe-test-state-history { + + description + "The history of the test results."; + + container items { + + description + "The list of items in the probe history. + The length depends on the history size."; + + list item { + key "id"; + description + "List of history items."; + + leaf id { + type leafref { + path "../state/id"; + } + description + "Reference to the history entry ID."; + } + + uses probe-test-state-history-item; + } + + } + + } + + grouping probe-test-state-results { + description + "The test results counters and statistics."; + + leaf timestamp { + type oc-types:timeticks64; + description + "The test timestamp. + This is not the timestamp when the test + was actually executed nither when it finished. + Should be the timestamp when the test has been scheduled. + It may not be the same with start-timestamp."; + } + + leaf start-timestamp { + type oc-types:timeticks64; + description + "The timestamp when the test started."; + } + + leaf last-test-timestamp { + type oc-types:timeticks64; + description + "The timestamp when the test finished."; + } + + leaf test-duration { + type yang:counter64; + description + "The duration of the test, in microseconds."; + } + + leaf failed { + type boolean; + description + "Whether the test failed or succeeded."; + } + + leaf probes-sent { + type yang:counter64; + description + "Number of test probes sent."; + } + + leaf probes-received { + type yang:counter64; + description + "Number of test probes received."; + } + + leaf loss-percentage { + type oc-types:percentage; + description + "The loss percentage."; + } + + leaf jitter { + type yang:counter64; + description + "The round trip jitter, in microseconds."; + } + + leaf min-delay { + type yang:counter64; + description + "The minimum delay recorded during the test, in microseconds."; + } + + leaf max-delay { + type yang:counter64; + description + "The maximum delay recorded during the test, in microseconds."; + } + + leaf avg-delay { + type yang:counter64; + description + "The average delay recorded during the test, in microseconds."; + } + + leaf stddev-delay { + type yang:counter64; + description + "The standard deviation of the delay of the test."; + } + + + } + + grouping probe-test-state { + + description + "Operational data and results for the probes."; + + } + + grouping probe-test-config { + description + "Definition of test details."; + + leaf name { + type string; + description + "The name of the test probe"; + mandatory true; + } + + leaf enabled { + type boolean; + default true; + description + "Whether the test is enabled."; + } + + uses probe-test-config-base; + + } + + grouping probe-tests-top { + description + "Top-level grouping for the tests withing a probe."; + + list test { + key "name"; + description + "List of tests associated with this probe."; + + leaf name { + type leafref { + path "../config/name"; + } + description + "Reference to the list key"; + } + + container config { + description + "Configuration data for the test of this probe."; + + uses probe-test-config; + } + + container state { + + config false; + + description + "Operational state data"; + + uses probe-test-config; + uses probe-test-state; + } + + uses test-target; + + container results { + description + "Contains the results of the tests."; + + container state { + + config false; + + description + "Results of this test: operational data only"; + + uses probe-test-state-results; + } + + container history { + + config false; + + description + "Historical data of the tests."; + + uses probe-test-state-history; + } + + } + + } + // end list of probes + + } + + grouping probe-config { + description + "Definition of probe details."; + + leaf name { + type string; + description + "The name of the probe."; + mandatory true; + } + + leaf enabled { + type boolean; + default true; + description + "Whether the probe is enabled."; + } + + } + + grouping probe-state { + description + "Definition of probes operation data."; + } + + grouping probes-top { + description + "Top-level grouping for probes model"; + + list probe { + key "name"; + description + "List of probes configured."; + + leaf name { + type leafref { + path "../config/name"; + } + description + "Reference to the list key"; + } + + container config { + description + "Configuration data for the probes."; + + uses probe-config; + } + + container state { + + config false; + + description + "Operational state data"; + + uses probe-config; + uses probe-state; + } + + container tests { + + description + "The tests associated to be executed for the probe."; + + uses probe-tests-top; + } + + } + // end list of probes + + } + + grouping openconfig-probes-top { + + description + "The top level grouping of the probes model."; + + container probes { + description + "The container containing the list of probes."; + + uses probes-top; + } + + } + + uses openconfig-probes-top; + +} \ No newline at end of file diff --git a/src/device/service/drivers/smartnic_probes/openconfig-types.yang b/src/device/service/drivers/smartnic_probes/openconfig-types.yang new file mode 100644 index 000000000..89e32d515 --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/openconfig-types.yang @@ -0,0 +1,471 @@ +module openconfig-types { + yang-version "1"; + + namespace "http://openconfig.net/yang/openconfig-types"; + + prefix "oc-types"; + + // import statements + import openconfig-extensions { prefix oc-ext; } + + // meta + organization + "OpenConfig working group"; + + contact + "OpenConfig working group + netopenconfig@googlegroups.com"; + + description + "This module contains a set of general type definitions that + are used across OpenConfig models. It can be imported by modules + that make use of these types."; + + oc-ext:openconfig-version "0.6.0"; + + revision "2019-04-16" { + description + "Clarify definition of timeticks64."; + reference "0.6.0"; + } + + revision "2018-11-21" { + description + "Add OpenConfig module metadata extensions."; + reference "0.5.1"; + } + + revision "2018-05-05" { + description + "Add grouping of min-max-time and + included them to all stats with min/max/avg"; + reference "0.5.0"; + } + + revision "2018-01-16" { + description + "Add interval to min/max/avg stats; add percentage stat"; + reference "0.4.0"; + } + + revision "2017-08-16" { + description + "Apply fix for ieetfloat32 length parameter"; + reference "0.3.3"; + } + + revision "2017-01-13" { + description + "Add ADDRESS_FAMILY identity"; + reference "0.3.2"; + } + + revision "2016-11-14" { + description + "Correct length of ieeefloat32"; + reference "0.3.1"; + } + + revision "2016-11-11" { + description + "Additional types - ieeefloat32 and routing-password"; + reference "0.3.0"; + } + + revision "2016-05-31" { + description + "OpenConfig public release"; + reference "0.2.0"; + } + + // OpenConfig specific extensions for module metadata. + oc-ext:regexp-posix; + oc-ext:catalog-organization "openconfig"; + oc-ext:origin "openconfig"; + + typedef percentage { + type uint8 { + range "0..100"; + } + description + "Integer indicating a percentage value"; + } + + typedef std-regexp { + type string; + description + "This type definition is a placeholder for a standard + definition of a regular expression that can be utilised in + OpenConfig models. Further discussion is required to + consider the type of regular expressions that are to be + supported. An initial proposal is POSIX compatible."; + } + + typedef timeticks64 { + type uint64; + units "nanoseconds"; + description + "The timeticks64 represents the time, modulo 2^64 in + nanoseconds between two epochs. The leaf using this + type must define the epochs that tests are relative to."; + } + + typedef ieeefloat32 { + type binary { + length "4"; + } + description + "An IEEE 32-bit floating point number. The format of this number + is of the form: + 1-bit sign + 8-bit exponent + 23-bit fraction + The floating point value is calculated using: + (-1)**S * 2**(Exponent-127) * (1+Fraction)"; + } + + typedef routing-password { + type string; + description + "This type is indicative of a password that is used within + a routing protocol which can be returned in plain text to the + NMS by the local system. Such passwords are typically stored + as encrypted strings. Since the encryption used is generally + well known, it is possible to extract the original value from + the string - and hence this format is not considered secure. + Leaves specified with this type should not be modified by + the system, and should be returned to the end-user in plain + text. This type exists to differentiate passwords, which + may be sensitive, from other string leaves. It could, for + example, be used by the NMS to censor this data when + viewed by particular users."; + } + + typedef stat-interval { + type uint64; + units nanoseconds; + description + "A time interval over which a set of statistics is computed. + A common usage is to report the interval over which + avg/min/max stats are computed and reported."; + } + + grouping stat-interval-state { + description + "Reusable leaf definition for stats computation interval"; + + leaf interval { + type oc-types:stat-interval; + description + "If supported by the system, this reports the time interval + over which the min/max/average statistics are computed by + the system."; + } + } + + grouping min-max-time { + description + "Common grouping for recording the absolute time at which + the minimum and maximum values occurred in the statistics"; + + leaf min-time { + type oc-types:timeticks64; + description + "The absolute time at which the minimum value occurred. + The value is the timestamp in nanoseconds relative to + the Unix Epoch (Jan 1, 1970 00:00:00 UTC)."; + } + + leaf max-time { + type oc-types:timeticks64; + description + "The absolute time at which the maximum value occurred. + The value is the timestamp in nanoseconds relative to + the Unix Epoch (Jan 1, 1970 00:00:00 UTC)."; + } + } + + grouping avg-min-max-stats-precision1 { + description + "Common nodes for recording average, minimum, and + maximum values for a statistic. These values all have + fraction-digits set to 1. Statistics are computed + and reported based on a moving time interval (e.g., the last + 30s). If supported by the device, the time interval over which + the statistics are computed is also reported."; + + leaf avg { + type decimal64 { + fraction-digits 1; + } + description + "The arithmetic mean value of the statistic over the + time interval."; + } + + leaf min { + type decimal64 { + fraction-digits 1; + } + description + "The minimum value of the statistic over the time + interval."; + } + + leaf max { + type decimal64 { + fraction-digits 1; + } + description + "The maximum value of the statitic over the time + interval."; + } + + uses stat-interval-state; + uses min-max-time; + } + + grouping avg-min-max-instant-stats-precision1 { + description + "Common grouping for recording an instantaneous statistic value + in addition to avg-min-max stats"; + + leaf instant { + type decimal64 { + fraction-digits 1; + } + description + "The instantaneous value of the statistic."; + } + + uses avg-min-max-stats-precision1; + } + + grouping avg-min-max-instant-stats-precision2-dB { + description + "Common grouping for recording dB values with 2 decimal + precision. Values include the instantaneous, average, + minimum, and maximum statistics. Statistics are computed + and reported based on a moving time interval (e.g., the last + 30s). If supported by the device, the time interval over which + the statistics are computed, and the times at which the minimum + and maximum values occurred, are also reported."; + + leaf instant { + type decimal64 { + fraction-digits 2; + } + units dB; + description + "The instantaneous value of the statistic."; + } + + leaf avg { + type decimal64 { + fraction-digits 2; + } + units dB; + description + "The arithmetic mean value of the statistic over the + time interval."; + } + + leaf min { + type decimal64 { + fraction-digits 2; + } + units dB; + description + "The minimum value of the statistic over the time interval."; + } + + leaf max { + type decimal64 { + fraction-digits 2; + } + units dB; + description + "The maximum value of the statistic over the time + interval."; + } + + uses stat-interval-state; + uses min-max-time; + } + + grouping avg-min-max-instant-stats-precision2-dBm { + description + "Common grouping for recording dBm values with 2 decimal + precision. Values include the instantaneous, average, + minimum, and maximum statistics. Statistics are computed + and reported based on a moving time interval (e.g., the last + 30s). If supported by the device, the time interval over which + the statistics are computed, and the times at which the minimum + and maximum values occurred, are also reported."; + + leaf instant { + type decimal64 { + fraction-digits 2; + } + units dBm; + description + "The instantaneous value of the statistic."; + } + + leaf avg { + type decimal64 { + fraction-digits 2; + } + units dBm; + description + "The arithmetic mean value of the statistic over the + time interval."; + } + + leaf min { + type decimal64 { + fraction-digits 2; + } + units dBm; + description + "The minimum value of the statistic over the time + interval."; + } + + leaf max { + type decimal64 { + fraction-digits 2; + } + units dBm; + description + "The maximum value of the statistic over the time interval."; + } + + uses stat-interval-state; + uses min-max-time; + } + + grouping avg-min-max-instant-stats-precision2-mA { + description + "Common grouping for recording mA values with 2 decimal + precision. Values include the instantaneous, average, + minimum, and maximum statistics. Statistics are computed + and reported based on a moving time interval (e.g., the last + 30s). If supported by the device, the time interval over which + the statistics are computed, and the times at which the minimum + and maximum values occurred, are also reported."; + + leaf instant { + type decimal64 { + fraction-digits 2; + } + units mA; + description + "The instantaneous value of the statistic."; + } + + leaf avg { + type decimal64 { + fraction-digits 2; + } + units mA; + description + "The arithmetic mean value of the statistic over the + time interval."; + } + + leaf min { + type decimal64 { + fraction-digits 2; + } + units mA; + description + "The minimum value of the statistic over the time + interval."; + } + + leaf max { + type decimal64 { + fraction-digits 2; + } + units mA; + description + "The maximum value of the statistic over the time + interval."; + } + + uses stat-interval-state; + uses min-max-time; + } + + grouping avg-min-max-instant-stats-pct { + description + "Common grouping for percentage statistics. + Values include the instantaneous, average, + minimum, and maximum statistics. Statistics are computed + and reported based on a moving time interval (e.g., the last + 30s). If supported by the device, the time interval over which + the statistics are computed, and the times at which the minimum + and maximum values occurred, are also reported."; + + leaf instant { + type oc-types:percentage; + description + "The instantaneous percentage value."; + } + + leaf avg { + type oc-types:percentage; + description + "The arithmetic mean value of the percentage measure of the + statistic over the time interval."; + } + + leaf min { + type oc-types:percentage; + description + "The minimum value of the percentage measure of the + statistic over the time interval."; + } + + leaf max { + type oc-types:percentage; + description + "The maximum value of the percentage measure of the + statistic over the time interval."; + } + + uses stat-interval-state; + uses min-max-time; + } + + identity ADDRESS_FAMILY { + description + "A base identity for all address families"; + } + + identity IPV4 { + base ADDRESS_FAMILY; + description + "The IPv4 address family"; + } + + identity IPV6 { + base ADDRESS_FAMILY; + description + "The IPv6 address family"; + } + + identity MPLS { + base ADDRESS_FAMILY; + description + "The MPLS address family"; + } + + identity L2_ETHERNET { + base ADDRESS_FAMILY; + description + "The 802.3 Ethernet address family"; + } + +} diff --git a/src/device/service/drivers/smartnic_probes/probes-agent.yang b/src/device/service/drivers/smartnic_probes/probes-agent.yang index 6029cd204..897d2f129 100644 --- a/src/device/service/drivers/smartnic_probes/probes-agent.yang +++ b/src/device/service/drivers/smartnic_probes/probes-agent.yang @@ -25,32 +25,7 @@ augment '/oc-probes:probes/oc-probes:probe/oc-probes:tests/oc-probes:test/oc-pro uses morpheus_pipelines; } -grouping morpheus_pipelines { - list morpheus_pipelines { - key name; - uses morpheus_pipeline; - } -} - -grouping morpheus_pipeline { - leaf name { type string; } - leaf num_threads { type uint16; } - leaf pipeline_batch_size {type uint64; } - leaf model_max_batch_size {type uint64; } - leaf input_file {type string; } - leaf output_file {type string; } - leaf model_fea_length {type uint16; } - leaf model_name {type string; } - leaf iterative {type boolean; } - leaf server_url {type string; } - leaf file_type {type files; } - list stages { - key name; - uses morpheus_pipeline_stage; - } -} - -typedef files { +typedef files{ type enumeration { enum "auto"; enum "csv"; @@ -58,27 +33,14 @@ typedef files { } } -typedef stage { - type enumeration { - enum "FileSourceStage"; - enum "DeserializeStage"; - enum "AbpPcapPreprocessingStage"; - enum "MonitorStage"; - enum "TritonInferenceStage"; - enum "AddClassificationsStage"; - enum "SerializeStage"; - enum "WriteToFileStage"; - } -} - -grouping prob { +grouping prob{ leaf value { - type string; + type string; // e.g. "probs" description "probs"; } } -grouping conf { +grouping conf{ leaf mode {type string; } leaf num_threads { type uint16; } leaf pipeline_batch_size { type uint64; } @@ -90,20 +52,98 @@ grouping conf { } grouping morpheus_pipeline_stage { - leaf name { type string; } - leaf stage_type {type stage; } - list configuration {key "mode"; uses conf;} - leaf filename {type string;} - leaf iterative {type boolean;} - leaf file_type {type files;} - leaf filter_null {type boolean;} - leaf descriptions {type string;} + leaf stage_name { type string; } + choice stage_type { + case FileSourceStage { + container FileSourceStage { + list fs_configuration {key "mode"; uses conf;} + leaf filename {type string;} + leaf iterative {type boolean;} + leaf file_type {type files;} + leaf filter_null {type boolean;} + } + } + case DeserializeStage { + container DeserializeStage { + list ds_configuration {key "mode"; uses conf;} + } + } + case AbpPcapPreprocessingStage { + container AbpPcapPreprocessingStage { + list apps_configuration {key "mode"; uses conf;} + } + } + case MonitorStage { + container MonitorStage { + list ms_configuration {key "mode"; uses conf;} + leaf descriptions {type string;} + leaf unit {type string;} + } + } + case TritonInferenceStage { + container TritonInferenceStage { + list tis_configuration {key "mode"; uses conf;} + leaf model_name {type string;} + leaf server_url {type string;} + leaf force_convert_inputs {type boolean;} + } + } + case AddClassificationsStage{ + container AddClassificationsStage { + list acs_configuration {key "mode"; uses conf;} + list labels {key "value"; uses prob;} + } + } + case SerializeStage { + container SerializeStage { + list ss_configuration {key "mode"; uses conf;} + leaf kwargs {type empty;} + } + } + case WriteToFileStage{ + container WriteToFileStage { + list wfs_configuration {key "mode"; uses conf;} + leaf wfs_filename {type string;} + leaf overwrite {type boolean;} + } + } + case CustomStage { + list custom { + key "field_name"; + leaf field_name {type string;} + leaf field_value {type string;} + } + } + } +} + +grouping morpheus_pipeline { + leaf pipeline_name {type string;} + leaf num_threads {type uint16;} + leaf pipeline_batch_size {type uint64; } + leaf model_max_batch_size {type uint64; } + leaf input_file {type string;} + leaf output_file {type string;} + leaf model_fea_length {type uint16;} leaf model_name {type string;} + leaf iterative {type boolean;} leaf server_url {type string;} - leaf force_convert_inputs {type boolean;} - leaf unit {type string;} - list labels {key "value"; uses prob;} - leaf kwargs {type empty;} - leaf overwrite {type boolean;} + leaf file_type {type files;} + list stages { + key "stage_name"; + uses morpheus_pipeline_stage; + } +} + +grouping morpheus_pipelines { + list morpheus_pipeline { + key "pipeline_name"; + uses morpheus_pipeline; + } +} + +container morpheus_pipelines { + uses morpheus_pipelines; } + } \ No newline at end of file diff --git a/src/device/service/drivers/smartnic_probes/references_probes_libraries.txt b/src/device/service/drivers/smartnic_probes/references_probes_libraries.txt new file mode 100644 index 000000000..7628b7c2f --- /dev/null +++ b/src/device/service/drivers/smartnic_probes/references_probes_libraries.txt @@ -0,0 +1,6 @@ +ietf-yang-types.yang -> https://github.com/YangModels/yang/blob/main/vendor/cisco/xe/1661/ietf-yang-types.yang +openconfig-extensions.yang -> https://github.com/openconfig/public/blob/master/release/models/openconfig-extensions.yang +openconfig-inet-types.yang -> https://github.com/openconfig/public/blob/master/release/models/types/openconfig-inet-types.yang +openconfig-probes-types.yang -> https://github.com/openconfig/public/blob/master/release/models/probes/openconfig-probes-types.yang +openconfig-probes.yang -> https://github.com/openconfig/public/blob/master/release/models/probes/openconfig-probes.yang +openconfig-types.yang -> https://github.com/openconfig/public/blob/master/release/models/types/openconfig-types.yang -- GitLab From 3e1df0378bc50c1d67a1c27674f810084bca0bc8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 4 Dec 2023 18:59:10 +0000 Subject: [PATCH 010/941] Device component - gNMI OpenConfig driver: - Updated Component Handler - Updated Interface Handler - Updated InterfaceCounter Handler - Partial update of NetworkInstance Handler - Updated Tools - Added scripts to collect OpenConfig data model and build yang model bindings - Minor cosmetic changes - Updated test_gnmi.py and test_gnmi.sh script - Updated requirements.in --- src/device/requirements.in | 3 +- .../gnmi_openconfig/01-clone-yang-models.sh | 27 +++ .../gnmi_openconfig/02-build-yang-bindings.sh | 106 +++++++++ .../gnmi_openconfig/GnmiSessionHandler.py | 4 +- .../gnmi_openconfig/handlers/Component.py | 42 ++-- .../gnmi_openconfig/handlers/Interface.py | 223 +++++------------- .../handlers/InterfaceCounter.py | 84 +++---- .../handlers/NetworkInstance.py | 9 +- .../drivers/gnmi_openconfig/handlers/Tools.py | 22 +- .../gnmi_openconfig/handlers/__init__.py | 11 +- .../drivers/gnmi_openconfig/tools/Path.py | 6 +- .../drivers/gnmi_openconfig/tools/Value.py | 37 ++- src/device/tests/test_gnmi.py | 50 ++-- test_gnmi.sh | 17 ++ 14 files changed, 366 insertions(+), 275 deletions(-) create mode 100755 src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh create mode 100755 src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh create mode 100755 test_gnmi.sh diff --git a/src/device/requirements.in b/src/device/requirements.in index ece761571..d8a33455e 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -15,6 +15,7 @@ anytree==2.8.0 APScheduler==3.10.1 +bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 Jinja2==3.0.3 @@ -32,7 +33,7 @@ tabulate ipaddress macaddress yattag -pyang +pyang==2.6.* git+https://github.com/robshakir/pyangbind.git websockets==10.4 diff --git a/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh new file mode 100755 index 000000000..fe852f0e1 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/01-clone-yang-models.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig + +rm -rf ${GIT_BASE_PATH} + +OC_PUBLIC_PATH=${GIT_BASE_PATH}/public +mkdir -p ${OC_PUBLIC_PATH} +git clone https://github.com/openconfig/public.git ${OC_PUBLIC_PATH} + +#OC_HERCULES_PATH=${GIT_BASE_PATH}/hercules +#mkdir -p ${OC_HERCULES_PATH} +#git clone https://github.com/openconfig/hercules.git ${OC_HERCULES_PATH} diff --git a/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh new file mode 100755 index 000000000..ed4cf263f --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/02-build-yang-bindings.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE_PATH=~/tfs-ctrl/src/device/service/drivers/gnmi_openconfig +GIT_BASE_PATH=${BASE_PATH}/git/openconfig +OC_PUBLIC_MODELS_PATH=${GIT_BASE_PATH}/public/release/models +IETF_MODELS_PATH=${GIT_BASE_PATH}/public/third_party/ietf +#OC_HERCULES_MODELS_PATH=${GIT_BASE_PATH}/hercules/yang + +OUT_FOLDER=openconfig +OUT_PATH=${BASE_PATH}/handlers +cd ${OUT_PATH} +export PYBINDPLUGIN=`/usr/bin/env python -c 'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'` + +# -p ${OC_HERCULES_MODELS_PATH}/ +# --split-class-dir openconfig_hercules +pyang --plugindir $PYBINDPLUGIN -p ${OC_PUBLIC_MODELS_PATH}/ -f pybind --split-class-dir ${OUT_FOLDER} \ + ${IETF_MODELS_PATH}/iana-if-type.yang \ + ${IETF_MODELS_PATH}/ietf-interfaces.yang \ + ${IETF_MODELS_PATH}/ietf-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv4-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-icmpv6-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/acl/openconfig-packet-match.yang \ + ${OC_PUBLIC_MODELS_PATH}/defined-sets/openconfig-defined-sets.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-aggregate.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ethernet.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-if-ip.yang \ + ${OC_PUBLIC_MODELS_PATH}/interfaces/openconfig-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/mpls/openconfig-mpls-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/openconfig-extensions.yang \ + ${OC_PUBLIC_MODELS_PATH}/optical-transport/openconfig-transport-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-common.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-controller-card.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-cpu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-ext.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fabric.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-fan.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-healthz.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-integrated-circuit.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-linecard.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-pipeline-counters.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-port.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-psu.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-software.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-transceiver.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/platform/openconfig-platform.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-elements.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-interfaces.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-mem-mgmt.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/qos/openconfig-qos.yang \ + ${OC_PUBLIC_MODELS_PATH}/system/openconfig-alarm-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-inet-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/types/openconfig-yang-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/vlan/openconfig-vlan.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l2.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-l3.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-network-instance-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn.yang \ + ${OC_PUBLIC_MODELS_PATH}/network-instance/openconfig-evpn-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-types.yang \ + ${OC_PUBLIC_MODELS_PATH}/bgp/openconfig-bgp-errors.yang \ + + +openconfig-aft +openconfig-bgp +openconfig-igmp +openconfig-isis +openconfig-local-routing +openconfig-mpls +openconfig-ospfv2 +openconfig-pcep +openconfig-pim +openconfig-policy-forwarding +openconfig-policy-types +openconfig-routing-policy +openconfig-segment-routing + + + + +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-interfaces.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-chassis.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-linecard.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-node.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform-port.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-platform.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules-qos.yang \ +# ${OC_HERCULES_MODELS_PATH}/openconfig-hercules.yang \ diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 04dae4f5f..6f80ee82f 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -132,8 +132,8 @@ class GnmiSessionHandler: #resource_key_tuple[2] = True results.extend(parse(str_path, value)) except Exception as e: # pylint: disable=broad-except - MSG = 'Exception processing notification {:s}' - self._logger.exception(MSG.format(grpc_message_to_json_string(notification))) + MSG = 'Exception processing update {:s}' + self._logger.exception(MSG.format(grpc_message_to_json_string(update))) results.append((str_path, e)) # if validation fails, store the exception #_results = sorted(results.items(), key=lambda x: x[1][0]) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 0b3c1f970..cddf40d56 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,45 +12,45 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import logging #, json +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +#pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' - def get_path(self) -> str: return '/components/component' + def get_path(self) -> str: return '/openconfig-platform:components' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_component_list : List[Dict] = json_data.get('component', []) - response = [] - for json_component in json_component_list: - #LOGGER.info('json_component = {:s}'.format(json.dumps(json_component))) - endpoint = {} + oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') + #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) - component_type = json_component.get('state', {}).get('type') - if component_type is None: continue - component_type = component_type.replace('oc-platform-types:', '') - component_type = component_type.replace('openconfig-platform-types:', '') - if component_type not in {'PORT'}: continue - endpoint['type'] = '-' + entries = [] + for component_key, oc_component in oc_components.component.items(): + #LOGGER.info('component_key={:s} oc_component={:s}'.format( + # component_key, pybindJSON.dumps(oc_component, mode='ietf') + #)) - #LOGGER.info('PORT json_component = {:s}'.format(json.dumps(json_component))) + component_name = oc_component.config.name - component_name = json_component.get('name') - if component_name is None: continue + component_type = oc_component.state.type + component_type = component_type.split(':')[-1] + if component_type not in {'PORT'}: continue # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. interface_name = component_name.lower().replace('-port', '') - endpoint['uuid'] = interface_name + endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { KpiSampleType.KPISAMPLETYPE_BYTES_RECEIVED : PATH_IF_CTR.format(interface_name, 'in-octets' ), KpiSampleType.KPISAMPLETYPE_BYTES_TRANSMITTED : PATH_IF_CTR.format(interface_name, 'out-octets'), @@ -59,5 +59,7 @@ class ComponentHandler(_Handler): } if len(endpoint) == 0: continue - response.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - return response + + entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 20f79b3c2..77310d51d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -13,15 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' - def get_path(self) -> str: return '/interfaces/interface' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: if_name = str (resource_value['name' ]) # ethernet-1/1 @@ -63,186 +64,88 @@ class InterfaceHandler(_Handler): def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - #LOGGER.info('json_interface = {:s}'.format(json.dumps(json_interface))) + entries = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + #)) interface = {} + interface['name'] = oc_interface.config.name - interface_name = json_interface.get('name') - if interface_name is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue - interface['name'] = interface_name - - CONFIG_FIELDS = ('config', 'openconfig-interface:config', 'oci:config') - json_config : Dict = dict_get_first(json_interface, CONFIG_FIELDS, default={}) - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state : Dict = dict_get_first(json_interface, STATE_FIELDS, default={}) - - interface_type = json_config.get('type') - if interface_type is None: interface_type = json_state.get('type') - if interface_type is None: - LOGGER.info('DISCARDED json_interface = {:s}'.format(json.dumps(json_interface))) - continue + interface_type = oc_interface.config.type interface_type = interface_type.replace('ianaift:', '') interface_type = interface_type.replace('iana-if-type:', '') interface['type'] = interface_type - interface_mtu = json_config.get('mtu') - if interface_mtu is None: interface_mtu = json_state.get('mtu') - if interface_mtu is not None: interface['mtu'] = int(interface_mtu) - - interface_enabled = json_config.get('enabled') - if interface_enabled is None: interface_enabled = json_state.get('enabled') - interface['enabled'] = False if interface_enabled is None else bool(interface_enabled) - - interface_management = json_config.get('management') - if interface_management is None: interface_management = json_state.get('management') - interface['management'] = False if interface_management is None else bool(interface_management) + interface['mtu' ] = oc_interface.config.mtu + interface['enabled' ] = oc_interface.config.enabled + interface['description' ] = oc_interface.config.description + interface['admin-status'] = oc_interface.state.admin_status + interface['oper-status' ] = oc_interface.state.oper_status + interface['management' ] = oc_interface.state.management - interface_descr = json_interface.get('config', {}).get('description') - if interface_descr is not None: interface['description'] = interface_descr + entry_interface_key = '/interface[{:s}]'.format(interface['name']) + entries.append((entry_interface_key, interface)) - json_subinterfaces = json_interface.get('subinterfaces', {}) - json_subinterface_list : List[Dict] = json_subinterfaces.get('subinterface', []) - - for json_subinterface in json_subinterface_list: - #LOGGER.info('json_subinterface = {:s}'.format(json.dumps(json_subinterface))) + for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): + #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( + # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') + #)) subinterface = {} + subinterface['index' ] = oc_subinterface.state.index + subinterface['name' ] = oc_subinterface.state.name + subinterface['enabled'] = oc_subinterface.state.enabled - subinterface_index = json_subinterface.get('state', {}).get('index') - if subinterface_index is None: continue - subinterface['index'] = int(subinterface_index) - - subinterface_name = json_subinterface.get('state', {}).get('name') - if subinterface_name is None: continue - subinterface['name'] = subinterface_name - - subinterface_enabled = json_subinterface.get('state', {}).get('enabled', False) - subinterface['enabled'] = bool(subinterface_enabled) - - VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - - # TODO: implement support for multiple IP addresses per subinterface - - IPV4_FIELDS = ('ipv4', 'openconfig-if-ip:ipv4', 'ociip:ipv4') - json_ipv4 = dict_get_first(json_subinterface, IPV4_FIELDS, default={}) - - IPV4_ADDRESSES_FIELDS = ('addresses', 'openconfig-if-ip:addresses', 'ociip:addresses') - json_ipv4_addresses = dict_get_first(json_ipv4, IPV4_ADDRESSES_FIELDS, default={}) - - IPV4_ADDRESS_FIELDS = ('address', 'openconfig-if-ip:address', 'ociip:address') - json_ipv4_address_list : List[Dict] = dict_get_first(json_ipv4_addresses, IPV4_ADDRESS_FIELDS, default=[]) - - #ipv4_addresses = [] - for json_ipv4_address in json_ipv4_address_list: - #LOGGER.info('json_ipv4_address = {:s}'.format(json.dumps(json_ipv4_address))) + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) + entries.append((entry_subinterface_key, subinterface)) - STATE_FIELDS = ('state', 'openconfig-if-ip:state', 'ociip:state') - json_ipv4_address_state = dict_get_first(json_ipv4_address, STATE_FIELDS, default={}) + #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') + #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - #ipv4_address = {} + #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') + #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - #ORIGIN_FIELDS = ('origin', 'openconfig-if-ip:origin', 'ociip:origin') - #ipv4_address_origin = dict_get_first(json_ipv4_address_state, ORIGIN_FIELDS, default={}) - #if ipv4_address_origin is not None: ipv4_address['origin'] = ipv4_address_origin + #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') + #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - IP_FIELDS = ('ip', 'openconfig-if-ip:ip', 'ociip:ip') - ipv4_address_ip = dict_get_first(json_ipv4_address_state, IP_FIELDS) - #if ipv4_address_ip is not None: ipv4_address['address_ip'] = ipv4_address_ip - if ipv4_address_ip is not None: subinterface['address_ip'] = ipv4_address_ip + #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') + #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - PREFIX_FIELDS = ('prefix-length', 'openconfig-if-ip:prefix-length', 'ociip:prefix-length') - ipv4_address_prefix = dict_get_first(json_ipv4_address_state, PREFIX_FIELDS) - #if ipv4_address_prefix is not None: ipv4_address['address_prefix'] = int(ipv4_address_prefix) - if ipv4_address_prefix is not None: subinterface['address_prefix'] = int(ipv4_address_prefix) + #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') + #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) + #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - #if len(ipv4_address) == 0: continue - #ipv4_addresses.append(ipv4_address) + for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): + #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #subinterface['ipv4_addresses'] = ipv4_addresses - - if len(subinterface) == 0: continue - resource_key = '/interface[{:s}]/subinterface[{:s}]'.format(interface['name'], str(subinterface['index'])) - response.append((resource_key, subinterface)) - - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) - - return response - - def parse_counters(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse_counters] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) - - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse_counters] json_interface = {:s}'.format(json.dumps(json_interface))) - - interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) + address_ipv4 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) + entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) + entries.append((entry_address_ipv4_key, address_ipv4)) - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) + for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): + #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( + # address_key, pybindJSON.dumps(oc_address, mode='ietf') + #)) - #LOGGER.info('[parse_counters] interface = {:s}'.format(str(interface))) + address_ipv6 = { + 'ip' : oc_address.state.ip, + 'origin': oc_address.state.origin, + 'prefix': oc_address.state.prefix_length, + } - if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) + entries.append((entry_address_ipv6_key, address_ipv6)) - return response + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index a45dc9e7f..1c2cfc17a 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -14,67 +14,51 @@ import json, logging from typing import Any, Dict, List, Tuple +import pyangbind.lib.pybindJSON as pybindJSON +from . import openconfig from ._Handler import _Handler -from .Tools import dict_get_first LOGGER = logging.getLogger(__name__) +#pylint: disable=abstract-method class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' - def get_path(self) -> str: return '/interfaces/interface/state/counters' + def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('[parse] json_data = {:s}'.format(json.dumps(json_data))) - json_interface_list : List[Dict] = json_data.get('interface', []) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') + LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - for json_interface in json_interface_list: - LOGGER.info('[parse] json_interface = {:s}'.format(json.dumps(json_interface))) + counters = [] + for interface_key, oc_interface in oc_interfaces.interface.items(): + LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( + interface_key, pybindJSON.dumps(oc_interface, mode='ietf') + )) interface = {} - - NAME_FIELDS = ('name', 'openconfig-interface:name', 'oci:name') - interface_name = dict_get_first(json_interface, NAME_FIELDS) - if interface_name is None: continue - interface['name'] = interface_name - - STATE_FIELDS = ('state', 'openconfig-interface:state', 'oci:state') - json_state = dict_get_first(json_interface, STATE_FIELDS, default={}) - - COUNTERS_FIELDS = ('counters', 'openconfig-interface:counters', 'oci:counters') - json_counters = dict_get_first(json_state, COUNTERS_FIELDS, default={}) - - IN_PKTS_FIELDS = ('in-pkts', 'openconfig-interface:in-pkts', 'oci:in-pkts') - interface_in_pkts = dict_get_first(json_counters, IN_PKTS_FIELDS) - if interface_in_pkts is not None: interface['in-pkts'] = int(interface_in_pkts) - - IN_OCTETS_FIELDS = ('in-octets', 'openconfig-interface:in-octets', 'oci:in-octets') - interface_in_octets = dict_get_first(json_counters, IN_OCTETS_FIELDS) - if interface_in_octets is not None: interface['in-octets'] = int(interface_in_octets) - - IN_ERRORS_FIELDS = ('in-errors', 'openconfig-interface:in-errors', 'oci:in-errors') - interface_in_errors = dict_get_first(json_counters, IN_ERRORS_FIELDS) - if interface_in_errors is not None: interface['in-errors'] = int(interface_in_errors) - - OUT_OCTETS_FIELDS = ('out-octets', 'openconfig-interface:out-octets', 'oci:out-octets') - interface_out_octets = dict_get_first(json_counters, OUT_OCTETS_FIELDS) - if interface_out_octets is not None: interface['out-octets'] = int(interface_out_octets) - - OUT_PKTS_FIELDS = ('out-pkts', 'openconfig-interface:out-pkts', 'oci:out-pkts') - interface_out_pkts = dict_get_first(json_counters, OUT_PKTS_FIELDS) - if interface_out_pkts is not None: interface['out-pkts'] = int(interface_out_pkts) - - OUT_ERRORS_FIELDS = ('out-errors', 'openconfig-interface:out-errors', 'oci:out-errors') - interface_out_errors = dict_get_first(json_counters, OUT_ERRORS_FIELDS) - if interface_out_errors is not None: interface['out-errors'] = int(interface_out_errors) - - OUT_DISCARDS_FIELDS = ('out-discards', 'openconfig-interface:out-discards', 'oci:out-discards') - interface_out_discards = dict_get_first(json_counters, OUT_DISCARDS_FIELDS) - if interface_out_discards is not None: interface['out-discards'] = int(interface_out_discards) - - #LOGGER.info('[parse] interface = {:s}'.format(str(interface))) + interface['name'] = oc_interface.name + + interface_counters = oc_interface.state.counters + interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts + interface['in-discards' ] = interface_counters.in_discards + interface['in-errors' ] = interface_counters.in_errors + interface['in-fcs-errors' ] = interface_counters.in_fcs_errors + interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts + interface['in-octets' ] = interface_counters.in_octets + interface['in-pkts' ] = interface_counters.in_pkts + interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts + interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts + interface['out-discards' ] = interface_counters.out_discards + interface['out-errors' ] = interface_counters.out_errors + interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts + interface['out-octets' ] = interface_counters.out_octets + interface['out-pkts' ] = interface_counters.out_pkts + interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + + LOGGER.info('interface = {:s}'.format(str(interface))) if len(interface) == 0: continue - response.append(('/interface[{:s}]'.format(interface['name']), interface)) + counters.append(('/interface[{:s}]'.format(interface['name']), interface)) - return response + return counters diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index aed821a06..c29ed263a 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -13,14 +13,16 @@ # limitations under the License. import json, logging +import pyangbind.lib.pybindJSON as pybindJSON from typing import Any, Dict, List, Tuple +from . import openconfig from ._Handler import _Handler LOGGER = logging.getLogger(__name__) class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' - def get_path(self) -> str: return '/network-instances/network-instance' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances' def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: ni_name = str(resource_value['name']) # test-svc @@ -58,5 +60,10 @@ class NetworkInstanceHandler(_Handler): } def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') + #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) response = [] return response + +openconfig-network-instance:network-instance \ No newline at end of file diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 30343ac28..8cf704e29 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -23,8 +23,20 @@ def get_schema(resource_key : str): resource_key = RE_REMOVE_NAMESPACES.sub('/', resource_key) return resource_key -def dict_get_first(d : Dict, field_names : Iterable[str], default=None) -> Any: - for field_name in field_names: - if field_name not in d: continue - return d[field_name] +def container_get_first( + container : Dict[str, Any], key_name : str, namespace : Optional[str]=None, namespaces : Iterable[str]=tuple(), + default : Optional[Any] = None +) -> Any: + value = container.get(key_name) + if value is not None: return value + + if namespace is not None: + if len(namespaces) > 0: + raise Exception('At maximum, one of namespace or namespaces can be specified') + namespaces = (namespace,) + + for namespace in namespaces: + namespace_key_name = '{:s}:{:s}'.format(namespace, key_name) + if namespace_key_name in container: return container[namespace_key_name] + return default diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 39cd7c66a..6d54ef28d 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -46,9 +46,10 @@ RESOURCE_KEY_MAPPER = { } PATH_MAPPER = { - '/components' : comph.get_path(), - '/interfaces' : ifaceh.get_path(), - '/network-instances' : nih.get_path(), + '/components' : comph.get_path(), + '/components/component' : comph.get_path(), + '/interfaces' : ifaceh.get_path(), + '/network-instances' : nih.get_path(), } RESOURCE_KEY_TO_HANDLER = { @@ -88,9 +89,9 @@ def get_handler( path_schema = PATH_MAPPER.get(path_schema, path_schema) handler = PATH_TO_HANDLER.get(path_schema) if handler is None and raise_if_not_found: - MSG = 'Handler not found: resource_key={:s} resource_key_schema={:s}' + MSG = 'Handler not found: path={:s} path_schema={:s}' # pylint: disable=broad-exception-raised - raise Exception(MSG.format(str(resource_key), str(resource_key_schema))) + raise Exception(MSG.format(str(path), str(path_schema))) return handler def get_path(resource_key : str) -> str: diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Path.py b/src/device/service/drivers/gnmi_openconfig/tools/Path.py index 40ab28dc6..2d6dc1e74 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Path.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Path.py @@ -19,8 +19,8 @@ from ..gnmi.gnmi_pb2 import Path, PathElem RE_PATH_SPLIT = re.compile(r'/(?=(?:[^\[\]]|\[[^\[\]]+\])*$)') RE_PATH_KEYS = re.compile(r'\[(.*?)\]') -def path_from_string(path='/'): - if not path: return Path(elem=[]) +def path_from_string(path='/'): #, origin='openconfig' + if not path: return Path(elem=[]) #, origin=origin if path[0] == '/': if path[-1] == '/': @@ -40,7 +40,7 @@ def path_from_string(path='/'): dict_keys = dict(x.split('=', 1) for x in elem_keys) path.append(PathElem(name=elem_name, key=dict_keys)) - return Path(elem=path) + return Path(elem=path) #, origin=origin def path_to_string(path : Path) -> str: path_parts = list() diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 4797930a1..9933cb858 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -13,9 +13,36 @@ # limitations under the License. import base64, json -from typing import Any +from typing import Any, Dict, List, Union from ..gnmi.gnmi_pb2 import TypedValue +REMOVE_NAMESPACES = ( + 'arista-intf-augments', + 'arista-netinst-augments', + 'openconfig-hercules-platform', +) + +def remove_fields(key : str) -> bool: + parts = key.split(':') + if len(parts) == 1: return False + namespace = parts[0].lower() + return namespace in REMOVE_NAMESPACES + +def recursive_remove_keys(container : Union[Dict, List, Any]) -> None: + if isinstance(container, dict): + remove_keys = [ + key + for key in container.keys() + if remove_fields(key) + ] + for key in remove_keys: + container.pop(key, None) + for value in container.values(): + recursive_remove_keys(value) + elif isinstance(container, list): + for value in container: + recursive_remove_keys(value) + def decode_value(value : TypedValue) -> Any: encoding = value.WhichOneof('value') if encoding == 'json_val': @@ -31,9 +58,13 @@ def decode_value(value : TypedValue) -> Any: raise NotImplementedError() #return value elif encoding == 'json_ietf_val': - value : str = value.json_ietf_val + str_value : str = value.json_ietf_val.decode('UTF-8') try: - return json.loads(value) + # Cleanup and normalize the records according to OpenConfig + str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + json_value = json.loads(str_value) + recursive_remove_keys(json_value) + return json_value except json.decoder.JSONDecodeError: # Assume is Base64-encoded b_b64_value = value.encode('UTF-8') diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py index 50c915582..684b9f4c3 100644 --- a/src/device/tests/test_gnmi.py +++ b/src/device/tests/test_gnmi.py @@ -16,9 +16,9 @@ import logging, os, sys, time from typing import Dict, Tuple os.environ['DEVICE_EMULATED_ONLY'] = 'YES' from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -#from device.service.driver_api._Driver import ( -# RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -#) +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -58,21 +58,21 @@ def main(): driver_settings = { 'protocol': 'gnmi', 'username': 'admin', - 'password': 'NokiaSrl1!', - 'use_tls' : True, + 'password': 'admin', + 'use_tls' : False, } - driver = GnmiOpenConfigDriver('172.100.100.102', 57400, **driver_settings) + driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) driver.Connect() #resources_to_get = [] #resources_to_get = [RESOURCE_ENDPOINTS] #resources_to_get = [RESOURCE_INTERFACES] - #resources_to_get = [RESOURCE_NETWORK_INSTANCES] + resources_to_get = [RESOURCE_NETWORK_INSTANCES] #resources_to_get = [RESOURCE_ROUTING_POLICIES] #resources_to_get = [RESOURCE_SERVICES] - #LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - #results_getconfig = driver.GetConfig(resources_to_get) - #LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) #resources_to_set = [ # network_instance('test-svc', 'L3VRF'), @@ -90,21 +90,21 @@ def main(): #results_setconfig = driver.SetConfig(resources_to_set) #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - resources_to_delete = [ - #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - - #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - - interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - - network_instance('20f66fb5', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + #resources_to_delete = [ + # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), + # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), + # + # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), + # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), + # + # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), + # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), + # + # #network_instance('20f66fb5', 'L3VRF'), + #] + #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + #results_deleteconfig = driver.DeleteConfig(resources_to_delete) + #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) time.sleep(1) diff --git a/test_gnmi.sh b/test_gnmi.sh new file mode 100755 index 000000000..d1fe36969 --- /dev/null +++ b/test_gnmi.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export PYTHONPATH=./src +python -m device.tests.test_gnmi -- GitLab From a345795694cb86ccbdace416441b32cffb42afe8 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 4 Dec 2023 19:02:09 +0000 Subject: [PATCH 011/941] DataPlane in a box: - Added management scripts - Added ContainerLab scenario descriptor - Added README.md - Added example TFS descriptors - Added links.txt --- dataplane-in-a-box/.gitignore | 2 + dataplane-in-a-box/README.md | 121 ++++++++++++++++ dataplane-in-a-box/arista.clab.yml | 54 ++++++++ dataplane-in-a-box/clab-deploy.sh | 17 +++ dataplane-in-a-box/clab-destroy.sh | 18 +++ dataplane-in-a-box/clab-inspect.sh | 17 +++ dataplane-in-a-box/clab-load-image.sh | 19 +++ dataplane-in-a-box/clab-pull-images.sh | 18 +++ dataplane-in-a-box/dc-2-dc-l3-service.json | 37 +++++ dataplane-in-a-box/deploy_specs.sh | 154 +++++++++++++++++++++ dataplane-in-a-box/links.json | 136 ++++++++++++++++++ dataplane-in-a-box/links.txt | 8 ++ dataplane-in-a-box/topology.json | 91 ++++++++++++ 13 files changed, 692 insertions(+) create mode 100644 dataplane-in-a-box/.gitignore create mode 100644 dataplane-in-a-box/README.md create mode 100644 dataplane-in-a-box/arista.clab.yml create mode 100755 dataplane-in-a-box/clab-deploy.sh create mode 100755 dataplane-in-a-box/clab-destroy.sh create mode 100755 dataplane-in-a-box/clab-inspect.sh create mode 100755 dataplane-in-a-box/clab-load-image.sh create mode 100755 dataplane-in-a-box/clab-pull-images.sh create mode 100644 dataplane-in-a-box/dc-2-dc-l3-service.json create mode 100755 dataplane-in-a-box/deploy_specs.sh create mode 100644 dataplane-in-a-box/links.json create mode 100644 dataplane-in-a-box/links.txt create mode 100644 dataplane-in-a-box/topology.json diff --git a/dataplane-in-a-box/.gitignore b/dataplane-in-a-box/.gitignore new file mode 100644 index 000000000..5de716bdd --- /dev/null +++ b/dataplane-in-a-box/.gitignore @@ -0,0 +1,2 @@ +clab-arista/ +.arista.clab.yml.bak diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md new file mode 100644 index 000000000..45e5dc5e0 --- /dev/null +++ b/dataplane-in-a-box/README.md @@ -0,0 +1,121 @@ +# DataPlane-in-a-Box - Control an Emulated DataPlane through TeraFlowSDN + +## Emulated DataPlane Deployment +- ContainerLab +- Scenario +- Descriptor + +## TeraFlowSDN Deployment +```bash +cd ~/tfs-ctrl +source dataplane-in-a-box/deploy_specs.sh +./deploy/all.sh +``` + +# ContainerLab - Arista cEOS - Commands + +## Download and install ContainerLab +```bash +sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 +``` + +## Deploy scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab deploy --topo arista.clab.yml +``` + +## Inspect scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab inspect --topo arista.clab.yml +``` + +## Destroy scenario +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +sudo containerlab destroy --topo arista.clab.yml +sudo rm -rf clab-arista/ .arista.clab.yml.bak +``` + +## Access cEOS Bash +```bash +docker exec -it clab-arista-wan1 bash +``` + +## Access cEOS CLI +```bash +docker exec -it clab-arista-wan1 Cli +``` + +## Configure ContainerLab clients +```bash +docker exec -it clab-arista-client1 bash + ip address add 192.168.1.10/24 dev eth1 + ip route add 192.168.2.0/24 via 192.168.1.1 + ip route add 192.168.3.0/24 via 192.168.1.1 + ping 192.168.2.10 + ping 192.168.3.10 + +docker exec -it clab-arista-client2 bash + ip address add 192.168.2.10/24 dev eth1 + ip route add 192.168.1.0/24 via 192.168.2.1 + ip route add 192.168.3.0/24 via 192.168.2.1 + ping 192.168.1.10 + ping 192.168.3.10 + +docker exec -it clab-arista-client3 bash + ip address add 192.168.3.10/24 dev eth1 + ip route add 192.168.2.0/24 via 192.168.3.1 + ip route add 192.168.3.0/24 via 192.168.3.1 + ping 192.168.2.10 + ping 192.168.3.10 +``` + +## Install gNMIc +```bash +sudo bash -c "$(curl -sL https://get-gnmic.kmrd.dev)" +``` + +## gNMI Capabilities request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure capabilities +``` + +## gNMI Get request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path / > wan1.json +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path /interfaces/interface > wan1-ifaces.json +``` + +## gNMI Set request +```bash +#gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --update-path /system/config/hostname --update-value srl11 +#gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path /system/config/hostname +``` + +## Subscribe request +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf subscribe --path /interfaces/interface[name=Management0]/state/ + +# In another terminal, you can generate traffic opening SSH connection +ssh admin@clab-arista-wan1 +``` + +# Check configurations done: +```bash +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path '/network-instances' > wan1-nis.json +gnmic --address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf get --path '/interfaces' > wan1-ifs.json +``` + +# Delete elements: +```bash +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/network-instances/network-instance[name=b19229e8]' +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/interfaces/interface[name=ethernet-1/1]/subinterfaces/subinterface[index=0]' +--address clab-arista-wan1 --port 6030 --username admin --password admin --insecure --encoding json_ietf set --delete '/interfaces/interface[name=ethernet-1/2]/subinterfaces/subinterface[index=0]' +``` + +# Run gNMI Driver in standalone mode (advanced) +```bash +PYTHONPATH=./src python -m src.device.tests.test_gnmi +``` diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml new file mode 100644 index 000000000..9a8bff73f --- /dev/null +++ b/dataplane-in-a-box/arista.clab.yml @@ -0,0 +1,54 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TFS - Arista devices + Linux clients + +name: arista + +mgmt: + network: mgmt-net + ipv4-subnet: 172.20.20.0/24 + +topology: + kinds: + arista_ceos: + kind: arista_ceos + image: ceos:4.30.4M + linux: + kind: linux + image: ghcr.io/hellt/network-multitool:latest + + nodes: + wan1: + kind: arista_ceos + mgmt-ipv4: 172.20.20.101 + ports: [6001:6030] + wan2: + kind: arista_ceos + mgmt-ipv4: 172.20.20.102 + ports: [6002:6030] + + client1: + kind: linux + mgmt-ipv4: 172.20.20.201 + ports: [2201:22] + client2: + kind: linux + mgmt-ipv4: 172.20.20.202 + ports: [2202:22] + + links: + - endpoints: ["wan1:eth1", "wan2:eth1"] + - endpoints: ["client1:eth1", "wan1:eth10"] + - endpoints: ["client2:eth1", "wan2:eth10"] diff --git a/dataplane-in-a-box/clab-deploy.sh b/dataplane-in-a-box/clab-deploy.sh new file mode 100755 index 000000000..2b8e49a07 --- /dev/null +++ b/dataplane-in-a-box/clab-deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab deploy --topo arista.clab.yml diff --git a/dataplane-in-a-box/clab-destroy.sh b/dataplane-in-a-box/clab-destroy.sh new file mode 100755 index 000000000..4030239dc --- /dev/null +++ b/dataplane-in-a-box/clab-destroy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab destroy --topo arista.clab.yml +sudo rm -rf clab-arista/ .arista.clab.yml.bak diff --git a/dataplane-in-a-box/clab-inspect.sh b/dataplane-in-a-box/clab-inspect.sh new file mode 100755 index 000000000..02024ec47 --- /dev/null +++ b/dataplane-in-a-box/clab-inspect.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +sudo containerlab inspect --topo arista.clab.yml diff --git a/dataplane-in-a-box/clab-load-image.sh b/dataplane-in-a-box/clab-load-image.sh new file mode 100755 index 000000000..87e666422 --- /dev/null +++ b/dataplane-in-a-box/clab-load-image.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download image from Arista account > software downloads + +cd /home/tfs/tfs-ctrl/dataplane-in-a-box +docker import cEOS64-lab-4.30.4M.tar ceos:4.30.4M diff --git a/dataplane-in-a-box/clab-pull-images.sh b/dataplane-in-a-box/clab-pull-images.sh new file mode 100755 index 000000000..8f2805c6b --- /dev/null +++ b/dataplane-in-a-box/clab-pull-images.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker pull ghcr.io/hellt/network-multitool:latest +#docker pull ghcr.io/nokia/srlinux:23.7.2 +#docker pull netreplica/docker-sonic-vs:20220111 diff --git a/dataplane-in-a-box/dc-2-dc-l3-service.json b/dataplane-in-a-box/dc-2-dc-l3-service.json new file mode 100644 index 000000000..cb9ef972e --- /dev/null +++ b/dataplane-in-a-box/dc-2-dc-l3-service.json @@ -0,0 +1,37 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc-2-dc-l3-svc"} + }, + "service_type": 1, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id":{"device_uuid":{"uuid":"DC1"}},"endpoint_uuid":{"uuid":"int"}}, + {"device_id":{"device_uuid":{"uuid":"DC2"}},"endpoint_uuid":{"uuid":"int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "/device[SRL1]/settings", "resource_value": { + "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/1]/settings", "resource_value": { + "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/2]/settings", "resource_value": { + "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 + }}}, + + {"action": 1, "custom": {"resource_key": "/device[SRL2]/settings", "resource_value": { + "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/1]/settings", "resource_value": { + "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 + }}}, + {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/2]/settings", "resource_value": { + "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 + }}} + ]} + } + ] +} diff --git a/dataplane-in-a-box/deploy_specs.sh b/dataplane-in-a-box/deploy_specs.sh new file mode 100755 index 000000000..1a978e3a9 --- /dev/null +++ b/dataplane-in-a-box/deploy_specs.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ----- TeraFlowSDN ------------------------------------------------------------ + +# Set the URL of the internal MicroK8s Docker registry where the images will be uploaded to. +export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" + +# Set the list of components, separated by spaces, you want to build images for, and deploy. +#export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp service slice nbi webui" + +# Uncomment to activate Monitoring +export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" + +# Uncomment to activate ZTP +#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp" + +# Uncomment to activate Policy Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} policy" + +# Uncomment to activate Optical CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} dbscanserving opticalattackmitigator opticalattackdetector opticalattackmanager" + +# Uncomment to activate L3 CyberSecurity +#export TFS_COMPONENTS="${TFS_COMPONENTS} l3_attackmitigator l3_centralizedattackdetector" + +# Uncomment to activate TE +#export TFS_COMPONENTS="${TFS_COMPONENTS} te" + +# Uncomment to activate Forecaster +#export TFS_COMPONENTS="${TFS_COMPONENTS} forecaster" + +# Set the tag you want to use for your images. +export TFS_IMAGE_TAG="dev" + +# Set the name of the Kubernetes namespace to deploy TFS to. +export TFS_K8S_NAMESPACE="tfs" + +# Set additional manifest files to be applied after the deployment +export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml" + +# Uncomment to monitor performance of components +export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/servicemonitors.yaml" + +# Uncomment when deploying Optical CyberSecurity +#export TFS_EXTRA_MANIFESTS="${TFS_EXTRA_MANIFESTS} manifests/cachingservice.yaml" + +# Set the new Grafana admin password +export TFS_GRAFANA_PASSWORD="admin123+" + +# Disable skip-build flag to rebuild the Docker images. +export TFS_SKIP_BUILD="" + + +# ----- CockroachDB ------------------------------------------------------------ + +# Set the namespace where CockroackDB will be deployed. +export CRDB_NAMESPACE="crdb" + +# Set the external port CockroackDB Postgre SQL interface will be exposed to. +export CRDB_EXT_PORT_SQL="26257" + +# Set the external port CockroackDB HTTP Mgmt GUI interface will be exposed to. +export CRDB_EXT_PORT_HTTP="8081" + +# Set the database username to be used by Context. +export CRDB_USERNAME="tfs" + +# Set the database user's password to be used by Context. +export CRDB_PASSWORD="tfs123" + +# Set the database name to be used by Context. +export CRDB_DATABASE="tfs" + +# Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/crdb.sh for additional details +export CRDB_DEPLOY_MODE="single" + +# Disable flag for dropping database, if it exists. +export CRDB_DROP_DATABASE_IF_EXISTS="YES" + +# Disable flag for re-deploying CockroachDB from scratch. +export CRDB_REDEPLOY="" + + +# ----- NATS ------------------------------------------------------------------- + +# Set the namespace where NATS will be deployed. +export NATS_NAMESPACE="nats" + +# Set the external port NATS Client interface will be exposed to. +export NATS_EXT_PORT_CLIENT="4222" + +# Set the external port NATS HTTP Mgmt GUI interface will be exposed to. +export NATS_EXT_PORT_HTTP="8222" + +# Disable flag for re-deploying NATS from scratch. +export NATS_REDEPLOY="" + + +# ----- QuestDB ---------------------------------------------------------------- + +# Set the namespace where QuestDB will be deployed. +export QDB_NAMESPACE="qdb" + +# Set the external port QuestDB Postgre SQL interface will be exposed to. +export QDB_EXT_PORT_SQL="8812" + +# Set the external port QuestDB Influx Line Protocol interface will be exposed to. +export QDB_EXT_PORT_ILP="9009" + +# Set the external port QuestDB HTTP Mgmt GUI interface will be exposed to. +export QDB_EXT_PORT_HTTP="9000" + +# Set the database username to be used for QuestDB. +export QDB_USERNAME="admin" + +# Set the database user's password to be used for QuestDB. +export QDB_PASSWORD="quest" + +# Set the table name to be used by Monitoring for KPIs. +export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis" + +# Set the table name to be used by Slice for plotting groups. +export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups" + +# Disable flag for dropping tables if they exist. +export QDB_DROP_TABLES_IF_EXIST="YES" + +# Disable flag for re-deploying QuestDB from scratch. +export QDB_REDEPLOY="" + + +# ----- K8s Observability ------------------------------------------------------ + +# Set the external port Prometheus Mgmt HTTP GUI interface will be exposed to. +export PROM_EXT_PORT_HTTP="9090" + +# Set the external port Grafana HTTP Dashboards will be exposed to. +export GRAF_EXT_PORT_HTTP="3000" diff --git a/dataplane-in-a-box/links.json b/dataplane-in-a-box/links.json new file mode 100644 index 000000000..832a24fdd --- /dev/null +++ b/dataplane-in-a-box/links.json @@ -0,0 +1,136 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": true + }}} + ]} + } + ], + "links": [ + { + "link_id": {"link_uuid": {"uuid": "DC1/eth1==WAN1/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/2==DC1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/1==WAN2/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/1==WAN1/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "DC2/eth1==WAN2/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/2==DC2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + } + ] +} diff --git a/dataplane-in-a-box/links.txt b/dataplane-in-a-box/links.txt new file mode 100644 index 000000000..a61ad5398 --- /dev/null +++ b/dataplane-in-a-box/links.txt @@ -0,0 +1,8 @@ +https://containerlab.dev/manual/multi-node/#exposing-services +https://containerlab.dev/manual/multi-node/#bridging +https://containerlab.dev/manual/kinds/bridge/ +https://containerlab.dev/lab-examples/ext-bridge/ + +https://containerlab.dev/manual/kinds/ceos/ +https://containerlab.dev/lab-examples/srl-ceos/#__tabbed_2_2 +https://github.com/srl-labs/containerlab/blob/main/lab-examples/srlceos01/srlceos01.clab.yml diff --git a/dataplane-in-a-box/topology.json b/dataplane-in-a-box/topology.json new file mode 100644 index 000000000..42752235d --- /dev/null +++ b/dataplane-in-a-box/topology.json @@ -0,0 +1,91 @@ +{ + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ + {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + ]}}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + }, + { + "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "use_tls": false + }}} + ]} + } + ], + "links": [] +} -- GitLab From 48c2413c0b6bd58a4a1bc571415175aed133120f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 5 Jan 2024 14:25:34 +0000 Subject: [PATCH 012/941] Device component - gNMI/OpenConfig Driver: WORK IN PROGRESS - Added unitary tests and scripts - Enhanced reporting of capabilities - Migrated Component and Interface code to libyang - Migrating NetworkInstance code to libyang - Disabled unneeded log messages - Temporarily disabled telemetry - Added LibYang-based YANG handler - Added helper methods --- ...un_tests_locally-device-gnmi-openconfig.sh | 25 + .../gnmi_openconfig/GnmiSessionHandler.py | 65 +- .../gnmi_openconfig/handlers/Component.py | 39 +- .../gnmi_openconfig/handlers/Interface.py | 276 ++++---- .../handlers/NetworkInstance.py | 145 ++++- .../drivers/gnmi_openconfig/handlers/Tools.py | 21 +- .../gnmi_openconfig/handlers/YangHandler.py | 109 ++++ .../gnmi_openconfig/handlers/_Handler.py | 9 +- .../gnmi_openconfig/handlers/__init__.py | 24 +- .../gnmi_openconfig/tools/Capabilities.py | 17 +- .../drivers/gnmi_openconfig/tools/Value.py | 2 +- src/device/tests/test_gnmi.py | 115 ---- .../tests/test_unitary_gnmi_openconfig.py | 616 ++++++++++++++++++ 13 files changed, 1149 insertions(+), 314 deletions(-) create mode 100755 scripts/run_tests_locally-device-gnmi-openconfig.sh create mode 100644 src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py delete mode 100644 src/device/tests/test_gnmi.py create mode 100644 src/device/tests/test_unitary_gnmi_openconfig.py diff --git a/scripts/run_tests_locally-device-gnmi-openconfig.sh b/scripts/run_tests_locally-device-gnmi-openconfig.sh new file mode 100755 index 000000000..d81684da1 --- /dev/null +++ b/scripts/run_tests_locally-device-gnmi-openconfig.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +PROJECTDIR=`pwd` + +cd $PROJECTDIR/src +RCFILE=$PROJECTDIR/coverage/.coveragerc + +# Run unitary tests and analyze coverage of code at same time +# helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 +coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ + device/tests/test_unitary_gnmi_openconfig.py diff --git a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py index 6f80ee82f..d9f73c958 100644 --- a/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py +++ b/src/device/service/drivers/gnmi_openconfig/GnmiSessionHandler.py @@ -19,12 +19,13 @@ from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk from .gnmi.gnmi_pb2_grpc import gNMIStub from .gnmi.gnmi_pb2 import Encoding, GetRequest, SetRequest, UpdateResult # pylint: disable=no-name-in-module from .handlers import ALL_RESOURCE_KEYS, compose, get_path, parse -from .tools.Capabilities import get_supported_encodings +from .handlers.YangHandler import YangHandler +from .tools.Capabilities import check_capabilities from .tools.Channel import get_grpc_channel from .tools.Path import path_from_string, path_to_string #, compose_path from .tools.Subscriptions import Subscriptions from .tools.Value import decode_value #, value_exists -from .MonitoringThread import MonitoringThread +#from .MonitoringThread import MonitoringThread class GnmiSessionHandler: def __init__(self, address : str, port : int, settings : Dict, logger : logging.Logger) -> None: @@ -39,12 +40,20 @@ class GnmiSessionHandler: self._use_tls = settings.get('use_tls', False) self._channel : Optional[grpc.Channel] = None self._stub : Optional[gNMIStub] = None - self._monit_thread = None - self._supported_encodings = None + self._yang_handler = YangHandler() + #self._monit_thread = None self._subscriptions = Subscriptions() self._in_subscriptions = queue.Queue() self._out_samples = queue.Queue() + def __del__(self) -> None: + self._logger.warning('Destroying YangValidator...') + self._logger.warning('yang_validator.data:') + for path, dnode in self._yang_handler.get_data_paths().items(): + self._logger.warning(' {:s}: {:s}'.format(str(path), json.dumps(dnode.print_dict()))) + self._yang_handler.destroy() + self._logger.warning('DONE') + @property def subscriptions(self): return self._subscriptions @@ -58,18 +67,17 @@ class GnmiSessionHandler: with self._lock: self._channel = get_grpc_channel(self._address, self._port, self._use_tls, self._logger) self._stub = gNMIStub(self._channel) - self._supported_encodings = get_supported_encodings( - self._stub, self._username, self._password, timeout=120) - self._monit_thread = MonitoringThread( - self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) - self._monit_thread.start() + check_capabilities(self._stub, self._username, self._password, timeout=120) + #self._monit_thread = MonitoringThread( + # self._stub, self._logger, self._settings, self._in_subscriptions, self._out_samples) + #self._monit_thread.start() self._connected.set() def disconnect(self): if not self._connected.is_set(): return with self._lock: - self._monit_thread.stop() - self._monit_thread.join() + #self._monit_thread.stop() + #self._monit_thread.join() self._channel.close() self._connected.clear() @@ -87,9 +95,9 @@ class GnmiSessionHandler: str_resource_name = 'resource_key[#{:d}]'.format(i) try: chk_string(str_resource_name, resource_key, allow_empty=False) - self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) + #self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key))) str_path = get_path(resource_key) - self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) + #self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path))) get_request.path.append(path_from_string(str_path)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception parsing {:s}: {:s}' @@ -130,7 +138,7 @@ class GnmiSessionHandler: value = decode_value(update.val) #resource_key_tuple[1] = value #resource_key_tuple[2] = True - results.extend(parse(str_path, value)) + results.extend(parse(str_path, value, self._yang_handler)) except Exception as e: # pylint: disable=broad-except MSG = 'Exception processing update {:s}' self._logger.exception(MSG.format(grpc_message_to_json_string(update))) @@ -159,17 +167,17 @@ class GnmiSessionHandler: set_request = SetRequest() #for resource_key in resource_keys: for resource_key, resource_value in resources: - self._logger.info('---1') - self._logger.info(str(resource_key)) - self._logger.info(str(resource_value)) + #self._logger.info('---1') + #self._logger.info(str(resource_key)) + #self._logger.info(str(resource_value)) #resource_tuple = resource_tuples.get(resource_key) #if resource_tuple is None: continue #_, value, exists, operation_done = resource_tuple if isinstance(resource_value, str): resource_value = json.loads(resource_value) - str_path, str_data = compose(resource_key, resource_value, delete=False) - self._logger.info('---3') - self._logger.info(str(str_path)) - self._logger.info(str(str_data)) + str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=False) + #self._logger.info('---3') + #self._logger.info(str(str_path)) + #self._logger.info(str(str_data)) set_request_list = set_request.update #if exists else set_request.replace set_request_entry = set_request_list.add() set_request_entry.path.CopyFrom(path_from_string(str_path)) @@ -228,18 +236,19 @@ class GnmiSessionHandler: set_request = SetRequest() #for resource_key in resource_keys: for resource_key, resource_value in resources: - self._logger.info('---1') - self._logger.info(str(resource_key)) - self._logger.info(str(resource_value)) + #self._logger.info('---1') + #self._logger.info(str(resource_key)) + #self._logger.info(str(resource_value)) #resource_tuple = resource_tuples.get(resource_key) #if resource_tuple is None: continue #_, value, exists, operation_done = resource_tuple #if not exists: continue if isinstance(resource_value, str): resource_value = json.loads(resource_value) - str_path, str_data = compose(resource_key, resource_value, delete=True) - self._logger.info('---3') - self._logger.info(str(str_path)) - self._logger.info(str(str_data)) + # pylint: disable=unused-variable + str_path, str_data = compose(resource_key, resource_value, self._yang_handler, delete=True) + #self._logger.info('---3') + #self._logger.info(str(str_path)) + #self._logger.info(str(str_data)) set_request_entry = set_request.delete.add() set_request_entry.CopyFrom(path_from_string(str_path)) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index cddf40d56..73728192f 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,37 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging #, json -import pyangbind.lib.pybindJSON as pybindJSON +import json, logging # libyang from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType -from . import openconfig from ._Handler import _Handler +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) -PATH_IF_CTR = "/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}" +PATH_IF_CTR = '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/{:s}' #pylint: disable=abstract-method class ComponentHandler(_Handler): def get_resource_key(self) -> str: return '/endpoints/endpoint' def get_path(self) -> str: return '/openconfig-platform:components' - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) - oc_components = pybindJSON.loads_ietf(json_data, openconfig.components, 'components') - #LOGGER.info('oc_components = {:s}'.format(pybindJSON.dumps(oc_components, mode='ietf'))) + yang_components_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_components_path, json_data, fmt='json') entries = [] - for component_key, oc_component in oc_components.component.items(): - #LOGGER.info('component_key={:s} oc_component={:s}'.format( - # component_key, pybindJSON.dumps(oc_component, mode='ietf') - #)) + for component in json_data_valid['components']['component']: + LOGGER.debug('component={:s}'.format(str(component))) - component_name = oc_component.config.name + component_name = component['name'] + #component_config = component.get('config', {}) - component_type = oc_component.state.type + #yang_components : libyang.DContainer = yang_handler.get_data_path(yang_components_path) + #yang_component_path = 'component[name="{:s}"]'.format(component_name) + #yang_component : libyang.DContainer = yang_components.create_path(yang_component_path) + #yang_component.merge_data_dict(component, strict=True, validate=False) + + component_state = component.get('state', {}) + component_type = component_state.get('type') + if component_type is None: continue component_type = component_type.split(':')[-1] if component_type not in {'PORT'}: continue @@ -58,8 +65,6 @@ class ComponentHandler(_Handler): KpiSampleType.KPISAMPLETYPE_PACKETS_TRANSMITTED: PATH_IF_CTR.format(interface_name, 'out-pkts' ), } - if len(endpoint) == 0: continue - entries.append(('/endpoints/endpoint[{:s}]'.format(endpoint['uuid']), endpoint)) - + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py index 77310d51d..f28cdcf36 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Interface.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging -import pyangbind.lib.pybindJSON as pybindJSON +import json, libyang, logging from typing import Any, Dict, List, Tuple -from . import openconfig from ._Handler import _Handler +from .Tools import get_bool, get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -24,9 +24,11 @@ class InterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/interface' def get_path(self) -> str: return '/openconfig-interfaces:interfaces' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - if_name = str (resource_value['name' ]) # ethernet-1/1 - sif_index = int (resource_value.get('sub_if_index' , 0 )) # 0 + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + if_name = get_str(resource_value, 'name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'sub_if_index', 0) # 0 if delete: PATH_TMPL = '/interfaces/interface[name={:s}]/subinterfaces/subinterface[index={:d}]' @@ -34,118 +36,166 @@ class InterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - if_enabled = bool(resource_value.get('enabled' , True)) # True/False - sif_enabled = bool(resource_value.get('sub_if_enabled' , True)) # True/False - sif_ipv4_enabled = bool(resource_value.get('sub_if_ipv4_enabled', True)) # True/False - sif_ipv4_address = str (resource_value['sub_if_ipv4_address' ]) # 172.16.0.1 - sif_ipv4_prefix = int (resource_value['sub_if_ipv4_prefix' ]) # 24 + if_enabled = get_bool(resource_value, 'enabled', True) # True/False + sif_enabled = get_bool(resource_value, 'sub_if_enabled', True) # True/False + sif_vlan_id = get_int (resource_value, 'sif_vlan_id', ) # 127 + sif_ipv4_enabled = get_bool(resource_value, 'sub_if_ipv4_enabled', True) # True/False + sif_ipv4_address = get_str (resource_value, 'sub_if_ipv4_address' ) # 172.16.0.1 + sif_ipv4_prefix = get_int (resource_value, 'sub_if_ipv4_prefix' ) # 24 + + yang_ifs : libyang.DContainer = yang_handler.get_data_path('/openconfig-interfaces:interfaces') + yang_if_path = 'interface[name="{:s}"]'.format(if_name) + yang_if : libyang.DContainer = yang_ifs.create_path(yang_if_path) + yang_if.create_path('config/name', if_name ) + if if_enabled is not None: yang_if.create_path('config/enabled', if_enabled) + + yang_sifs : libyang.DContainer = yang_if.create_path('subinterfaces') + yang_sif_path = 'subinterface[index="{:d}"]'.format(sif_index) + yang_sif : libyang.DContainer = yang_sifs.create_path(yang_sif_path) + yang_sif.create_path('config/index', sif_index) + if sif_enabled is not None: yang_sif.create_path('config/enabled', sif_enabled) + + if sif_vlan_id is not None: + yang_subif_vlan : libyang.DContainer = yang_sif.create_path('openconfig-vlan:vlan') + yang_subif_vlan.create_path('match/single-tagged/config/vlan-id', sif_vlan_id) + + yang_ipv4 : libyang.DContainer = yang_sif.create_path('openconfig-if-ip:ipv4') + if sif_ipv4_enabled is not None: yang_ipv4.create_path('config/enabled', sif_ipv4_enabled) + + if sif_ipv4_address is not None: + yang_ipv4_addrs : libyang.DContainer = yang_ipv4.create_path('addresses') + yang_ipv4_addr_path = 'address[ip="{:s}"]'.format(sif_ipv4_address) + yang_ipv4_addr : libyang.DContainer = yang_ipv4_addrs.create_path(yang_ipv4_addr_path) + yang_ipv4_addr.create_path('config/ip', sif_ipv4_address) + yang_ipv4_addr.create_path('config/prefix-length', sif_ipv4_prefix ) str_path = '/interfaces/interface[name={:s}]'.format(if_name) - str_data = json.dumps({ - 'name': if_name, - 'config': {'name': if_name, 'enabled': if_enabled}, - 'subinterfaces': { - 'subinterface': { - 'index': sif_index, - 'config': {'index': sif_index, 'enabled': sif_enabled}, - 'ipv4': { - 'config': {'enabled': sif_ipv4_enabled}, - 'addresses': { - 'address': { - 'ip': sif_ipv4_address, - 'config': {'ip': sif_ipv4_address, 'prefix_length': sif_ipv4_prefix}, - } - } - } - } - } - }) + str_data = yang_if.print_mem('json') + json_data = json.loads(str_data) + json_data = json_data['openconfig-interfaces:interface'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - #LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') - #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + + yang_interfaces_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_interfaces_path, json_data, fmt='json') entries = [] - for interface_key, oc_interface in oc_interfaces.interface.items(): - #LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( - # interface_key, pybindJSON.dumps(oc_interface, mode='ietf') - #)) - - interface = {} - interface['name'] = oc_interface.config.name - - interface_type = oc_interface.config.type - interface_type = interface_type.replace('ianaift:', '') - interface_type = interface_type.replace('iana-if-type:', '') - interface['type'] = interface_type - - interface['mtu' ] = oc_interface.config.mtu - interface['enabled' ] = oc_interface.config.enabled - interface['description' ] = oc_interface.config.description - interface['admin-status'] = oc_interface.state.admin_status - interface['oper-status' ] = oc_interface.state.oper_status - interface['management' ] = oc_interface.state.management - - entry_interface_key = '/interface[{:s}]'.format(interface['name']) - entries.append((entry_interface_key, interface)) - - for subinterface_key, oc_subinterface in oc_interface.subinterfaces.subinterface.items(): - #LOGGER.info('subinterface_key={:d} oc_subinterfaces={:s}'.format( - # subinterface_key, pybindJSON.dumps(oc_subinterface, mode='ietf') - #)) - - subinterface = {} - subinterface['index' ] = oc_subinterface.state.index - subinterface['name' ] = oc_subinterface.state.name - subinterface['enabled'] = oc_subinterface.state.enabled - - entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface['index']) - entries.append((entry_subinterface_key, subinterface)) - - #VLAN_FIELDS = ('vlan', 'openconfig-vlan:vlan', 'ocv:vlan') - #json_vlan = dict_get_first(json_subinterface, VLAN_FIELDS, default={}) - - #MATCH_FIELDS = ('match', 'openconfig-vlan:match', 'ocv:match') - #json_vlan = dict_get_first(json_vlan, MATCH_FIELDS, default={}) - - #SIN_TAG_FIELDS = ('single-tagged', 'openconfig-vlan:single-tagged', 'ocv:single-tagged') - #json_vlan = dict_get_first(json_vlan, SIN_TAG_FIELDS, default={}) - - #CONFIG_FIELDS = ('config', 'openconfig-vlan:config', 'ocv:config') - #json_vlan = dict_get_first(json_vlan, CONFIG_FIELDS, default={}) - - #VLAN_ID_FIELDS = ('vlan-id', 'openconfig-vlan:vlan-id', 'ocv:vlan-id') - #subinterface_vlan_id = dict_get_first(json_vlan, VLAN_ID_FIELDS) - #if subinterface_vlan_id is not None: subinterface['vlan_id'] = subinterface_vlan_id - - for address_key, oc_address in oc_subinterface.ipv4.addresses.address.items(): - #LOGGER.info('ipv4: address_key={:s} oc_address={:s}'.format( - # address_key, pybindJSON.dumps(oc_address, mode='ietf') - #)) - - address_ipv4 = { - 'ip' : oc_address.state.ip, - 'origin': oc_address.state.origin, - 'prefix': oc_address.state.prefix_length, - } - - entry_address_ipv4_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, address_ipv4['ip']) - entries.append((entry_address_ipv4_key, address_ipv4)) - - for address_key, oc_address in oc_subinterface.ipv6.addresses.address.items(): - #LOGGER.info('ipv6: address_key={:s} oc_address={:s}'.format( - # address_key, pybindJSON.dumps(oc_address, mode='ietf') - #)) - - address_ipv6 = { - 'ip' : oc_address.state.ip, - 'origin': oc_address.state.origin, - 'prefix': oc_address.state.prefix_length, - } - - entry_address_ipv6_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, address_ipv6['ip']) - entries.append((entry_address_ipv6_key, address_ipv6)) + for interface in json_data_valid['interfaces']['interface']: + LOGGER.debug('interface={:s}'.format(str(interface))) + + interface_name = interface['name'] + interface_config = interface.get('config', {}) + + #yang_interfaces : libyang.DContainer = yang_handler.get_data_path(yang_interfaces_path) + #yang_interface_path = 'interface[name="{:s}"]'.format(interface_name) + #yang_interface : libyang.DContainer = yang_interfaces.create_path(yang_interface_path) + #yang_interface.merge_data_dict(interface, strict=True, validate=False) + + interface_state = interface.get('state', {}) + interface_type = interface_state.get('type') + if interface_type is None: continue + interface_type = interface_type.split(':')[-1] + if interface_type not in {'ethernetCsmacd'}: continue + + _interface = { + 'name' : interface_name, + 'type' : interface_type, + 'mtu' : interface_state['mtu'], + 'ifindex' : interface_state['ifindex'], + 'admin-status' : interface_state['admin-status'], + 'oper-status' : interface_state['oper-status'], + 'management' : interface_state['management'], + } + if 'description' in interface_config: + _interface['description'] = interface_config['description'] + if 'enabled' in interface_config: + _interface['enabled'] = interface_config['enabled'] + if 'hardware-port' in interface_state: + _interface['hardware-port'] = interface_state['hardware-port'] + if 'transceiver' in interface_state: + _interface['transceiver'] = interface_state['transceiver'] + + entry_interface_key = '/interface[{:s}]'.format(interface_name) + entries.append((entry_interface_key, _interface)) + + if interface_type == 'ethernetCsmacd': + ethernet_state = interface['ethernet']['state'] + + _ethernet = { + 'mac-address' : ethernet_state['mac-address'], + 'hw-mac-address' : ethernet_state['hw-mac-address'], + 'port-speed' : ethernet_state['port-speed'].split(':')[-1], + 'negotiated-port-speed' : ethernet_state['negotiated-port-speed'].split(':')[-1], + } + entry_ethernet_key = '{:s}/ethernet'.format(entry_interface_key) + entries.append((entry_ethernet_key, _ethernet)) + + subinterfaces = interface.get('subinterfaces', {}).get('subinterface', []) + for subinterface in subinterfaces: + LOGGER.debug('subinterface={:s}'.format(str(subinterface))) + + subinterface_index = subinterface['index'] + subinterface_state = subinterface.get('state', {}) + + _subinterface = {'index': subinterface_index} + if 'name' in subinterface_state: + _subinterface['name'] = subinterface_state['name'] + if 'enabled' in subinterface_state: + _subinterface['enabled'] = subinterface_state['enabled'] + entry_subinterface_key = '{:s}/subinterface[{:d}]'.format(entry_interface_key, subinterface_index) + entries.append((entry_subinterface_key, _subinterface)) + + if 'vlan' in subinterface: + vlan = subinterface['vlan'] + vlan_match = vlan['match'] + + single_tagged = vlan_match.pop('single-tagged', None) + if single_tagged is not None: + single_tagged_config = single_tagged['config'] + vlan_id = single_tagged_config['vlan-id'] + + _vlan = {'vlan_id': vlan_id} + entry_vlan_key = '{:s}/vlan[single:{:s}]'.format(entry_subinterface_key, vlan_id) + entries.append((entry_vlan_key, _vlan)) + + if len(vlan_match) > 0: + raise Exception('Unsupported VLAN schema: {:s}'.format(str(vlan))) + + ipv4_addresses = subinterface.get('ipv4', {}).get('addresses', {}).get('address', []) + for ipv4_address in ipv4_addresses: + LOGGER.debug('ipv4_address={:s}'.format(str(ipv4_address))) + + ipv4_address_ip = ipv4_address['ip'] + ipv4_address_state = ipv4_address.get('state', {}) + + _ipv4_address = {'ip': ipv4_address_ip} + if 'origin' in ipv4_address_state: + _ipv4_address['origin'] = ipv4_address_state['origin'] + if 'prefix-length' in ipv4_address_state: + _ipv4_address['prefix'] = ipv4_address_state['prefix-length'] + + entry_ipv4_address_key = '{:s}/ipv4[{:s}]'.format(entry_subinterface_key, ipv4_address_ip) + entries.append((entry_ipv4_address_key, _ipv4_address)) + + ipv6_addresses = subinterface.get('ipv6', {}).get('addresses', {}).get('address', []) + for ipv6_address in ipv6_addresses: + LOGGER.debug('ipv6_address={:s}'.format(str(ipv6_address))) + + ipv6_address_ip = ipv6_address['ip'] + ipv6_address_state = ipv6_address.get('state', {}) + + _ipv6_address = {'ip': ipv6_address_ip} + if 'origin' in ipv6_address_state: + _ipv6_address['origin'] = ipv6_address_state['origin'] + if 'prefix-length' in ipv6_address_state: + _ipv6_address['prefix'] = ipv6_address_state['prefix-length'] + + entry_ipv6_address_key = '{:s}/ipv6[{:s}]'.format(entry_subinterface_key, ipv6_address_ip) + entries.append((entry_ipv6_address_key, _ipv6_address)) return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index c29ed263a..0b4d15745 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,20 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging -import pyangbind.lib.pybindJSON as pybindJSON +import json, libyang, logging +import operator from typing import Any, Dict, List, Tuple -from . import openconfig from ._Handler import _Handler +from .Tools import get_bool, get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) +MAP_NETWORK_INSTANCE_TYPE = { + # special routing instance; acts as default/global routing instance for a network device + 'DEFAULT': 'openconfig-network-instance-types:DEFAULT_INSTANCE', + + # private L3-only routing instance; formed of one or more RIBs + 'L3VRF': 'openconfig-network-instance-types:L3VRF', + + # private L2-only switch instance; formed of one or more L2 forwarding tables + 'L2VSI': 'openconfig-network-instance-types:L2VSI', + + # private L2-only forwarding instance; point to point connection between two endpoints + 'L2P2P': 'openconfig-network-instance-types:L2P2P', + + # private Layer 2 and Layer 3 forwarding instance + 'L2L3': 'openconfig-network-instance-types:L2L3', +} + class NetworkInstanceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance' def get_path(self) -> str: return '/openconfig-network-instance:network-instances' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name']) # test-svc + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name') # test-svc if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]' @@ -33,15 +53,11 @@ class NetworkInstanceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - ni_type = str(resource_value['type']) # L3VRF / L2VSI / ... + ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ... + ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type) - # not works: [FailedPrecondition] unsupported identifier 'DIRECTLY_CONNECTED' - #protocols = [self._compose_directly_connected()] + # 'DIRECTLY_CONNECTED' is implicitly added - MAP_OC_NI_TYPE = { - 'L3VRF': 'openconfig-network-instance-types:L3VRF', - } - ni_type = MAP_OC_NI_TYPE.get(ni_type, ni_type) str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) str_data = json.dumps({ @@ -51,19 +67,92 @@ class NetworkInstanceHandler(_Handler): }) return str_path, str_data - def _compose_directly_connected(self, name=None, enabled=True) -> Dict: - identifier = 'DIRECTLY_CONNECTED' - if name is None: name = 'DIRECTLY_CONNECTED' - return { - 'identifier': identifier, 'name': name, - 'config': {'identifier': identifier, 'name': name, 'enabled': enabled}, - } - - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_network_instances = pybindJSON.loads_ietf(json_data, openconfig., 'interfaces') - #LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - response = [] - return response - -openconfig-network-instance:network-instance \ No newline at end of file + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + + # Arista Parsing Fixes: + # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0 + # overwrite with .../hellos/config/hello-interval + network_instances = json_data.get('openconfig-network-instance:network-instance', []) + for network_instance in network_instances: + if network_instance['name'] != 'default': continue + mpls_rsvp_te = network_instance.get('mpls', {}).get('signaling-protocols', {}).get('rsvp-te', {}) + mpls_rsvp_te_hellos = mpls_rsvp_te.get('global', {}).get('hellos', {}) + hello_interval = mpls_rsvp_te_hellos.get('config', {}).get('hello-interval', 9000) + mpls_rsvp_te_hellos.get('state', {})['hello-interval'] = hello_interval + + yang_network_instances_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_network_instances_path, json_data, fmt='json', strict=False) + + entries = [] + for network_instance in json_data_valid['network-instances']['network-instance']: + LOGGER.debug('network_instance={:s}'.format(str(network_instance))) + ni_name = network_instance['name'] + + ni_config = network_instance['config'] + ni_type = ni_config['type'].split(':')[-1] + + _net_inst = {'name': ni_name, 'type': ni_type} + entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name) + entries.append((entry_net_inst_key, _net_inst)) + + ni_protocols = network_instance.get('protocols', {}).get('protocol', []) + for ni_protocol in ni_protocols: + ni_protocol_id = ni_protocol['identifier'].split(':')[-1] + ni_protocol_name = ni_protocol['name'] + + _protocol = {'id': ni_protocol_id, 'name': ni_protocol_name} + entry_protocol_key = '{:s}/protocol[{:s}]'.format(entry_net_inst_key, ni_protocol_id) + entries.append((entry_protocol_key, _protocol)) + + if ni_protocol_id == 'STATIC': + static_routes = ni_protocol.get('static-routes', {}).get('static', []) + for static_route in static_routes: + static_route_prefix = static_route['prefix'] + + next_hops = static_route.get('next-hops', {}).get('next-hop', []) + _next_hops = [ + { + 'index' : next_hop['index'], + 'gateway': next_hop['config']['next-hop'], + 'metric' : next_hop['config']['metric'], + } + for next_hop in next_hops + ] + _next_hops = sorted(_next_hops, key=operator.itemgetter('index')) + + _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops} + entry_static_route_key = '{:s}/static_routes[{:s}]'.format( + entry_protocol_key, static_route_prefix + ) + entries.append((entry_static_route_key, _static_route)) + + ni_tables = network_instance.get('tables', {}).get('table', []) + for ni_table in ni_tables: + ni_table_protocol = ni_table['protocol'].split(':')[-1] + ni_table_address_family = ni_table['address-family'].split(':')[-1] + _table = {'protocol': ni_table_protocol, 'address_family': ni_table_address_family} + entry_table_key = '{:s}/table[{:s},{:s}]'.format( + entry_net_inst_key, ni_table_protocol, ni_table_address_family + ) + entries.append((entry_table_key, _table)) + + ni_vlans = network_instance.get('vlans', {}).get('vlan', []) + for ni_vlan in ni_vlans: + ni_vlan_id = ni_vlan['vlan-id'] + + #ni_vlan_config = ni_vlan['config'] + ni_vlan_state = ni_vlan['state'] + ni_vlan_name = ni_vlan_state['name'] + + _members = [ + member['state']['interface'] + for member in ni_vlan.get('members', {}).get('member', []) + ] + _vlan = {'vlan_id': ni_vlan_id, 'name': ni_vlan_name, 'members': _members} + entry_vlan_key = '{:s}/vlan[{:d}]'.format(entry_net_inst_key, ni_vlan_id) + entries.append((entry_vlan_key, _vlan)) + + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py index 8cf704e29..dfb8eabaf 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Tools.py @@ -13,7 +13,7 @@ # limitations under the License. import re -from typing import Any, Dict, Iterable, Optional +from typing import Any, Callable, Dict, Iterable, Optional RE_REMOVE_FILTERS = re.compile(r'\[[^\]]+\]') RE_REMOVE_NAMESPACES = re.compile(r'\/[a-zA-Z0-9\_\-]+:') @@ -40,3 +40,22 @@ def container_get_first( if namespace_key_name in container: return container[namespace_key_name] return default + +def get_value( + resource_value : Dict, field_name : str, cast_func : Callable = lambda x:x, default : Optional[Any] = None +) -> Optional[Any]: + field_value = resource_value.get(field_name, default) + if field_value is not None: field_value = cast_func(field_value) + return field_value + +def get_bool(resource_value : Dict, field_name : bool, default : Optional[Any] = None) -> bool: + return get_value(resource_value, field_name, cast_func=bool, default=default) + +def get_float(resource_value : Dict, field_name : float, default : Optional[Any] = None) -> float: + return get_value(resource_value, field_name, cast_func=float, default=default) + +def get_int(resource_value : Dict, field_name : int, default : Optional[Any] = None) -> int: + return get_value(resource_value, field_name, cast_func=int, default=default) + +def get_str(resource_value : Dict, field_name : str, default : Optional[Any] = None) -> str: + return get_value(resource_value, field_name, cast_func=str, default=default) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py new file mode 100644 index 000000000..fe8672187 --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/handlers/YangHandler.py @@ -0,0 +1,109 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, libyang, logging, os +from typing import Dict, Optional + +YANG_BASE_PATH = os.path.join(os.path.dirname(__file__), '..', 'git', 'openconfig', 'public') +YANG_SEARCH_PATHS = ':'.join([ + os.path.join(YANG_BASE_PATH, 'release'), + os.path.join(YANG_BASE_PATH, 'third_party'), +]) + +YANG_MODULES = [ + 'iana-if-type', + 'openconfig-vlan-types', + + 'openconfig-interfaces', + 'openconfig-if-8021x', + 'openconfig-if-aggregate', + 'openconfig-if-ethernet-ext', + 'openconfig-if-ethernet', + 'openconfig-if-ip-ext', + 'openconfig-if-ip', + 'openconfig-if-poe', + 'openconfig-if-sdn-ext', + 'openconfig-if-tunnel', + + 'openconfig-vlan', + + 'openconfig-types', + 'openconfig-policy-types', + 'openconfig-mpls-types', + 'openconfig-network-instance-types', + 'openconfig-network-instance', + + 'openconfig-platform', + 'openconfig-platform-controller-card', + 'openconfig-platform-cpu', + 'openconfig-platform-ext', + 'openconfig-platform-fabric', + 'openconfig-platform-fan', + 'openconfig-platform-integrated-circuit', + 'openconfig-platform-linecard', + 'openconfig-platform-pipeline-counters', + 'openconfig-platform-port', + 'openconfig-platform-psu', + 'openconfig-platform-software', + 'openconfig-platform-transceiver', + 'openconfig-platform-types', +] + +LOGGER = logging.getLogger(__name__) + +class YangHandler: + def __init__(self) -> None: + self._yang_context = libyang.Context(YANG_SEARCH_PATHS) + self._loaded_modules = set() + for yang_module_name in YANG_MODULES: + LOGGER.info('Loading module: {:s}'.format(str(yang_module_name))) + self._yang_context.load_module(yang_module_name).feature_enable_all() + self._loaded_modules.add(yang_module_name) + self._data_path_instances = dict() + + def get_data_paths(self) -> Dict[str, libyang.DNode]: + return self._data_path_instances + + def get_data_path(self, path : str) -> libyang.DNode: + data_path_instance = self._data_path_instances.get(path) + if data_path_instance is None: + data_path_instance = self._yang_context.create_data_path(path) + self._data_path_instances[path] = data_path_instance + return data_path_instance + + def parse_to_dict( + self, request_path : str, json_data : Dict, fmt : str = 'json', strict : bool = True + ) -> Dict: + if fmt != 'json': raise Exception('Unsupported format: {:s}'.format(str(fmt))) + LOGGER.debug('request_path = {:s}'.format(str(request_path))) + LOGGER.debug('json_data = {:s}'.format(str(json_data))) + LOGGER.debug('format = {:s}'.format(str(fmt))) + + parent_path_parts = list(filter(lambda s: len(s) > 0, request_path.split('/'))) + for parent_path_part in reversed(parent_path_parts): + json_data = {parent_path_part: json_data} + str_data = json.dumps(json_data) + + dnode : Optional[libyang.DNode] = self._yang_context.parse_data_mem( + str_data, fmt, strict=strict, parse_only=True, #validate_present=True, #validate=True, + ) + if dnode is None: raise Exception('Unable to parse Data({:s})'.format(str(json_data))) + + parsed = dnode.print_dict() + LOGGER.debug('parsed = {:s}'.format(json.dumps(parsed))) + dnode.free() + return parsed + + def destroy(self) -> None: + self._yang_context.destroy() diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py b/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py index d20c77b11..a03692d95 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/_Handler.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Any, Dict, List, Tuple +from .YangHandler import YangHandler class _Handler: def get_resource_key(self) -> str: @@ -23,10 +24,14 @@ class _Handler: # Retrieve the OpenConfig path schema used to interrogate the device raise NotImplementedError() - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: # Compose a Set/Delete message based on the resource_key/resource_value fields, and the delete flag raise NotImplementedError() - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: # Parse a Reply from the device and return a list of resource_key/resource_value pairs raise NotImplementedError() diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py index 6d54ef28d..38bc4db40 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES from ._Handler import _Handler from .Component import ComponentHandler @@ -23,6 +23,7 @@ from .NetworkInstance import NetworkInstanceHandler from .NetworkInstanceInterface import NetworkInstanceInterfaceHandler from .NetworkInstanceStaticRoute import NetworkInstanceStaticRouteHandler from .Tools import get_schema +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -71,7 +72,8 @@ PATH_TO_HANDLER = { } def get_handler( - resource_key : Optional[str] = None, path : Optional[str] = None, raise_if_not_found=True + resource_key : Optional[str] = None, path : Optional[str] = None, + raise_if_not_found=True ) -> Optional[_Handler]: if (resource_key is None) == (path is None): MSG = 'Exactly one of resource_key({:s}) or path({:s}) must be specified' @@ -95,10 +97,18 @@ def get_handler( return handler def get_path(resource_key : str) -> str: - return get_handler(resource_key=resource_key).get_path() + handler = get_handler(resource_key=resource_key) + return handler.get_path() -def parse(str_path : str, value : Union[Dict, List]): - return get_handler(path=str_path).parse(value) +def parse( + str_path : str, value : Union[Dict, List], yang_handler : YangHandler +) -> List[Tuple[str, Dict[str, Any]]]: + handler = get_handler(path=str_path) + return handler.parse(value, yang_handler) -def compose(resource_key : str, resource_value : Union[Dict, List], delete : bool = False) -> Tuple[str, str]: - return get_handler(resource_key=resource_key).compose(resource_key, resource_value, delete=delete) +def compose( + resource_key : str, resource_value : Union[Dict, List], + yang_handler : YangHandler, delete : bool = False +) -> Tuple[str, str]: + handler = get_handler(resource_key=resource_key) + return handler.compose(resource_key, resource_value, yang_handler, delete=delete) diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py b/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py index b90bf3db8..4c202da2c 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Capabilities.py @@ -17,7 +17,7 @@ from common.tools.grpc.Tools import grpc_message_to_json from ..gnmi.gnmi_pb2 import CapabilityRequest # pylint: disable=no-name-in-module from ..gnmi.gnmi_pb2_grpc import gNMIStub -def get_supported_encodings( +def check_capabilities( stub : gNMIStub, username : str, password : str, timeout : Optional[int] = None ) -> Set[Union[str, int]]: metadata = [('username', username), ('password', password)] @@ -25,6 +25,17 @@ def get_supported_encodings( reply = stub.Capabilities(req, metadata=metadata, timeout=timeout) data = grpc_message_to_json(reply) + + gnmi_version = data.get('gNMI_version') + if gnmi_version is None or gnmi_version != '0.7.0': + raise Exception('Unsupported gNMI version: {:s}'.format(str(gnmi_version))) + + #supported_models = { + # supported_model['name']: supported_model['version'] + # for supported_model in data.get('supported_models', []) + #} + # TODO: check supported models and versions + supported_encodings = { supported_encoding for supported_encoding in data.get('supported_encodings', []) @@ -33,4 +44,6 @@ def get_supported_encodings( if len(supported_encodings) == 0: # pylint: disable=broad-exception-raised raise Exception('No supported encodings found') - return supported_encodings + if 'JSON_IETF' not in supported_encodings: + # pylint: disable=broad-exception-raised + raise Exception('JSON_IETF encoding not supported') diff --git a/src/device/service/drivers/gnmi_openconfig/tools/Value.py b/src/device/service/drivers/gnmi_openconfig/tools/Value.py index 9933cb858..73e43b87c 100644 --- a/src/device/service/drivers/gnmi_openconfig/tools/Value.py +++ b/src/device/service/drivers/gnmi_openconfig/tools/Value.py @@ -61,7 +61,7 @@ def decode_value(value : TypedValue) -> Any: str_value : str = value.json_ietf_val.decode('UTF-8') try: # Cleanup and normalize the records according to OpenConfig - str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') + #str_value = str_value.replace('openconfig-platform-types:', 'oc-platform-types:') json_value = json.loads(str_value) recursive_remove_keys(json_value) return json_value diff --git a/src/device/tests/test_gnmi.py b/src/device/tests/test_gnmi.py deleted file mode 100644 index 684b9f4c3..000000000 --- a/src/device/tests/test_gnmi.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging, os, sys, time -from typing import Dict, Tuple -os.environ['DEVICE_EMULATED_ONLY'] = 'YES' -from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver # pylint: disable=wrong-import-position -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) - -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ -# | # | Name | Container ID | Image | Kind | State | IPv4 Address | IPv6 Address | -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ -# | 1 | clab-tfs-scenario-client1 | a8d48ec3265a | ghcr.io/hellt/network-multitool | linux | running | 172.100.100.201/24 | N/A | -# | 2 | clab-tfs-scenario-client2 | fc88436d2b32 | ghcr.io/hellt/network-multitool | linux | running | 172.100.100.202/24 | N/A | -# | 3 | clab-tfs-scenario-srl1 | b995b9bdadda | ghcr.io/nokia/srlinux | srl | running | 172.100.100.101/24 | N/A | -# | 4 | clab-tfs-scenario-srl2 | aacfc38cc376 | ghcr.io/nokia/srlinux | srl | running | 172.100.100.102/24 | N/A | -# +---+---------------------------+--------------+---------------------------------+-------+---------+--------------------+--------------+ - -def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = {'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, - 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix} - return str_path, str_data - -def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data - -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data - -def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data - -def main(): - driver_settings = { - 'protocol': 'gnmi', - 'username': 'admin', - 'password': 'admin', - 'use_tls' : False, - } - driver = GnmiOpenConfigDriver('172.20.20.101', 6030, **driver_settings) - driver.Connect() - - #resources_to_get = [] - #resources_to_get = [RESOURCE_ENDPOINTS] - #resources_to_get = [RESOURCE_INTERFACES] - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - #resources_to_get = [RESOURCE_ROUTING_POLICIES] - #resources_to_get = [RESOURCE_SERVICES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - #resources_to_set = [ - # network_instance('test-svc', 'L3VRF'), - # - # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), - # network_instance_interface('test-svc', 'ethernet-1/1', 0), - # - # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), - # network_instance_interface('test-svc', 'ethernet-1/2', 0), - # - # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), - # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), - #] - #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - #results_setconfig = driver.SetConfig(resources_to_set) - #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - #resources_to_delete = [ - # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), - # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), - # - # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), - # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), - # - # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), - # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), - # - # #network_instance('20f66fb5', 'L3VRF'), - #] - #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - #results_deleteconfig = driver.DeleteConfig(resources_to_delete) - #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - time.sleep(1) - - driver.Disconnect() - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py new file mode 100644 index 000000000..4c2dca5d5 --- /dev/null +++ b/src/device/tests/test_unitary_gnmi_openconfig.py @@ -0,0 +1,616 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import deepdiff, logging, os, pytest, re, time +from typing import Dict, List, Tuple +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' + +# pylint: disable=wrong-import-position +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + +DRIVER_SETTING_ADDRESS = '172.20.20.101' +DRIVER_SETTING_PORT = 6030 +DRIVER_SETTING_USERNAME = 'admin' +DRIVER_SETTING_PASSWORD = 'admin' +DRIVER_SETTING_USE_TLS = False + +@pytest.fixture(scope='session') +def driver() -> GnmiOpenConfigDriver: + _driver = GnmiOpenConfigDriver( + DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, + username=DRIVER_SETTING_USERNAME, + password=DRIVER_SETTING_PASSWORD, + use_tls=DRIVER_SETTING_USE_TLS, + ) + _driver.Connect() + yield _driver + time.sleep(1) + _driver.Disconnect() + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield dict() + + +##### STORAGE POPULATORS ############################################################################################### + +def populate_interfaces_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['name' ] = if_name + if_storage['type' ] = resource_value.get('type' ) + if_storage['admin-status' ] = resource_value.get('admin-status' ) + if_storage['oper-status' ] = resource_value.get('oper-status' ) + if_storage['ifindex' ] = resource_value.get('ifindex' ) + if_storage['mtu' ] = resource_value.get('mtu' ) + if_storage['management' ] = resource_value.get('management' ) + if_storage['hardware-port'] = resource_value.get('hardware-port') + if_storage['transceiver' ] = resource_value.get('transceiver' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['port-speed' ] = resource_value.get('port-speed' ) + if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') + if_storage['mac-address' ] = resource_value.get('mac-address' ) + if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) + subif_storage['index'] = subif_index + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + ipv4_addr = match.group(3) + ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) + ipv4_address_storage['ip' ] = ipv4_addr + ipv4_address_storage['origin'] = resource_value.get('origin') + ipv4_address_storage['prefix'] = resource_value.get('prefix') + continue + +def populate_network_instances_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + ni_storage = network_instances_storage.setdefault(name, dict()) + ni_storage['name'] = name + ni_storage['type'] = resource_value.get('type') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) + ni_p_storage['id' ] = protocol + ni_p_storage['name'] = protocol + continue + + pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' + match = re.match(pattern, resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + prefix = match.group(3) + ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) + ni_p_s_storage['prefix' ] = prefix + ni_p_s_storage['next_hops'] = resource_value.get('next_hops') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + address_family = match.group(3) + ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) + ni_t_storage['protocol' ] = protocol + ni_t_storage['address_family'] = address_family + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + vlan_id = int(match.group(2)) + ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) + ni_v_storage['vlan_id'] = vlan_id + ni_v_storage['name' ] = resource_value.get('name') + ni_v_storage['members'] = resource_value.get('members') + continue + + +##### EXPECTED CONFIG COMPOSERS ######################################################################################## + +INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]', [ + 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' + ]), + ('/interface[{if_name:s}]/ethernet', [ + 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' + ]), +] + +INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), +] + +INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), +] + +def get_expected_interface_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + expected_interface_config = list() + for if_name, if_storage in interfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name) + resource_value = { + field_name : if_storage[field_name] + for field_name in resource_key_field_names + if field_name in if_storage and if_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) + resource_value = { + field_name : subif_storage[field_name] + for field_name in resource_key_field_names + if field_name in subif_storage and subif_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) + resource_value = { + field_name : ipv4_storage[field_name] + for field_name in resource_key_field_names + if field_name in ipv4_storage and ipv4_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + return expected_interface_config + +NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]', ['name', 'type']), +] + +NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), +] + +NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), +] + +NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), +] + +NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), +] + +def get_expected_network_instance_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + expected_network_instance_config = list() + for ni_name, ni_storage in network_instances_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name) + resource_value = { + field_name : ni_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_storage and ni_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) + resource_value = { + field_name : ni_p_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_storage and ni_p_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) + resource_value = { + field_name : ni_p_s_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format( + ni_name=ni_name, protocol=protocol, address_family=address_family + ) + resource_value = { + field_name : ni_t_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_t_storage and ni_t_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) + resource_value = { + field_name : ni_v_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_v_storage and ni_v_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + return expected_network_instance_config + + +##### REQUEST COMPOSERS ################################################################################################ + +def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: + str_path = '/interface[{:s}]'.format(if_name) + str_data = { + 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, + 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix + } + return str_path, str_data + +def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]'.format(ni_name) + str_data = {'name': ni_name, 'type': ni_type} + return str_path, str_data + +def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) + str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} + return str_path, str_data + +def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) + str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} + return str_path, str_data + +def test_get_endpoints( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_ENDPOINTS] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + expected_getconfig = [ + ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' + }}), + ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' + }}) + ] + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_get_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_interfaces_storage(storage, results_getconfig) + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_get_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_network_instances_storage(storage, results_getconfig) + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_set_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + }), + ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + }) + ]) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_set_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance('test-l3-svc', 'L3VRF'), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + network_instances = sorted(['test-l3-svc']) + results = set(results_setconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[test-l3-svc]', { + 'name': 'test-l3-svc', 'type': 'L3VRF' + }), + ('/network_instance[test-l3-svc]/protocol[DIRECTLY_CONNECTED]', { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV4]', { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV6]', { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_del_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_deleteconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + +def test_del_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance('test-l3-svc', 'L3VRF'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + network_instances = sorted(['test-l3-svc']) + results = set(results_deleteconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +#def test_unitary_gnmi_openconfig( +# driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +#) -> None: +# #resources_to_get = [] +# resources_to_get = [RESOURCE_ENDPOINTS] +# #resources_to_get = [RESOURCE_INTERFACES] +# #resources_to_get = [RESOURCE_NETWORK_INSTANCES] +# #resources_to_get = [RESOURCE_ROUTING_POLICIES] +# #resources_to_get = [RESOURCE_SERVICES] +# LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) +# results_getconfig = driver.GetConfig(resources_to_get) +# LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) +# +# #resources_to_set = [ +# # network_instance('test-svc', 'L3VRF'), +# # +# # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), +# # network_instance_interface('test-svc', 'ethernet-1/1', 0), +# # +# # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), +# # network_instance_interface('test-svc', 'ethernet-1/2', 0), +# # +# # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), +# # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), +# #] +# #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) +# #results_setconfig = driver.SetConfig(resources_to_set) +# #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) +# +# #resources_to_delete = [ +# # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), +# # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), +# # +# # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), +# # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), +# # +# # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), +# # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), +# # +# # #network_instance('20f66fb5', 'L3VRF'), +# #] +# #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) +# #results_deleteconfig = driver.DeleteConfig(resources_to_delete) +# #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) -- GitLab From f49ad8703b3e2cd54614f4ac95d22689e9ea67ba Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 5 Jan 2024 14:40:02 +0000 Subject: [PATCH 013/941] Device component - gNMI/OpenConfig Driver: WORK IN PROGRESS - Corrected basic unitary test for network instances --- .../tests/test_unitary_gnmi_openconfig.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py index 4c2dca5d5..7d33d1a71 100644 --- a/src/device/tests/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/test_unitary_gnmi_openconfig.py @@ -141,7 +141,7 @@ def populate_network_instances_storage( prefix = match.group(3) ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = resource_value.get('next_hops') + ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) continue match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) @@ -161,7 +161,7 @@ def populate_network_instances_storage( ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) ni_v_storage['vlan_id'] = vlan_id ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = resource_value.get('members') + ni_v_storage['members'] = sorted(resource_value.get('members')) continue @@ -389,6 +389,12 @@ def test_get_network_instances( populate_network_instances_storage(storage, results_getconfig) expected_getconfig = get_expected_network_instance_config(storage) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) num_diffs = len(diff_data) if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) @@ -481,6 +487,9 @@ def test_set_network_instances( 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' }) ]) + for resource_key, resource_value in expected_getconfig: + if resource_key == '/network_instance[default]/vlan[1]': + resource_value['members'] = list() LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) permitted_retries = 5 @@ -490,6 +499,12 @@ def test_set_network_instances( results_getconfig = driver.GetConfig(resources_to_get) LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) num_diffs = len(diff_data) if num_diffs == 0: break @@ -562,6 +577,12 @@ def test_del_network_instances( results_getconfig = driver.GetConfig(resources_to_get) LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + expected_getconfig = get_expected_network_instance_config(storage) diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) -- GitLab From a3df08b8600e8aa1c54e27516b5784a126f881c2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 13:57:04 +0000 Subject: [PATCH 014/941] DataPlane-in-a-box: - Added scripts to connect to cEOS CLI - Updated cEOS version in ContainerLab descriptors --- dataplane-in-a-box/arista.clab.yml | 3 ++- dataplane-in-a-box/ceos-cli-wan1.sh | 3 +++ dataplane-in-a-box/ceos-cli-wan2.sh | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100755 dataplane-in-a-box/ceos-cli-wan1.sh create mode 100755 dataplane-in-a-box/ceos-cli-wan2.sh diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 9a8bff73f..4f3b77129 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -24,7 +24,8 @@ topology: kinds: arista_ceos: kind: arista_ceos - image: ceos:4.30.4M + #image: ceos:4.30.4M + image: ceos:4.31.1F linux: kind: linux image: ghcr.io/hellt/network-multitool:latest diff --git a/dataplane-in-a-box/ceos-cli-wan1.sh b/dataplane-in-a-box/ceos-cli-wan1.sh new file mode 100755 index 000000000..4ae21bcb5 --- /dev/null +++ b/dataplane-in-a-box/ceos-cli-wan1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan1 Cli diff --git a/dataplane-in-a-box/ceos-cli-wan2.sh b/dataplane-in-a-box/ceos-cli-wan2.sh new file mode 100755 index 000000000..c931ac940 --- /dev/null +++ b/dataplane-in-a-box/ceos-cli-wan2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan2 Cli -- GitLab From 32d82663c45cb5a53f2980e066a838cc571b6b70 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 13:58:30 +0000 Subject: [PATCH 015/941] Device component - GNMI OpenConfig: - advanced development of driver - updated unitary tests and related scripts --- ...un_tests_locally-device-gnmi-openconfig.sh | 2 +- .../handlers/InterfaceCounter.py | 66 +- .../handlers/NetworkInstance.py | 34 +- .../handlers/NetworkInstanceInterface.py | 52 +- .../handlers/NetworkInstanceStaticRoute.py | 79 ++- src/device/tests/gnmi_openconfig/__init__.py | 14 + .../gnmi_openconfig/request_composers.py | 44 ++ src/device/tests/gnmi_openconfig/storage.py | 285 ++++++++ .../test_unitary_gnmi_openconfig.py | 556 +++++++++++++++ .../tests/test_unitary_gnmi_openconfig.py | 637 ------------------ 10 files changed, 1051 insertions(+), 718 deletions(-) create mode 100644 src/device/tests/gnmi_openconfig/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/request_composers.py create mode 100644 src/device/tests/gnmi_openconfig/storage.py create mode 100644 src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py delete mode 100644 src/device/tests/test_unitary_gnmi_openconfig.py diff --git a/scripts/run_tests_locally-device-gnmi-openconfig.sh b/scripts/run_tests_locally-device-gnmi-openconfig.sh index d81684da1..7183b4104 100755 --- a/scripts/run_tests_locally-device-gnmi-openconfig.sh +++ b/scripts/run_tests_locally-device-gnmi-openconfig.sh @@ -22,4 +22,4 @@ RCFILE=$PROJECTDIR/coverage/.coveragerc # Run unitary tests and analyze coverage of code at same time # helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0 coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \ - device/tests/test_unitary_gnmi_openconfig.py + device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py index 1c2cfc17a..d4701826c 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/InterfaceCounter.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple -import pyangbind.lib.pybindJSON as pybindJSON -from . import openconfig from ._Handler import _Handler +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -25,40 +24,41 @@ class InterfaceCounterHandler(_Handler): def get_resource_key(self) -> str: return '/interface/counters' def get_path(self) -> str: return '/openconfig-interfaces:interfaces/interface/state/counters' - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) - oc_interfaces = pybindJSON.loads_ietf(json_data, openconfig.interfaces, 'interfaces') - LOGGER.info('oc_interfaces = {:s}'.format(pybindJSON.dumps(oc_interfaces, mode='ietf'))) - counters = [] - for interface_key, oc_interface in oc_interfaces.interface.items(): - LOGGER.info('interface_key={:s} oc_interfaces={:s}'.format( - interface_key, pybindJSON.dumps(oc_interface, mode='ietf') - )) + yang_interfaces_path = self.get_path() + json_data_valid = yang_handler.parse_to_dict(yang_interfaces_path, json_data, fmt='json') - interface = {} - interface['name'] = oc_interface.name - - interface_counters = oc_interface.state.counters - interface['in-broadcast-pkts' ] = interface_counters.in_broadcast_pkts - interface['in-discards' ] = interface_counters.in_discards - interface['in-errors' ] = interface_counters.in_errors - interface['in-fcs-errors' ] = interface_counters.in_fcs_errors - interface['in-multicast-pkts' ] = interface_counters.in_multicast_pkts - interface['in-octets' ] = interface_counters.in_octets - interface['in-pkts' ] = interface_counters.in_pkts - interface['in-unicast-pkts' ] = interface_counters.in_unicast_pkts - interface['out-broadcast-pkts'] = interface_counters.out_broadcast_pkts - interface['out-discards' ] = interface_counters.out_discards - interface['out-errors' ] = interface_counters.out_errors - interface['out-multicast-pkts'] = interface_counters.out_multicast_pkts - interface['out-octets' ] = interface_counters.out_octets - interface['out-pkts' ] = interface_counters.out_pkts - interface['out-unicast-pkts' ] = interface_counters.out_unicast_pkts + entries = [] + for interface in json_data_valid['interfaces']['interface']: + LOGGER.info('interface={:s}'.format(str(interface))) + interface_name = interface['name'] + interface_counters = interface.get('state', {}).get('counters', {}) + _interface = { + 'name' : interface_name, + 'in-broadcast-pkts' : interface_counters['in_broadcast_pkts' ], + 'in-discards' : interface_counters['in_discards' ], + 'in-errors' : interface_counters['in_errors' ], + 'in-fcs-errors' : interface_counters['in_fcs_errors' ], + 'in-multicast-pkts' : interface_counters['in_multicast_pkts' ], + 'in-octets' : interface_counters['in_octets' ], + 'in-pkts' : interface_counters['in_pkts' ], + 'in-unicast-pkts' : interface_counters['in_unicast_pkts' ], + 'out-broadcast-pkts': interface_counters['out_broadcast_pkts'], + 'out-discards' : interface_counters['out_discards' ], + 'out-errors' : interface_counters['out_errors' ], + 'out-multicast-pkts': interface_counters['out_multicast_pkts'], + 'out-octets' : interface_counters['out_octets' ], + 'out-pkts' : interface_counters['out_pkts' ], + 'out-unicast-pkts' : interface_counters['out_unicast_pkts' ], + } LOGGER.info('interface = {:s}'.format(str(interface))) - if len(interface) == 0: continue - counters.append(('/interface[{:s}]'.format(interface['name']), interface)) + entry_interface_key = '/interface[{:s}]'.format(interface_name) + entries.append((entry_interface_key, _interface)) - return counters + return entries diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 0b4d15745..1efed024c 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, libyang, logging -import operator +import json, libyang, logging, operator from typing import Any, Dict, List, Tuple from ._Handler import _Handler -from .Tools import get_bool, get_int, get_str +from .Tools import get_str from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) @@ -45,7 +44,7 @@ class NetworkInstanceHandler(_Handler): def compose( self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False ) -> Tuple[str, str]: - ni_name = get_str(resource_value, 'name') # test-svc + ni_name = get_str(resource_value, 'name') # test-svc if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]' @@ -56,21 +55,32 @@ class NetworkInstanceHandler(_Handler): ni_type = get_str(resource_value, 'type') # L3VRF / L2VSI / ... ni_type = MAP_NETWORK_INSTANCE_TYPE.get(ni_type, ni_type) - # 'DIRECTLY_CONNECTED' is implicitly added + str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) + #str_data = json.dumps({ + # 'name': ni_name, + # 'config': {'name': ni_name, 'type': ni_type}, + #}) + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni_path = 'network-instance[name="{:s}"]'.format(ni_name) + yang_ni : libyang.DContainer = yang_nis.create_path(yang_ni_path) + yang_ni.create_path('config/name', ni_name) + yang_ni.create_path('config/type', ni_type) - str_path = '/network-instances/network-instance[name={:s}]'.format(ni_name) - str_data = json.dumps({ - 'name': ni_name, - 'config': {'name': ni_name, 'type': ni_type}, - #'protocols': {'protocol': protocols}, - }) + # 'DIRECTLY_CONNECTED' is implicitly added + #'protocols': {'protocol': protocols}, + + str_data = yang_ni.print_mem('json') + LOGGER.warning('str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:network-instance'][0] + str_data = json.dumps(json_data) return str_path, str_data def parse( self, json_data : Dict, yang_handler : YangHandler ) -> List[Tuple[str, Dict[str, Any]]]: - LOGGER.debug('json_data = {:s}'.format(json.dumps(json_data))) + LOGGER.info('json_data = {:s}'.format(json.dumps(json_data))) # Arista Parsing Fixes: # - Default instance comes with mpls/signaling-protocols/rsvp-te/global/hellos/state/hello-interval set to 0 diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index 205373fca..ab105c2b0 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -12,21 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler +from .Tools import get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) +IS_CEOS = True + class NetworkInstanceInterfaceHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance/interface' - def get_path(self) -> str: return '/network-instances/network-instance/interfaces' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/interfaces' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name' ]) # test-svc - if_name = str(resource_value['if_name' ]) # ethernet-1/1 - sif_index = int(resource_value['sif_index']) # 0 - if_id = '{:s}.{:d}'.format(if_name, sif_index) + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 + sif_index = get_int(resource_value, 'sif_index', 0) # 0 + + if IS_CEOS: + if_id = if_name + else: + if_id = '{:s}.{:d}'.format(if_name, sif_index) if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' @@ -35,12 +45,30 @@ class NetworkInstanceInterfaceHandler(_Handler): return str_path, str_data str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) - str_data = json.dumps({ - 'id': if_id, - 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, - }) + #str_data = json.dumps({ + # 'id': if_id, + # 'config': {'id': if_id, 'interface': if_name, 'subinterface': sif_index}, + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_ifs : libyang.DContainer = yang_ni.create_path('interfaces') + yang_ni_if_path = 'interface[id="{:s}"]'.format(if_id) + yang_ni_if : libyang.DContainer = yang_ni_ifs.create_path(yang_ni_if_path) + yang_ni_if.create_path('config/id', if_id) + yang_ni_if.create_path('config/interface', if_name) + yang_ni_if.create_path('config/subinterface', sif_index) + + str_data = yang_ni_if.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:interface'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) response = [] return response diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 9d75e9ac6..0343e3cba 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -12,21 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging +import json, libyang, logging from typing import Any, Dict, List, Tuple from ._Handler import _Handler +from .Tools import get_int, get_str +from .YangHandler import YangHandler LOGGER = logging.getLogger(__name__) class NetworkInstanceStaticRouteHandler(_Handler): def get_resource_key(self) -> str: return '/network_instance/static_route' - def get_path(self) -> str: return '/network-instances/network-instance/static_route' + def get_path(self) -> str: return '/openconfig-network-instance:network-instances/network-instance/static_route' - def compose(self, resource_key : str, resource_value : Dict, delete : bool = False) -> Tuple[str, str]: - ni_name = str(resource_value['name' ]) # test-svc - prefix = str(resource_value['prefix' ]) # '172.0.1.0/24' + def compose( + self, resource_key : str, resource_value : Dict, yang_handler : YangHandler, delete : bool = False + ) -> Tuple[str, str]: + ni_name = get_str(resource_value, 'name' ) # test-svc + prefix = get_str(resource_value, 'prefix') # '172.0.1.0/24' - identifier = 'STATIC' + identifier = 'openconfig-policy-types:STATIC' name = 'static' if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols' @@ -35,27 +39,56 @@ class NetworkInstanceStaticRouteHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - next_hop = str(resource_value['next_hop' ]) # '172.0.0.1' - next_hop_index = int(resource_value.get('next_hop_index', 0)) # 0 + next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + next_hop_index = get_int(resource_value, 'next_hop_index', 0) # 0 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' str_path = PATH_TMPL.format(ni_name, identifier, name) - str_data = json.dumps({ - 'identifier': identifier, 'name': name, - 'config': {'identifier': identifier, 'name': name, 'enabled': True}, - 'static_routes': {'static': [{ - 'prefix': prefix, - 'config': {'prefix': prefix}, - 'next_hops': { - 'next-hop': [{ - 'index': next_hop_index, - 'config': {'index': next_hop_index, 'next_hop': next_hop} - }] - } - }]} - }) + #str_data = json.dumps({ + # 'identifier': identifier, 'name': name, + # 'config': {'identifier': identifier, 'name': name, 'enabled': True}, + # 'static_routes': {'static': [{ + # 'prefix': prefix, + # 'config': {'prefix': prefix}, + # 'next_hops': { + # 'next-hop': [{ + # 'index': next_hop_index, + # 'config': {'index': next_hop_index, 'next_hop': next_hop} + # }] + # } + # }]} + #}) + + yang_nis : libyang.DContainer = yang_handler.get_data_path('/openconfig-network-instance:network-instances') + yang_ni : libyang.DContainer = yang_nis.create_path('network-instance[name="{:s}"]'.format(ni_name)) + yang_ni_prs : libyang.DContainer = yang_ni.create_path('protocols') + yang_ni_pr_path = 'protocol[identifier="{:s}"][name="{:s}"]'.format(identifier, name) + yang_ni_pr : libyang.DContainer = yang_ni_prs.create_path(yang_ni_pr_path) + yang_ni_pr.create_path('config/identifier', identifier) + yang_ni_pr.create_path('config/name', name ) + yang_ni_pr.create_path('config/enabled', True ) + + yang_ni_pr_srs : libyang.DContainer = yang_ni_pr.create_path('static-routes') + yang_ni_pr_sr_path = 'static[prefix="{:s}"]'.format(prefix) + yang_ni_pr_sr : libyang.DContainer = yang_ni_pr_srs.create_path(yang_ni_pr_sr_path) + yang_ni_pr_sr.create_path('config/prefix', prefix) + + yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') + yang_ni_pr_sr_nh_path = 'next-hop[index="{:d}"]'.format(next_hop_index) + yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) + yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) + yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) + + str_data = yang_ni_pr.print_mem('json') + LOGGER.warning('[compose] str_data = {:s}'.format(str(str_data))) + json_data = json.loads(str_data) + json_data = json_data['openconfig-network-instance:protocol'][0] + str_data = json.dumps(json_data) return str_path, str_data - def parse(self, json_data : Dict) -> List[Tuple[str, Dict[str, Any]]]: + def parse( + self, json_data : Dict, yang_handler : YangHandler + ) -> List[Tuple[str, Dict[str, Any]]]: + LOGGER.warning('[parse] json_data = {:s}'.format(str(json_data))) response = [] return response diff --git a/src/device/tests/gnmi_openconfig/__init__.py b/src/device/tests/gnmi_openconfig/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/request_composers.py b/src/device/tests/gnmi_openconfig/request_composers.py new file mode 100644 index 000000000..faa8425c8 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/request_composers.py @@ -0,0 +1,44 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple + +def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: + str_path = '/interface[{:s}]'.format(if_name) + str_data = { + 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, + 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix + } + return str_path, str_data + +def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]'.format(ni_name) + str_data = { + 'name': ni_name, 'type': ni_type + } + return str_path, str_data + +def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0, metric=1) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) + str_data = { + 'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index, 'metric': metric + } + return str_path, str_data + +def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: + str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) + str_data = { + 'name': ni_name, 'if_name': if_name, 'sif_index': sif_index + } + return str_path, str_data diff --git a/src/device/tests/gnmi_openconfig/storage.py b/src/device/tests/gnmi_openconfig/storage.py new file mode 100644 index 000000000..4271b002f --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage.py @@ -0,0 +1,285 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest, re +from typing import Dict, List, Tuple + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield dict() + + +##### POPULATE INTERFACE STORAGE ####################################################################################### + +def populate_interfaces_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['name' ] = if_name + if_storage['type' ] = resource_value.get('type' ) + if_storage['admin-status' ] = resource_value.get('admin-status' ) + if_storage['oper-status' ] = resource_value.get('oper-status' ) + if_storage['ifindex' ] = resource_value.get('ifindex' ) + if_storage['mtu' ] = resource_value.get('mtu' ) + if_storage['management' ] = resource_value.get('management' ) + if_storage['hardware-port'] = resource_value.get('hardware-port') + if_storage['transceiver' ] = resource_value.get('transceiver' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) + if match is not None: + if_name = match.group(1) + if_storage = interfaces_storage.setdefault(if_name, dict()) + if_storage['port-speed' ] = resource_value.get('port-speed' ) + if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') + if_storage['mac-address' ] = resource_value.get('mac-address' ) + if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) + subif_storage['index'] = subif_index + continue + + match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) + if match is not None: + if_name = match.group(1) + subif_index = int(match.group(2)) + ipv4_addr = match.group(3) + ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) + ipv4_address_storage['ip' ] = ipv4_addr + ipv4_address_storage['origin'] = resource_value.get('origin') + ipv4_address_storage['prefix'] = resource_value.get('prefix') + continue + + +##### POPULATE NETWORK INSTANCE STORAGE ################################################################################ + +def populate_network_instances_storage( + storage : Dict, # pylint: disable=redefined-outer-name + resources : List[Tuple[str, Dict]], +) -> None: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + for resource_key, resource_value in resources: + match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + ni_storage = network_instances_storage.setdefault(name, dict()) + ni_storage['name'] = name + ni_storage['type'] = resource_value.get('type') + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) + ni_p_storage['id' ] = protocol + ni_p_storage['name'] = protocol + continue + + pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' + match = re.match(pattern, resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + prefix = match.group(3) + ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) + ni_p_s_storage['prefix' ] = prefix + ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + protocol = match.group(2) + address_family = match.group(3) + ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) + ni_t_storage['protocol' ] = protocol + ni_t_storage['address_family'] = address_family + continue + + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + name = match.group(1) + vlan_id = int(match.group(2)) + ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) + ni_v_storage['vlan_id'] = vlan_id + ni_v_storage['name' ] = resource_value.get('name') + ni_v_storage['members'] = sorted(resource_value.get('members')) + continue + + +##### GET EXPECTED INTERFACE CONFIG #################################################################################### + +INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]', [ + 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' + ]), + ('/interface[{if_name:s}]/ethernet', [ + 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' + ]), +] + +INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), +] + +INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), +] + +def get_expected_interface_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + interfaces_storage : Dict = storage.setdefault('interfaces', dict()) + subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) + ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) + + expected_interface_config = list() + for if_name, if_storage in interfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name) + resource_value = { + field_name : if_storage[field_name] + for field_name in resource_key_field_names + if field_name in if_storage and if_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) + resource_value = { + field_name : subif_storage[field_name] + for field_name in resource_key_field_names + if field_name in subif_storage and subif_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): + for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) + resource_value = { + field_name : ipv4_storage[field_name] + for field_name in resource_key_field_names + if field_name in ipv4_storage and ipv4_storage[field_name] is not None + } + expected_interface_config.append((resource_key, resource_value)) + + return expected_interface_config + + +##### GET EXPECTED NETWORK INSTANCE CONFIG ############################################################################# + +NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]', ['name', 'type']), +] + +NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), +] + +NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), +] + +NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), +] + +NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ + ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), +] + +def get_expected_network_instance_config( + storage : Dict, # pylint: disable=redefined-outer-name +) -> List[Tuple[str, Dict]]: + network_instances_storage : Dict = storage.setdefault('network_instances', dict()) + network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) + network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) + network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) + network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) + + expected_network_instance_config = list() + for ni_name, ni_storage in network_instances_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name) + resource_value = { + field_name : ni_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_storage and ni_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) + resource_value = { + field_name : ni_p_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_storage and ni_p_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) + resource_value = { + field_name : ni_p_s_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: + resource_key = resource_key_template.format( + ni_name=ni_name, protocol=protocol, address_family=address_family + ) + resource_value = { + field_name : ni_t_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_t_storage and ni_t_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): + for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: + resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) + resource_value = { + field_name : ni_v_storage[field_name] + for field_name in resource_key_field_names + if field_name in ni_v_storage and ni_v_storage[field_name] is not None + } + expected_network_instance_config.append((resource_key, resource_value)) + + return expected_network_instance_config diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py new file mode 100644 index 000000000..69b7a609a --- /dev/null +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -0,0 +1,556 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import deepdiff, logging, os, pytest, re, time +from typing import Dict +os.environ['DEVICE_EMULATED_ONLY'] = 'YES' + +# pylint: disable=wrong-import-position +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver + +from .request_composers import interface, network_instance, network_instance_interface, network_instance_static_route +from .storage import ( # pylint: disable=unused-import + storage, # be careful, order of symbols is important here!; storage should be the first one + get_expected_interface_config, get_expected_network_instance_config, populate_interfaces_storage, + populate_network_instances_storage +) + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) + + +##### DRIVER FIXTURE ################################################################################################### + +DRIVER_SETTING_ADDRESS = '172.20.20.101' +DRIVER_SETTING_PORT = 6030 +DRIVER_SETTING_USERNAME = 'admin' +DRIVER_SETTING_PASSWORD = 'admin' +DRIVER_SETTING_USE_TLS = False + +@pytest.fixture(scope='session') +def driver() -> GnmiOpenConfigDriver: + _driver = GnmiOpenConfigDriver( + DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, + username=DRIVER_SETTING_USERNAME, + password=DRIVER_SETTING_PASSWORD, + use_tls=DRIVER_SETTING_USE_TLS, + ) + _driver.Connect() + yield _driver + time.sleep(1) + _driver.Disconnect() + + +##### NETWORK INSTANCE DETAILS ######################################################################################### + +NI_NAME = 'test-l3-svc' +NI_TYPE = 'L3VRF' +NI_INTERFACES = [ + # interface_name, subinterface_index, ipv4 address, ipv4 prefix, enabled + ('Ethernet1', 0, '192.168.1.1', 24, True), + ('Ethernet10', 0, '192.168.10.1', 24, True), +] +NI_STATIC_ROUTES = [ + # prefix, gateway, metric + ('172.0.0.0/24', '172.16.0.2', 1), + ('172.2.0.0/24', '172.16.0.3', 1), +] + + +##### TEST METHODS ##################################################################################################### + +def test_get_endpoints( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_ENDPOINTS] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + expected_getconfig = [ + ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' + }}), + ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { + 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', + 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', + 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', + 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' + }}) + ] + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_get_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_interfaces_storage(storage, results_getconfig) + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_get_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + populate_network_instances_storage(storage, results_getconfig) + expected_getconfig = get_expected_network_instance_config(storage) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + raise Exception() + + +def test_set_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance(NI_NAME, NI_TYPE), + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + network_instances = sorted([NI_NAME]) + results = set(results_setconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]'.format(NI_NAME), { + 'name': NI_NAME, 'type': NI_TYPE + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + #for resource_key, resource_value in expected_getconfig: + # if resource_key == '/network_instance[default]/vlan[1]': + # resource_value['members'] = list() + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_set_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + interface(if_name, sif_index, ipv4_addr, ipv4_prefix, enabled) + for if_name, sif_index, ipv4_addr, ipv4_prefix, enabled in NI_INTERFACES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + interfaces = sorted([ + if_name + for if_name, _, _, _, _ in NI_INTERFACES + ]) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { + 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix + }) + for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES + ]) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_add_interfaces_to_network_instance( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance_interface(NI_NAME, if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_setconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + #expected_getconfig = get_expected_interface_config(storage) + #expected_getconfig.extend([ + # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + # }), + # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + # }) + #]) + + #permitted_retries = 5 + #while permitted_retries > 0: + # resources_to_get = [RESOURCE_INTERFACES] + # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + # results_getconfig = driver.GetConfig(resources_to_get) + # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + # + # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + # num_diffs = len(diff_data) + # if num_diffs == 0: break + # # let the device take some time to reconfigure + # time.sleep(0.5) + # permitted_retries -= 1 + + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_set_network_instance_static_routes( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_set = [ + network_instance_static_route(NI_NAME, prefix, gateway, metric=metric) + for prefix, gateway, metric in NI_STATIC_ROUTES + ] + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + + + + + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_setconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + #expected_getconfig = get_expected_interface_config(storage) + #expected_getconfig.extend([ + # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + # }), + # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + # }) + #]) + + #permitted_retries = 5 + #while permitted_retries > 0: + # resources_to_get = [RESOURCE_INTERFACES] + # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + # results_getconfig = driver.GetConfig(resources_to_get) + # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + # + # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + # num_diffs = len(diff_data) + # if num_diffs == 0: break + # # let the device take some time to reconfigure + # time.sleep(0.5) + # permitted_retries -= 1 + # + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_del_network_instance_static_routes( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance_static_route(NI_NAME, '172.0.0.0/24', '172.16.0.2'), + network_instance_static_route(NI_NAME, '172.2.0.0/24', '172.16.0.3'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + #interfaces = sorted(['Ethernet1', 'Ethernet10']) + #results = set(results_deleteconfig) + #assert len(results) == len(interfaces) + #for if_name in interfaces: + # assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + #expected_getconfig = get_expected_interface_config(storage) + + #diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + #num_diffs = len(diff_data) + #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + #assert num_diffs == 0 + raise Exception() + + +def test_del_interfaces_from_network_instance( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance_interface(NI_NAME, ni_if_name, ni_sif_index) + for ni_if_name, ni_sif_index in NI_INTERFACES + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interface_ids = sorted([ + '{:s}.{:d}'.format(ni_if_name, ni_sif_index) + for ni_if_name, ni_sif_index in NI_INTERFACES + ]) + results = set(results_deleteconfig) + assert len(results) == len(interface_ids) + for interface_id in interface_ids: + assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, interface_id), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { + 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 + }), + ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { + 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 + }) + ]) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + raise Exception() + + +def test_del_interfaces( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + interface('Ethernet1', 0, '192.168.1.1', 24, True), + interface('Ethernet10', 0, '192.168.10.1', 24, True), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + interfaces = sorted(['Ethernet1', 'Ethernet10']) + results = set(results_deleteconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/interface[{:s}]'.format(if_name), True) in results + + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + expected_getconfig = get_expected_interface_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + +def test_del_network_instances( + driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Dict, # pylint: disable=redefined-outer-name +) -> None: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + resources_to_delete = [ + network_instance(NI_NAME, 'L3VRF'), + ] + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + + network_instances = sorted([NI_NAME]) + results = set(results_deleteconfig) + assert len(results) == len(network_instances) + for ni_name in network_instances: + assert ('/network_instance[{:s}]'.format(ni_name), True) in results + + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + expected_getconfig = get_expected_network_instance_config(storage) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 diff --git a/src/device/tests/test_unitary_gnmi_openconfig.py b/src/device/tests/test_unitary_gnmi_openconfig.py deleted file mode 100644 index 7d33d1a71..000000000 --- a/src/device/tests/test_unitary_gnmi_openconfig.py +++ /dev/null @@ -1,637 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import deepdiff, logging, os, pytest, re, time -from typing import Dict, List, Tuple -os.environ['DEVICE_EMULATED_ONLY'] = 'YES' - -# pylint: disable=wrong-import-position -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) -from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver - -logging.basicConfig(level=logging.DEBUG) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) - -DRIVER_SETTING_ADDRESS = '172.20.20.101' -DRIVER_SETTING_PORT = 6030 -DRIVER_SETTING_USERNAME = 'admin' -DRIVER_SETTING_PASSWORD = 'admin' -DRIVER_SETTING_USE_TLS = False - -@pytest.fixture(scope='session') -def driver() -> GnmiOpenConfigDriver: - _driver = GnmiOpenConfigDriver( - DRIVER_SETTING_ADDRESS, DRIVER_SETTING_PORT, - username=DRIVER_SETTING_USERNAME, - password=DRIVER_SETTING_PASSWORD, - use_tls=DRIVER_SETTING_USE_TLS, - ) - _driver.Connect() - yield _driver - time.sleep(1) - _driver.Disconnect() - -@pytest.fixture(scope='session') -def storage() -> Dict: - yield dict() - - -##### STORAGE POPULATORS ############################################################################################### - -def populate_interfaces_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['name' ] = if_name - if_storage['type' ] = resource_value.get('type' ) - if_storage['admin-status' ] = resource_value.get('admin-status' ) - if_storage['oper-status' ] = resource_value.get('oper-status' ) - if_storage['ifindex' ] = resource_value.get('ifindex' ) - if_storage['mtu' ] = resource_value.get('mtu' ) - if_storage['management' ] = resource_value.get('management' ) - if_storage['hardware-port'] = resource_value.get('hardware-port') - if_storage['transceiver' ] = resource_value.get('transceiver' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['port-speed' ] = resource_value.get('port-speed' ) - if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') - if_storage['mac-address' ] = resource_value.get('mac-address' ) - if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) - subif_storage['index'] = subif_index - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - ipv4_addr = match.group(3) - ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) - ipv4_address_storage['ip' ] = ipv4_addr - ipv4_address_storage['origin'] = resource_value.get('origin') - ipv4_address_storage['prefix'] = resource_value.get('prefix') - continue - -def populate_network_instances_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - ni_storage = network_instances_storage.setdefault(name, dict()) - ni_storage['name'] = name - ni_storage['type'] = resource_value.get('type') - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) - ni_p_storage['id' ] = protocol - ni_p_storage['name'] = protocol - continue - - pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' - match = re.match(pattern, resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - prefix = match.group(3) - ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) - ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - address_family = match.group(3) - ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) - ni_t_storage['protocol' ] = protocol - ni_t_storage['address_family'] = address_family - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - vlan_id = int(match.group(2)) - ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) - ni_v_storage['vlan_id'] = vlan_id - ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = sorted(resource_value.get('members')) - continue - - -##### EXPECTED CONFIG COMPOSERS ######################################################################################## - -INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]', [ - 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' - ]), - ('/interface[{if_name:s}]/ethernet', [ - 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' - ]), -] - -INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), -] - -INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), -] - -def get_expected_interface_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - expected_interface_config = list() - for if_name, if_storage in interfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name) - resource_value = { - field_name : if_storage[field_name] - for field_name in resource_key_field_names - if field_name in if_storage and if_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) - resource_value = { - field_name : subif_storage[field_name] - for field_name in resource_key_field_names - if field_name in subif_storage and subif_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) - resource_value = { - field_name : ipv4_storage[field_name] - for field_name in resource_key_field_names - if field_name in ipv4_storage and ipv4_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - return expected_interface_config - -NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]', ['name', 'type']), -] - -NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), -] - -NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), -] - -NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), -] - -NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), -] - -def get_expected_network_instance_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - expected_network_instance_config = list() - for ni_name, ni_storage in network_instances_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name) - resource_value = { - field_name : ni_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_storage and ni_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) - resource_value = { - field_name : ni_p_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_storage and ni_p_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) - resource_value = { - field_name : ni_p_s_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format( - ni_name=ni_name, protocol=protocol, address_family=address_family - ) - resource_value = { - field_name : ni_t_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_t_storage and ni_t_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) - resource_value = { - field_name : ni_v_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_v_storage and ni_v_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - return expected_network_instance_config - - -##### REQUEST COMPOSERS ################################################################################################ - -def interface(if_name, sif_index, ipv4_address, ipv4_prefix, enabled) -> Tuple[str, Dict]: - str_path = '/interface[{:s}]'.format(if_name) - str_data = { - 'name': if_name, 'enabled': enabled, 'sub_if_index': sif_index, 'sub_if_enabled': enabled, - 'sub_if_ipv4_enabled': enabled, 'sub_if_ipv4_address': ipv4_address, 'sub_if_ipv4_prefix': ipv4_prefix - } - return str_path, str_data - -def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]'.format(ni_name) - str_data = {'name': ni_name, 'type': ni_type} - return str_path, str_data - -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) - str_data = {'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index} - return str_path, str_data - -def network_instance_interface(ni_name, if_name, sif_index) -> Tuple[str, Dict]: - str_path = '/network_instance[{:s}]/interface[{:s}.{:d}]'.format(ni_name, if_name, sif_index) - str_data = {'name': ni_name, 'if_name': if_name, 'sif_index': sif_index} - return str_path, str_data - -def test_get_endpoints( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_ENDPOINTS] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - expected_getconfig = [ - ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' - }}), - ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' - }}) - ] - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_get_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_interfaces_storage(storage, results_getconfig) - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_get_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_network_instances_storage(storage, results_getconfig) - expected_getconfig = get_expected_network_instance_config(storage) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_set_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - }), - ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - }) - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_set_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance('test-l3-svc', 'L3VRF'), - ] - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - - network_instances = sorted(['test-l3-svc']) - results = set(results_setconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[test-l3-svc]', { - 'name': 'test-l3-svc', 'type': 'L3VRF' - }), - ('/network_instance[test-l3-svc]/protocol[DIRECTLY_CONNECTED]', { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV4]', { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[test-l3-svc]/table[DIRECTLY_CONNECTED,IPV6]', { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - for resource_key, resource_value in expected_getconfig: - if resource_key == '/network_instance[default]/vlan[1]': - resource_value['members'] = list() - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_del_interfaces( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_deleteconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - -def test_del_network_instances( - driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name -) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance('test-l3-svc', 'L3VRF'), - ] - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - - network_instances = sorted(['test-l3-svc']) - results = set(results_deleteconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - -#def test_unitary_gnmi_openconfig( -# driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name -#) -> None: -# #resources_to_get = [] -# resources_to_get = [RESOURCE_ENDPOINTS] -# #resources_to_get = [RESOURCE_INTERFACES] -# #resources_to_get = [RESOURCE_NETWORK_INSTANCES] -# #resources_to_get = [RESOURCE_ROUTING_POLICIES] -# #resources_to_get = [RESOURCE_SERVICES] -# LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) -# results_getconfig = driver.GetConfig(resources_to_get) -# LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) -# -# #resources_to_set = [ -# # network_instance('test-svc', 'L3VRF'), -# # -# # interface('ethernet-1/1', 0, '172.16.0.1', 24, True), -# # network_instance_interface('test-svc', 'ethernet-1/1', 0), -# # -# # interface('ethernet-1/2', 0, '172.0.0.1', 24, True), -# # network_instance_interface('test-svc', 'ethernet-1/2', 0), -# # -# # network_instance_static_route('test-svc', '172.0.0.0/24', '172.16.0.2'), -# # network_instance_static_route('test-svc', '172.2.0.0/24', '172.16.0.3'), -# #] -# #LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) -# #results_setconfig = driver.SetConfig(resources_to_set) -# #LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) -# -# #resources_to_delete = [ -# # #network_instance_static_route('d35fc1d9', '172.0.0.0/24', '172.16.0.2'), -# # #network_instance_static_route('d35fc1d9', '172.2.0.0/24', '172.16.0.3'), -# # -# # #network_instance_interface('d35fc1d9', 'ethernet-1/1', 0), -# # #network_instance_interface('d35fc1d9', 'ethernet-1/2', 0), -# # -# # #interface('ethernet-1/1', 0, '172.16.1.1', 24, True), -# # #interface('ethernet-1/2', 0, '172.0.0.2', 24, True), -# # -# # #network_instance('20f66fb5', 'L3VRF'), -# #] -# #LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) -# #results_deleteconfig = driver.DeleteConfig(resources_to_delete) -# #LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) -- GitLab From 74a75434c591e0a4a74d94c64ea61600b2cdc3e2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 9 Jan 2024 17:04:35 +0000 Subject: [PATCH 016/941] Device component - GNMI OpenConfig: - Corrected management of network instance interfaces - Improved unitary tests --- .../handlers/NetworkInstance.py | 12 ++ .../handlers/NetworkInstanceInterface.py | 8 +- .../test_unitary_gnmi_openconfig.py | 178 +++++++++++------- 3 files changed, 125 insertions(+), 73 deletions(-) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index 1efed024c..b97612987 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -108,6 +108,18 @@ class NetworkInstanceHandler(_Handler): entry_net_inst_key = '/network_instance[{:s}]'.format(ni_name) entries.append((entry_net_inst_key, _net_inst)) + ni_interfaces = network_instance.get('interfaces', {}).get('interface', []) + for ni_interface in ni_interfaces: + #ni_if_id = ni_interface['id'] + ni_if_config = ni_interface['config'] + ni_if_name = ni_if_config['interface'] + ni_sif_index = ni_if_config['subinterface'] + ni_if_id = '{:s}.{:d}'.format(ni_if_name, ni_sif_index) + + _interface = {'name': ni_name, 'id': ni_if_id, 'if_name': ni_if_name, 'sif_index': ni_sif_index} + entry_interface_key = '{:s}/interface[{:s}]'.format(entry_net_inst_key, ni_if_id) + entries.append((entry_interface_key, _interface)) + ni_protocols = network_instance.get('protocols', {}).get('protocol', []) for ni_protocol in ni_protocols: ni_protocol_id = ni_protocol['identifier'].split(':')[-1] diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py index ab105c2b0..af2178fe9 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceInterface.py @@ -32,11 +32,7 @@ class NetworkInstanceInterfaceHandler(_Handler): ni_name = get_str(resource_value, 'name' ) # test-svc if_name = get_str(resource_value, 'if_name' ) # ethernet-1/1 sif_index = get_int(resource_value, 'sif_index', 0) # 0 - - if IS_CEOS: - if_id = if_name - else: - if_id = '{:s}.{:d}'.format(if_name, sif_index) + if_id = '{:s}.{:d}'.format(if_name, sif_index) if delete: PATH_TMPL = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]' @@ -44,6 +40,8 @@ class NetworkInstanceInterfaceHandler(_Handler): str_data = json.dumps({}) return str_path, str_data + if IS_CEOS: if_id = if_name + str_path = '/network-instances/network-instance[name={:s}]/interfaces/interface[id={:s}]'.format(ni_name, if_id) #str_data = json.dumps({ # 'id': if_id, diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 69b7a609a..47c8e1cdb 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -141,7 +141,6 @@ def test_get_network_instances( num_diffs = len(diff_data) if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) assert num_diffs == 0 - raise Exception() def test_set_network_instances( @@ -284,39 +283,80 @@ def test_add_interfaces_to_network_instance( results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_setconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results + interfaces = sorted([ + '{:s}.{:d}'.format(if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES + ]) + results = set(results_setconfig) + assert len(results) == len(interfaces) + for if_name in interfaces: + assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, if_name), True) in results - #expected_getconfig = get_expected_interface_config(storage) - #expected_getconfig.extend([ - # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - # }), - # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - # }) - #]) - - #permitted_retries = 5 - #while permitted_retries > 0: - # resources_to_get = [RESOURCE_INTERFACES] - # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - # results_getconfig = driver.GetConfig(resources_to_get) - # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - # - # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - # num_diffs = len(diff_data) - # if num_diffs == 0: break - # # let the device take some time to reconfigure - # time.sleep(0.5) - # permitted_retries -= 1 + expected_getconfig = get_expected_interface_config(storage) + expected_getconfig.extend([ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { + 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix + }) + for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_INTERFACES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]'.format(NI_NAME), { + 'name': NI_NAME, 'type': NI_TYPE + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) + + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 def test_set_network_instance_static_routes( @@ -336,43 +376,45 @@ def test_set_network_instance_static_routes( results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + prefixes = sorted([ + prefix + for prefix, _, _ in NI_STATIC_ROUTES + ]) + results = set(results_setconfig) + assert len(results) == len(prefixes) + for prefix in prefixes: + assert ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), True) in results + expected_getconfig = get_expected_network_instance_config(storage) + expected_getconfig.extend([ + ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), { + 'name': NI_NAME, 'prefix': prefix, 'next_hop': gateway, 'next_hop_index': 0, 'metric': metric + }) + for prefix, gateway, metric in NI_STATIC_ROUTES + ]) + permitted_retries = 5 + while permitted_retries > 0: + resources_to_get = [RESOURCE_NETWORK_INSTANCES] + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + for resource_key, resource_value in results_getconfig: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is None: continue + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_setconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results + diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(0.5) + permitted_retries -= 1 - #expected_getconfig = get_expected_interface_config(storage) - #expected_getconfig.extend([ - # ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - # 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - # }), - # ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - # 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - # }) - #]) - - #permitted_retries = 5 - #while permitted_retries > 0: - # resources_to_get = [RESOURCE_INTERFACES] - # LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - # results_getconfig = driver.GetConfig(resources_to_get) - # LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - # - # diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - # num_diffs = len(diff_data) - # if num_diffs == 0: break - # # let the device take some time to reconfigure - # time.sleep(0.5) - # permitted_retries -= 1 - # - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 def test_del_network_instance_static_routes( @@ -429,16 +471,16 @@ def test_del_interfaces_from_network_instance( LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) resources_to_delete = [ - network_instance_interface(NI_NAME, ni_if_name, ni_sif_index) - for ni_if_name, ni_sif_index in NI_INTERFACES + network_instance_interface(NI_NAME, if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) interface_ids = sorted([ - '{:s}.{:d}'.format(ni_if_name, ni_sif_index) - for ni_if_name, ni_sif_index in NI_INTERFACES + '{:s}.{:d}'.format(if_name, sif_index) + for if_name, sif_index, _, _, _ in NI_INTERFACES ]) results = set(results_deleteconfig) assert len(results) == len(interface_ids) -- GitLab From 03f748da04512622f43bb3c0551737434a25ec72 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Wed, 10 Jan 2024 18:36:50 +0000 Subject: [PATCH 017/941] Device component - GNMI OpenConfig: - Updated unitary tests --- src/device/tests/gnmi_openconfig/storage.py | 285 -------- .../tests/gnmi_openconfig/storage/Storage.py | 23 + .../storage/StorageEndpoints.py | 72 ++ .../storage/StorageInterface.py | 122 ++++ .../storage/StorageNetworkInstance.py | 194 ++++++ .../tests/gnmi_openconfig/storage/Tools.py | 32 + .../tests/gnmi_openconfig/storage/__init__.py | 14 + .../test_unitary_gnmi_openconfig.py | 656 ++++++------------ .../tests/gnmi_openconfig/tools/__init__.py | 14 + .../gnmi_openconfig/tools/check_config.py | 82 +++ .../gnmi_openconfig/tools/check_updates.py | 21 + .../tools/expected_config_composers.py | 58 ++ .../{ => tools}/request_composers.py | 0 .../tools/result_config_adapters.py | 29 + 14 files changed, 864 insertions(+), 738 deletions(-) delete mode 100644 src/device/tests/gnmi_openconfig/storage.py create mode 100644 src/device/tests/gnmi_openconfig/storage/Storage.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageInterface.py create mode 100644 src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py create mode 100644 src/device/tests/gnmi_openconfig/storage/Tools.py create mode 100644 src/device/tests/gnmi_openconfig/storage/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/tools/__init__.py create mode 100644 src/device/tests/gnmi_openconfig/tools/check_config.py create mode 100644 src/device/tests/gnmi_openconfig/tools/check_updates.py create mode 100644 src/device/tests/gnmi_openconfig/tools/expected_config_composers.py rename src/device/tests/gnmi_openconfig/{ => tools}/request_composers.py (100%) create mode 100644 src/device/tests/gnmi_openconfig/tools/result_config_adapters.py diff --git a/src/device/tests/gnmi_openconfig/storage.py b/src/device/tests/gnmi_openconfig/storage.py deleted file mode 100644 index 4271b002f..000000000 --- a/src/device/tests/gnmi_openconfig/storage.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest, re -from typing import Dict, List, Tuple - -@pytest.fixture(scope='session') -def storage() -> Dict: - yield dict() - - -##### POPULATE INTERFACE STORAGE ####################################################################################### - -def populate_interfaces_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/interface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['name' ] = if_name - if_storage['type' ] = resource_value.get('type' ) - if_storage['admin-status' ] = resource_value.get('admin-status' ) - if_storage['oper-status' ] = resource_value.get('oper-status' ) - if_storage['ifindex' ] = resource_value.get('ifindex' ) - if_storage['mtu' ] = resource_value.get('mtu' ) - if_storage['management' ] = resource_value.get('management' ) - if_storage['hardware-port'] = resource_value.get('hardware-port') - if_storage['transceiver' ] = resource_value.get('transceiver' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/ethernet$', resource_key) - if match is not None: - if_name = match.group(1) - if_storage = interfaces_storage.setdefault(if_name, dict()) - if_storage['port-speed' ] = resource_value.get('port-speed' ) - if_storage['negotiated-port-speed'] = resource_value.get('negotiated-port-speed') - if_storage['mac-address' ] = resource_value.get('mac-address' ) - if_storage['hw-mac-address' ] = resource_value.get('hw-mac-address' ) - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - subif_storage = subinterfaces_storage.setdefault((if_name, subif_index), dict()) - subif_storage['index'] = subif_index - continue - - match = re.match(r'^\/interface\[([^\]]+)\]\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$', resource_key) - if match is not None: - if_name = match.group(1) - subif_index = int(match.group(2)) - ipv4_addr = match.group(3) - ipv4_address_storage = ipv4_addresses_storage.setdefault((if_name, subif_index, ipv4_addr), dict()) - ipv4_address_storage['ip' ] = ipv4_addr - ipv4_address_storage['origin'] = resource_value.get('origin') - ipv4_address_storage['prefix'] = resource_value.get('prefix') - continue - - -##### POPULATE NETWORK INSTANCE STORAGE ################################################################################ - -def populate_network_instances_storage( - storage : Dict, # pylint: disable=redefined-outer-name - resources : List[Tuple[str, Dict]], -) -> None: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - for resource_key, resource_value in resources: - match = re.match(r'^\/network\_instance\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - ni_storage = network_instances_storage.setdefault(name, dict()) - ni_storage['name'] = name - ni_storage['type'] = resource_value.get('type') - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - ni_p_storage = network_instance_protocols_storage.setdefault((name, protocol), dict()) - ni_p_storage['id' ] = protocol - ni_p_storage['name'] = protocol - continue - - pattern = r'^\/network\_instance\[([^\]]+)\]\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$' - match = re.match(pattern, resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - prefix = match.group(3) - ni_p_s_storage = network_instance_protocol_static_storage.setdefault((name, protocol, prefix), dict()) - ni_p_s_storage['prefix' ] = prefix - ni_p_s_storage['next_hops'] = sorted(resource_value.get('next_hops')) - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/table\[([^\,]+)\,([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - protocol = match.group(2) - address_family = match.group(3) - ni_t_storage = network_instance_tables_storage.setdefault((name, protocol, address_family), dict()) - ni_t_storage['protocol' ] = protocol - ni_t_storage['address_family'] = address_family - continue - - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is not None: - name = match.group(1) - vlan_id = int(match.group(2)) - ni_v_storage = network_instance_vlans_storage.setdefault((name, vlan_id), dict()) - ni_v_storage['vlan_id'] = vlan_id - ni_v_storage['name' ] = resource_value.get('name') - ni_v_storage['members'] = sorted(resource_value.get('members')) - continue - - -##### GET EXPECTED INTERFACE CONFIG #################################################################################### - -INTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]', [ - 'name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', 'hardware-port', 'transceiver' - ]), - ('/interface[{if_name:s}]/ethernet', [ - 'port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address' - ]), -] - -INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]', ['index']), -] - -INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/interface[{if_name:s}]/subinterface[{subif_index:d}]/ipv4[{ipv4_addr:s}]', ['ip', 'origin', 'prefix']), -] - -def get_expected_interface_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - interfaces_storage : Dict = storage.setdefault('interfaces', dict()) - subinterfaces_storage : Dict = storage.setdefault('interface_subinterfaces', dict()) - ipv4_addresses_storage : Dict = storage.setdefault('interface_subinterface_ipv4_addresses', dict()) - - expected_interface_config = list() - for if_name, if_storage in interfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name) - resource_value = { - field_name : if_storage[field_name] - for field_name in resource_key_field_names - if field_name in if_storage and if_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index), subif_storage in subinterfaces_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index) - resource_value = { - field_name : subif_storage[field_name] - for field_name in resource_key_field_names - if field_name in subif_storage and subif_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - for (if_name, subif_index, ipv4_addr), ipv4_storage in ipv4_addresses_storage.items(): - for resource_key_template, resource_key_field_names in INTERFACE_SUBINTERFACE_IPV4_ADDRESS_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(if_name=if_name, subif_index=subif_index, ipv4_addr=ipv4_addr) - resource_value = { - field_name : ipv4_storage[field_name] - for field_name in resource_key_field_names - if field_name in ipv4_storage and ipv4_storage[field_name] is not None - } - expected_interface_config.append((resource_key, resource_value)) - - return expected_interface_config - - -##### GET EXPECTED NETWORK INSTANCE CONFIG ############################################################################# - -NETWORK_INSTANCE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]', ['name', 'type']), -] - -NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]', ['id', 'name']), -] - -NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/protocol[{protocol:s}]/static_routes[{prefix:s}]', ['prefix', 'next_hops']), -] - -NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/table[{protocol:s},{address_family:s}]', ['protocol', 'address_family']), -] - -NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE : List[Tuple[str, List[str]]] = [ - ('/network_instance[{ni_name:s}]/vlan[{vlan_id:d}]', ['vlan_id', 'name', 'members']), -] - -def get_expected_network_instance_config( - storage : Dict, # pylint: disable=redefined-outer-name -) -> List[Tuple[str, Dict]]: - network_instances_storage : Dict = storage.setdefault('network_instances', dict()) - network_instance_protocols_storage : Dict = storage.setdefault('network_instance_protocols', dict()) - network_instance_protocol_static_storage : Dict = storage.setdefault('network_instance_protocol_static', dict()) - network_instance_tables_storage : Dict = storage.setdefault('network_instance_tables', dict()) - network_instance_vlans_storage : Dict = storage.setdefault('network_instance_vlans', dict()) - - expected_network_instance_config = list() - for ni_name, ni_storage in network_instances_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name) - resource_value = { - field_name : ni_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_storage and ni_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol), ni_p_storage in network_instance_protocols_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol) - resource_value = { - field_name : ni_p_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_storage and ni_p_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, prefix), ni_p_s_storage in network_instance_protocol_static_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_PROTOCOL_STATIC_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, protocol=protocol, prefix=prefix) - resource_value = { - field_name : ni_p_s_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_p_s_storage and ni_p_s_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, protocol, address_family), ni_t_storage in network_instance_tables_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_TABLE_CONFIG_STRUCTURE: - resource_key = resource_key_template.format( - ni_name=ni_name, protocol=protocol, address_family=address_family - ) - resource_value = { - field_name : ni_t_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_t_storage and ni_t_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - for (ni_name, vlan_id), ni_v_storage in network_instance_vlans_storage.items(): - for resource_key_template, resource_key_field_names in NETWORK_INSTANCE_VLAN_CONFIG_STRUCTURE: - resource_key = resource_key_template.format(ni_name=ni_name, vlan_id=vlan_id) - resource_value = { - field_name : ni_v_storage[field_name] - for field_name in resource_key_field_names - if field_name in ni_v_storage and ni_v_storage[field_name] is not None - } - expected_network_instance_config.append((resource_key, resource_value)) - - return expected_network_instance_config diff --git a/src/device/tests/gnmi_openconfig/storage/Storage.py b/src/device/tests/gnmi_openconfig/storage/Storage.py new file mode 100644 index 000000000..4aaf29c99 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/Storage.py @@ -0,0 +1,23 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .StorageEndpoints import StorageEndpoints +from .StorageInterface import StorageInterface +from .StorageNetworkInstance import StorageNetworkInstance + +class Storage: + def __init__(self) -> None: + self.endpoints = StorageEndpoints() + self.interfaces = StorageInterface() + self.network_instances = StorageNetworkInstance() diff --git a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py new file mode 100644 index 000000000..815a1b0ad --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py @@ -0,0 +1,72 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +RE_RESKEY_ENDPOINT = re.compile(r'^\/endpoints\/endpoint\[([^\]]+)\]$') + +ENDPOINT_PACKET_SAMPLE_TYPES : Dict[int, str] = { + 101: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/out-pkts', + 102: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/in-pkts', + 201: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/out-octets', + 202: '/openconfig-interfaces:interfaces/interface[name={:s}]/state/counters/in-octets', +} + +class Endpoints: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/endpoints/endpoint[{:s}]', ['uuid', 'type', 'sample_types']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, ep_uuid : str, resource_value : Dict) -> None: + item = self._items.setdefault(ep_uuid, dict()) + item['uuid'] = ep_uuid + + for _, field_names in Endpoints.STRUCT: + field_names = set(field_names) + item.update({k:v for k,v in resource_value if k in field_names}) + + item['sample_types'] = { + sample_type_id : sample_type_path.format(ep_uuid) + for sample_type_id, sample_type_path in ENDPOINT_PACKET_SAMPLE_TYPES.items() + } + + def remove(self, ep_uuid : str) -> None: + self._items.pop(ep_uuid, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Endpoints.STRUCT) + +class StorageEndpoints: + def __init__(self) -> None: + self.endpoints = Endpoints() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_ENDPOINT.match(resource_key) + if match is not None: + self.endpoints.add(match.group(1), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.endpoints.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py new file mode 100644 index 000000000..a0391e92f --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py @@ -0,0 +1,122 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +PREFIX = r'^\/interface\[([^\]]+)\]' +RE_RESKEY_INTERFACE = re.compile(PREFIX + r'$') +RE_RESKEY_ETHERNET = re.compile(PREFIX + r'\/ethernet$') +RE_RESKEY_SUBINTERFACE = re.compile(PREFIX + r'\/subinterface\[([^\]]+)\]$') +RE_RESKEY_IPV4_ADDRESS = re.compile(PREFIX + r'\/subinterface\[([^\]]+)\]\/ipv4\[([^\]]+)\]$') + +class Interfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]', ['name', 'type', 'admin-status', 'oper-status', 'management', 'mtu', 'ifindex', + 'hardware-port', 'transceiver']), + ('/interface[{:s}]/ethernet', ['port-speed', 'negotiated-port-speed', 'mac-address', 'hw-mac-address']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, if_name : str, resource_value : Dict) -> None: + item = self._items.setdefault(if_name, dict()) + item['name'] = if_name + for _, field_names in Interfaces.STRUCT: + field_names = set(field_names) + item.update({k:v for k,v in resource_value if k in field_names}) + + def remove(self, if_name : str) -> None: + self._items.pop(if_name, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Interfaces.STRUCT) + +class SubInterfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]/subinterface[{:d}]', ['index']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int], Dict] = dict() + + def add(self, if_name : str, subif_index : int) -> None: + item = self._items.setdefault((if_name, subif_index), dict()) + item['index'] = subif_index + + def remove(self, if_name : str, subif_index : int) -> None: + self._items.pop((if_name, subif_index), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, SubInterfaces.STRUCT) + +class IPv4Addresses: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]', ['ip', 'origin', 'prefix']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int, str], Dict] = dict() + + def add(self, if_name : str, subif_index : int, ipv4_address : str, resource_value : Dict) -> None: + item = self._items.setdefault((if_name, subif_index, ipv4_address), dict()) + item['ip' ] = ipv4_address + item['origin'] = resource_value.get('origin') + item['prefix'] = resource_value.get('prefix') + + def remove(self, if_name : str, subif_index : int, ipv4_address : str) -> None: + self._items.pop((if_name, subif_index, ipv4_address), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, IPv4Addresses.STRUCT) + +class StorageInterface: + def __init__(self) -> None: + self.interfaces = Interfaces() + self.subinterfaces = SubInterfaces() + self.ipv4_addresses = IPv4Addresses() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_INTERFACE.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_ETHERNET.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_SUBINTERFACE.match(resource_key) + if match is not None: + self.subinterfaces.add(match.group(1), int(match.group(2))) + continue + + match = RE_RESKEY_IPV4_ADDRESS.match(resource_key) + if match is not None: + self.ipv4_addresses.add(match.group(1), int(match.group(2)), match.group(3), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.interfaces.compose_resources()) + expected_config.extend(self.subinterfaces.compose_resources()) + expected_config.extend(self.ipv4_addresses.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py new file mode 100644 index 000000000..558cc032c --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -0,0 +1,194 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, List, Tuple +from .Tools import compose_resources + +PREFIX = r'^\/network\_instance\[([^\]]+)\]' +RE_RESKEY_NET_INST = re.compile(PREFIX + r'$') +RE_RESKEY_INTERFACE = re.compile(PREFIX + r'\/interface\[([^\]]+)\]$') +RE_RESKEY_PROTOCOL = re.compile(PREFIX + r'\/protocol\[([^\]]+)\]$') +RE_RESKEY_PROTO_STATIC = re.compile(PREFIX + r'\/protocol\[([^\]]+)\]\/static\_routes\[([^\]]+)\]$') +RE_RESKEY_TABLE = re.compile(PREFIX + r'\/table\[([^\,]+)\,([^\]]+)\]$') +RE_RESKEY_VLAN = re.compile(PREFIX + r'\/vlan\[([^\]]+)\]$') + +class NetworkInstances: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]', ['name', 'type']), + ] + + def __init__(self) -> None: + self._items : Dict[str, Dict] = dict() + + def add(self, ni_name : str, resource_value : Dict) -> None: + item = self._items.setdefault(ni_name, dict()) + item['name'] = ni_name + item['type'] = resource_value.get('type') + + def remove(self, ni_name : str) -> None: + self._items.pop(ni_name, None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, NetworkInstances.STRUCT) + +class Interfaces: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/interface[{:s}]', ['ni_name', 'if_name']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str], Dict] = dict() + + def add(self, ni_name : str, if_name : str) -> None: + item = self._items.setdefault((ni_name, if_name), dict()) + item['ni_name'] = ni_name + item['if_name'] = if_name + + def remove(self, ni_name : str, if_name : str) -> None: + self._items.pop((ni_name, if_name), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Interfaces.STRUCT) + +class Protocols: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/protocol[{:s}]', ['id', 'name']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str) -> None: + item = self._items.setdefault((ni_name, protocol), dict()) + item['id' ] = protocol + item['name'] = protocol + + def remove(self, ni_name : str, protocol : str) -> None: + self._items.pop((ni_name, protocol), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Protocols.STRUCT) + +class StaticRoutes: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str, prefix : str, resource_value : Dict) -> None: + item = self._items.setdefault((ni_name, protocol, prefix), dict()) + item['prefix' ] = prefix + item['next_hops'] = sorted(resource_value.get('next_hops')) + + def remove(self, ni_name : str, protocol : str, prefix : str) -> None: + self._items.pop((ni_name, protocol, prefix), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, StaticRoutes.STRUCT) + +class Tables: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/table[{:s},{:s}]', ['protocol', 'address_family']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, str, str], Dict] = dict() + + def add(self, ni_name : str, protocol : str, address_family : str) -> None: + item = self._items.setdefault((ni_name, protocol, address_family), dict()) + item['protocol' ] = protocol + item['address_family'] = address_family + + def remove(self, ni_name : str, protocol : str, address_family : str) -> None: + self._items.pop((ni_name, protocol, address_family), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Tables.STRUCT) + +class Vlans: + STRUCT : List[Tuple[str, List[str]]] = [ + ('/network_instance[{:s}]/vlan[{:d}]', ['vlan_id', 'name', 'members']), + ] + + def __init__(self) -> None: + self._items : Dict[Tuple[str, int], Dict] = dict() + + def add(self, ni_name : str, vlan_id : int, resource_value : Dict) -> None: + item = self._items.setdefault((ni_name, vlan_id), dict()) + item['vlan_id'] = vlan_id + item['name' ] = resource_value.get('name') + item['members'] = sorted(resource_value.get('members')) + + def remove(self, ni_name : str, vlan_id : int) -> None: + self._items.pop((ni_name, vlan_id), None) + + def compose_resources(self) -> List[Dict]: + return compose_resources(self._items, Vlans.STRUCT) + +class StorageNetworkInstance: + def __init__(self) -> None: + self.network_instances = NetworkInstances() + self.interfaces = Interfaces() + self.protocols = Protocols() + self.protocol_static = StaticRoutes() + self.tables = Tables() + self.vlans = Vlans() + + def populate(self, resources : List[Tuple[str, Dict]]) -> None: + for resource_key, resource_value in resources: + match = RE_RESKEY_NET_INST.match(resource_key) + if match is not None: + self.network_instances.add(match.group(1), resource_value) + continue + + match = RE_RESKEY_INTERFACE.match(resource_key) + if match is not None: + self.interfaces.add(match.group(1), match.group(2)) + continue + + match = RE_RESKEY_PROTOCOL.match(resource_key) + if match is not None: + self.protocols.add(match.group(1), match.group(2)) + continue + + match = RE_RESKEY_PROTO_STATIC.match(resource_key) + if match is not None: + self.protocol_static.add(match.group(1), match.group(2), match.group(3), resource_value) + continue + + match = RE_RESKEY_TABLE.match(resource_key) + if match is not None: + self.tables.add(match.group(1), match.group(2), match.group(3)) + continue + + match = RE_RESKEY_VLAN.match(resource_key) + if match is not None: + self.vlans.add(match.group(1), int(match.group(2)), resource_value) + continue + + MSG = 'Unhandled Resource Key: {:s} => {:s}' + raise Exception(MSG.format(str(resource_key), str(resource_value))) + + def get_expected_config(self) -> List[Tuple[str, Dict]]: + expected_config = list() + expected_config.extend(self.network_instances.compose_resources()) + expected_config.extend(self.interfaces.compose_resources()) + expected_config.extend(self.protocols.compose_resources()) + expected_config.extend(self.protocol_static.compose_resources()) + expected_config.extend(self.tables.compose_resources()) + expected_config.extend(self.vlans.compose_resources()) + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/Tools.py b/src/device/tests/gnmi_openconfig/storage/Tools.py new file mode 100644 index 000000000..4da48af46 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/Tools.py @@ -0,0 +1,32 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Tuple + +def compose_resources( + storage : Dict[Tuple, Dict], config_struct : List[Tuple[str, List[str]]] +) -> List[Dict]: + expected_config = list() + + for resource_key_fields, resource_value_data in storage.items(): + for resource_key_template, resource_key_field_names in config_struct: + resource_key = resource_key_template.format(*resource_key_fields) + resource_value = { + field_name : resource_value_data[field_name] + for field_name in resource_key_field_names + if field_name in resource_value_data and resource_value_data[field_name] is not None + } + expected_config.append((resource_key, resource_value)) + + return expected_config diff --git a/src/device/tests/gnmi_openconfig/storage/__init__.py b/src/device/tests/gnmi_openconfig/storage/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/storage/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 47c8e1cdb..dd0561a2b 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -12,22 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import deepdiff, logging, os, pytest, re, time -from typing import Dict +import os os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position +import itertools, logging, pytest, time +from typing import Dict from device.service.driver_api._Driver import ( RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES ) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver - -from .request_composers import interface, network_instance, network_instance_interface, network_instance_static_route -from .storage import ( # pylint: disable=unused-import - storage, # be careful, order of symbols is important here!; storage should be the first one - get_expected_interface_config, get_expected_network_instance_config, populate_interfaces_storage, - populate_network_instances_storage +from .tools.check_config import check_config_endpoints, check_config_interfaces, check_config_network_instances +from .tools.check_updates import check_updates +from .tools.expected_config_composers import ( + compose_expected_config__interface, compose_expected_config__network_instance ) +from .tools.request_composers import ( + interface, network_instance, network_instance_interface, network_instance_static_route +) +from .storage.Storage import Storage logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -56,19 +59,40 @@ def driver() -> GnmiOpenConfigDriver: _driver.Disconnect() +##### STORAGE FIXTURE ################################################################################################## + +@pytest.fixture(scope='session') +def storage() -> Dict: + yield Storage() + + ##### NETWORK INSTANCE DETAILS ######################################################################################### -NI_NAME = 'test-l3-svc' -NI_TYPE = 'L3VRF' -NI_INTERFACES = [ - # interface_name, subinterface_index, ipv4 address, ipv4 prefix, enabled - ('Ethernet1', 0, '192.168.1.1', 24, True), - ('Ethernet10', 0, '192.168.10.1', 24, True), -] -NI_STATIC_ROUTES = [ - # prefix, gateway, metric - ('172.0.0.0/24', '172.16.0.2', 1), - ('172.2.0.0/24', '172.16.0.3', 1), +NETWORK_INSTANCES = [ + { + 'name': 'test-l3-svc', + 'type': 'L3VRF', + 'interfaces': [ + {'name': 'Ethernet1', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet10', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + ], + 'static_routes': [ + {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + ] + }, + { + 'name': 'test-l2-svc', + 'type': 'L2VSI', + 'interfaces': [ + {'name': 'Ethernet2', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet4', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + ], + 'static_routes': [ + {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + ] + } ] @@ -76,523 +100,249 @@ NI_STATIC_ROUTES = [ def test_get_endpoints( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_ENDPOINTS] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - expected_getconfig = [ - ('/endpoints/endpoint[ethernet1]', {'uuid': 'ethernet1', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet1]/state/counters/out-pkts' - }}), - ('/endpoints/endpoint[ethernet10]', {'uuid': 'ethernet10', 'type': '-', 'sample_types': { - 202: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-octets', - 201: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-octets', - 102: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/in-pkts', - 101: '/openconfig-interfaces:interfaces/interface[name=ethernet10]/state/counters/out-pkts' - }}) - ] - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_endpoints(driver, storage) + storage.endpoints.populate(results_getconfig) + check_config_endpoints(driver, storage) def test_get_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_interfaces_storage(storage, results_getconfig) - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_interfaces(driver, storage) + storage.interfaces.populate(results_getconfig) + check_config_interfaces(driver, storage) def test_get_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - populate_network_instances_storage(storage, results_getconfig) - expected_getconfig = get_expected_network_instance_config(storage) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + results_getconfig = check_config_network_instances(driver, storage) + storage.network_instances.populate(results_getconfig) + check_config_network_instances(driver, storage) def test_set_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance(NI_NAME, NI_TYPE), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + ni_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + ni_type = ni['type'] + resources_to_set.append(network_instance(ni_name, ni_type)) + ni_names.append(ni_name) + storage.network_instances.network_instances.add(ni_name, {'type': ni_type}) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]', ni_names) - network_instances = sorted([NI_NAME]) - results = set(results_setconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]'.format(NI_NAME), { - 'name': NI_NAME, 'type': NI_TYPE - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - #for resource_key, resource_value in expected_getconfig: - # if resource_key == '/network_instance[default]/vlan[1]': - # resource_value['members'] = list() - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_set_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - interface(if_name, sif_index, ipv4_addr, ipv4_prefix, enabled) - for if_name, sif_index, ipv4_addr, ipv4_prefix, enabled in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + ipv4_address = ni_if['ipv4_addr'] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled'] + resources_to_set.append(interface( + if_name, subif_index, ipv4_address, ipv4_prefix, enabled + )) + if_names.append(ni_name) + storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { + 'origin' : 'STATIC', 'prefix': ipv4_prefix + }) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/interface[{:s}]', if_names) - interfaces = sorted([ - if_name - for if_name, _, _, _, _ in NI_INTERFACES - ]) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { - 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix - }) - for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_add_interfaces_to_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance_interface(NI_NAME, if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + resources_to_set = list() + ni_if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.add(ni_name, if_name, subif_index) + LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - interfaces = sorted([ - '{:s}.{:d}'.format(if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ]) - results = set(results_setconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, if_name), True) in results - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[{:s}]/subinterface[{:d}]/ipv4[{:s}]'.format(if_name, sif_index, ipv4_addr), { - 'ip': ipv4_addr, 'origin': 'STATIC', 'prefix': ipv4_prefix - }) - for if_name, sif_index, ipv4_addr, ipv4_prefix, _ in NI_INTERFACES - ]) - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]'.format(NI_NAME), { - 'name': NI_NAME, 'type': NI_TYPE - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(NI_NAME), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(NI_NAME), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - LOGGER.info('expected_getconfig = {:s}'.format(str(sorted(expected_getconfig)))) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_set_network_instance_static_routes( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_set = [ - network_instance_static_route(NI_NAME, prefix, gateway, metric=metric) - for prefix, gateway, metric in NI_STATIC_ROUTES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_set = list(itertools.chain(*[ + [ + network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) + for ni_sr in ni.get('static_routes', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ + [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] + for ni in NETWORK_INSTANCES + ]))) - prefixes = sorted([ - prefix - for prefix, _, _ in NI_STATIC_ROUTES - ]) - results = set(results_setconfig) - assert len(results) == len(prefixes) - for prefix in prefixes: - assert ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), True) in results - - expected_getconfig = get_expected_network_instance_config(storage) - expected_getconfig.extend([ - ('/network_instance[{:s}]/static_route[{:s}]'.format(NI_NAME, prefix), { - 'name': NI_NAME, 'prefix': prefix, 'next_hop': gateway, 'next_hop_index': 0, 'metric': metric - }) - for prefix, gateway, metric in NI_STATIC_ROUTES - ]) - - permitted_retries = 5 - while permitted_retries > 0: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs == 0: break - # let the device take some time to reconfigure - time.sleep(0.5) - permitted_retries -= 1 - - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_network_instance_static_routes( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance_static_route(NI_NAME, '172.0.0.0/24', '172.16.0.2'), - network_instance_static_route(NI_NAME, '172.2.0.0/24', '172.16.0.3'), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) + for ni_sr in ni.get('static_routes', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ + [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] + for ni in NETWORK_INSTANCES + ]))) - #interfaces = sorted(['Ethernet1', 'Ethernet10']) - #results = set(results_deleteconfig) - #assert len(results) == len(interfaces) - #for if_name in interfaces: - # assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - #expected_getconfig = get_expected_interface_config(storage) - - #diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - #num_diffs = len(diff_data) - #if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - #assert num_diffs == 0 - raise Exception() + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_interfaces_from_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - network_instance_interface(NI_NAME, if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + network_instance_interface(ni['name'], ni_if['if_name'], ni_if['subif_index']) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', list(itertools.chain(*[ + [ + (ni['name'], '{:s}.{:d}'.format(ni_if['if_name'], ni_if['subif_index'])) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ]))) - interface_ids = sorted([ - '{:s}.{:d}'.format(if_name, sif_index) - for if_name, sif_index, _, _, _ in NI_INTERFACES - ]) - results = set(results_deleteconfig) - assert len(results) == len(interface_ids) - for interface_id in interface_ids: - assert ('/network_instance[{:s}]/interface[{:s}]'.format(NI_NAME, interface_id), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - expected_getconfig.extend([ - ('/interface[Ethernet1]/subinterface[0]/ipv4[192.168.1.1]', { - 'ip': '192.168.1.1', 'origin': 'STATIC', 'prefix': 24 - }), - ('/interface[Ethernet10]/subinterface[0]/ipv4[192.168.10.1]', { - 'ip': '192.168.10.1', 'origin': 'STATIC', 'prefix': 24 - }) - ]) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 - raise Exception() + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - resources_to_delete = [ - interface('Ethernet1', 0, '192.168.1.1', 24, True), - interface('Ethernet10', 0, '192.168.10.1', 24, True), - ] + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure + + resources_to_delete = list(itertools.chain(*[ + [ + interface(ni_if['if_name'], ni_if['sif_index'], ni_if['ipv4_addr'], ni_if['ipv4_prefix'], ni_if['enabled']) + for ni_if in ni.get('interfaces', list()) + ] + for ni in NETWORK_INSTANCES + ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/interface[{:s}]', list(itertools.chain(*[ + [ni_if['name'] for ni_if in ni.get('interfaces', list())] + for ni in NETWORK_INSTANCES + ]))) - interfaces = sorted(['Ethernet1', 'Ethernet10']) - results = set(results_deleteconfig) - assert len(results) == len(interfaces) - for if_name in interfaces: - assert ('/interface[{:s}]'.format(if_name), True) in results - - resources_to_get = [RESOURCE_INTERFACES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - expected_getconfig = get_expected_interface_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) def test_del_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name - storage : Dict, # pylint: disable=redefined-outer-name + storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + check_config_interfaces(driver, storage) + check_config_network_instances(driver, storage) + + # TODO: update structure resources_to_delete = [ - network_instance(NI_NAME, 'L3VRF'), + network_instance(ni['name'], ni['type']) + for ni in NETWORK_INSTANCES ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + check_updates(results_deleteconfig, '/network_instance[{:s}]', [ni['name'] for ni in NETWORK_INSTANCES]) - network_instances = sorted([NI_NAME]) - results = set(results_deleteconfig) - assert len(results) == len(network_instances) - for ni_name in network_instances: - assert ('/network_instance[{:s}]'.format(ni_name), True) in results - - resources_to_get = [RESOURCE_NETWORK_INSTANCES] - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) - - for resource_key, resource_value in results_getconfig: - match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) - if match is None: continue - members = resource_value.get('members') - if len(members) > 0: resource_value['members'] = sorted(members) - - expected_getconfig = get_expected_network_instance_config(storage) - - diff_data = deepdiff.DeepDiff(sorted(expected_getconfig), sorted(results_getconfig)) - num_diffs = len(diff_data) - if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) - assert num_diffs == 0 + check_config_interfaces(driver, storage, max_retries=5) + check_config_network_instances(driver, storage, max_retries=5) diff --git a/src/device/tests/gnmi_openconfig/tools/__init__.py b/src/device/tests/gnmi_openconfig/tools/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py new file mode 100644 index 000000000..017a7038e --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -0,0 +1,82 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy, deepdiff, logging, time +from typing import Callable, Dict, List, Tuple +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, + RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) +from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver +from device.tests.gnmi_openconfig.storage.Storage import Storage +from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_network_instance + +LOGGER = logging.getLogger(__name__) + +def check_expected_config( + driver : GnmiOpenConfigDriver, resources_to_get : Dict[str], expected_config : List[Dict], + func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + num_retry = 0 + return_data = None + while num_retry < max_retries: + LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + return_data = copy.deepcopy(results_getconfig) + + results_getconfig = [ + func_adapt_returned_config(resource_key, resource_value) + for resource_key, resource_value in results_getconfig + ] + + diff_data = deepdiff.DeepDiff(sorted(expected_config), sorted(results_getconfig)) + num_diffs = len(diff_data) + if num_diffs == 0: break + # let the device take some time to reconfigure + time.sleep(retry_delay) + num_retry -= 1 + + if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) + assert num_diffs == 0 + return return_data + +def check_config_endpoints( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + return check_expected_config( + driver, [RESOURCE_ENDPOINTS], storage.endpoints.get_expected_config(), + adapt_endpoint, max_retries=max_retries, retry_delay=retry_delay + ) + +def check_config_interfaces( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + return check_expected_config( + driver, [RESOURCE_INTERFACES], storage.interfaces.get_expected_config(), + adapt_interface, max_retries=max_retries, retry_delay=retry_delay + ) + +def check_config_network_instances( + driver : GnmiOpenConfigDriver, storage : Storage, + max_retries : int = 1, retry_delay : float = 0.5 +) -> List[Dict]: + expected_config = + return check_expected_config( + driver, [RESOURCE_NETWORK_INSTANCES], storage.network_instances.get_expected_config(), + adapt_network_instance, max_retries=max_retries, retry_delay=retry_delay + ) diff --git a/src/device/tests/gnmi_openconfig/tools/check_updates.py b/src/device/tests/gnmi_openconfig/tools/check_updates.py new file mode 100644 index 000000000..7f31844cf --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/check_updates.py @@ -0,0 +1,21 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Iterable, List, Tuple + +def check_updates(results : Iterable[Tuple[str, bool]], format_str : str, item_ids : List[Tuple]) -> None: + results = set(results) + assert len(results) == len(item_ids) + for item_id in item_ids: + assert (format_str.format(*item_id), True) in results diff --git a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py new file mode 100644 index 000000000..487476c01 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py @@ -0,0 +1,58 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + + +def compose_expected_config__network_instance( + network_instances : List[Dict], include_interfaces : bool = False, include_static_routes : bool = False +) -> List[Dict]: + expected_config = list() + for network_instance in network_instances: + ni_name = network_instance['name'] + ni_type = network_instance['type'] + + expected_config.extend([ + ('/network_instance[{:s}]'.format(ni_name), { + 'name': ni_name, 'type': ni_type + }), + ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(ni_name), { + 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(ni_name), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' + }), + ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(ni_name), { + 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' + }) + ]) + + if include_interfaces: + expected_config.extend([ + ('/network_instance[{:s}]/interface[{:s}]'.format(ni_name, interface['name']), { + + }) + for interface in network_instance.get('interfaces', list()) + ]) + + if include_static_routes: + expected_config.extend([ + ('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { + 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], + 'next_hop_index': 0, 'metric': static_route['metric'] + }) + for static_route in network_instance.get('static_routes', list()) + ]) + + return expected_config diff --git a/src/device/tests/gnmi_openconfig/request_composers.py b/src/device/tests/gnmi_openconfig/tools/request_composers.py similarity index 100% rename from src/device/tests/gnmi_openconfig/request_composers.py rename to src/device/tests/gnmi_openconfig/tools/request_composers.py diff --git a/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py b/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py new file mode 100644 index 000000000..3712f9365 --- /dev/null +++ b/src/device/tests/gnmi_openconfig/tools/result_config_adapters.py @@ -0,0 +1,29 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import Dict, Tuple + +def adapt_endpoint(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + return resource_key, resource_value + +def adapt_interface(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + return resource_key, resource_value + +def adapt_network_instance(resource_key : str, resource_value : Dict) -> Tuple[str, Dict]: + match = re.match(r'^\/network\_instance\[([^\]]+)\]\/vlan\[([^\]]+)\]$', resource_key) + if match is not None: + members = resource_value.get('members') + if len(members) > 0: resource_value['members'] = sorted(members) + return resource_key, resource_value -- GitLab From 3a2f62eb760354066323ce060c835ed43e1363a2 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 11 Jan 2024 15:16:54 +0000 Subject: [PATCH 018/941] Device component - GNMI OpenConfig: - Updated unitary tests --- .../storage/StorageNetworkInstance.py | 11 +- .../test_unitary_gnmi_openconfig.py | 139 +++++++++--------- .../gnmi_openconfig/tools/check_config.py | 1 - .../tools/expected_config_composers.py | 58 -------- 4 files changed, 78 insertions(+), 131 deletions(-) delete mode 100644 src/device/tests/gnmi_openconfig/tools/expected_config_composers.py diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py index 558cc032c..fa3364883 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -45,16 +45,14 @@ class NetworkInstances: class Interfaces: STRUCT : List[Tuple[str, List[str]]] = [ - ('/network_instance[{:s}]/interface[{:s}]', ['ni_name', 'if_name']), + ('/network_instance[{:s}]/interface[{:s}]', []), ] def __init__(self) -> None: self._items : Dict[Tuple[str, str], Dict] = dict() def add(self, ni_name : str, if_name : str) -> None: - item = self._items.setdefault((ni_name, if_name), dict()) - item['ni_name'] = ni_name - item['if_name'] = if_name + self._items.setdefault((ni_name, if_name), dict()) def remove(self, ni_name : str, if_name : str) -> None: self._items.pop((ni_name, if_name), None) @@ -86,6 +84,11 @@ class StaticRoutes: ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), ] + #('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { + # 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], + # 'next_hop_index': 0, 'metric': static_route['metric'] + #}) + def __init__(self) -> None: self._items : Dict[Tuple[str, str, str], Dict] = dict() diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index dd0561a2b..3970e65a6 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -16,21 +16,17 @@ import os os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position -import itertools, logging, pytest, time +import logging, pytest, time from typing import Dict -from device.service.driver_api._Driver import ( - RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES -) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver -from .tools.check_config import check_config_endpoints, check_config_interfaces, check_config_network_instances -from .tools.check_updates import check_updates -from .tools.expected_config_composers import ( - compose_expected_config__interface, compose_expected_config__network_instance +from .storage.Storage import Storage +from .tools.check_config import ( + check_config_endpoints, check_config_interfaces, check_config_network_instances ) +from .tools.check_updates import check_updates from .tools.request_composers import ( interface, network_instance, network_instance_interface, network_instance_static_route ) -from .storage.Storage import Storage logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger(__name__) @@ -200,7 +196,7 @@ def test_add_interfaces_to_network_instance( subif_index = ni_if['sif_index'] resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.add(ni_name, if_name, subif_index) + storage.network_instances.interfaces.add(ni_name, if_name) # TODO: add subif_index LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) @@ -218,22 +214,26 @@ def test_set_network_instance_static_routes( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_set = list() + ni_sr_prefixes = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_sr in ni.get('static_routes', list()): + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_gateway = ni_sr['gateway'] + ni_sr_metric = ni_sr['metric' ] + resources_to_set.append( + network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) + ) + ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocol_static.add(ni_name, 'STATIC', ni_sr_prefix, { + 'prefix': ni_sr_prefix, + }) - resources_to_set = list(itertools.chain(*[ - [ - network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) - for ni_sr in ni.get('static_routes', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) results_setconfig = driver.SetConfig(resources_to_set) LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ - [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -246,22 +246,24 @@ def test_del_network_instance_static_routes( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_sr_prefixes = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_sr in ni.get('static_routes', list()): + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_gateway = ni_sr['gateway'] + ni_sr_metric = ni_sr['metric' ] + resources_to_delete.append( + network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) + ) + ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocol_static.remove(ni_name, 'STATIC', ni_sr_prefix) - resources_to_delete = list(itertools.chain(*[ - [ - network_instance_static_route(ni['name'], ni_sr['prefix'], ni_sr['gateway'], metric=ni_sr['metric']) - for ni_sr in ni.get('static_routes', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', list(itertools.chain(*[ - [(ni['name'], ni_sr['prefix']) for ni_sr in ni.get('static_routes', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -274,26 +276,22 @@ def test_del_interfaces_from_network_instance( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.remove(ni_name, if_name) # TODO: add subif_index - resources_to_delete = list(itertools.chain(*[ - [ - network_instance_interface(ni['name'], ni_if['if_name'], ni_if['subif_index']) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', list(itertools.chain(*[ - [ - (ni['name'], '{:s}.{:d}'.format(ni_if['if_name'], ni_if['subif_index'])) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ]))) - + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -305,22 +303,24 @@ def test_del_interfaces( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + if_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + for ni_if in ni.get('interfaces', list()): + if_name = ni_if['if_name'] + subif_index = ni_if['sif_index'] + ipv4_address = ni_if['ipv4_addr'] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled'] + resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) + if_names.append(ni_name) + storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) - resources_to_delete = list(itertools.chain(*[ - [ - interface(ni_if['if_name'], ni_if['sif_index'], ni_if['ipv4_addr'], ni_if['ipv4_prefix'], ni_if['enabled']) - for ni_if in ni.get('interfaces', list()) - ] - for ni in NETWORK_INSTANCES - ])) LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/interface[{:s}]', list(itertools.chain(*[ - [ni_if['name'] for ni_if in ni.get('interfaces', list())] - for ni in NETWORK_INSTANCES - ]))) + check_updates(results_deleteconfig, '/interface[{:s}]', if_names) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) @@ -333,16 +333,19 @@ def test_del_network_instances( check_config_interfaces(driver, storage) check_config_network_instances(driver, storage) - # TODO: update structure + resources_to_delete = list() + ni_names = list() + for ni in NETWORK_INSTANCES: + ni_name = ni['name'] + ni_type = ni['type'] + resources_to_delete.append(network_instance(ni_name, ni_type)) + ni_names.append(ni_name) + storage.network_instances.network_instances.remove(ni_name) - resources_to_delete = [ - network_instance(ni['name'], ni['type']) - for ni in NETWORK_INSTANCES - ] LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) results_deleteconfig = driver.DeleteConfig(resources_to_delete) LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]', [ni['name'] for ni in NETWORK_INSTANCES]) + check_updates(results_deleteconfig, '/network_instance[{:s}]', ni_names) check_config_interfaces(driver, storage, max_retries=5) check_config_network_instances(driver, storage, max_retries=5) diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py index 017a7038e..974acdeba 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -75,7 +75,6 @@ def check_config_network_instances( driver : GnmiOpenConfigDriver, storage : Storage, max_retries : int = 1, retry_delay : float = 0.5 ) -> List[Dict]: - expected_config = return check_expected_config( driver, [RESOURCE_NETWORK_INSTANCES], storage.network_instances.get_expected_config(), adapt_network_instance, max_retries=max_retries, retry_delay=retry_delay diff --git a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py b/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py deleted file mode 100644 index 487476c01..000000000 --- a/src/device/tests/gnmi_openconfig/tools/expected_config_composers.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, List - - -def compose_expected_config__network_instance( - network_instances : List[Dict], include_interfaces : bool = False, include_static_routes : bool = False -) -> List[Dict]: - expected_config = list() - for network_instance in network_instances: - ni_name = network_instance['name'] - ni_type = network_instance['type'] - - expected_config.extend([ - ('/network_instance[{:s}]'.format(ni_name), { - 'name': ni_name, 'type': ni_type - }), - ('/network_instance[{:s}]/protocol[DIRECTLY_CONNECTED]'.format(ni_name), { - 'id': 'DIRECTLY_CONNECTED', 'name': 'DIRECTLY_CONNECTED' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV4]'.format(ni_name), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV4' - }), - ('/network_instance[{:s}]/table[DIRECTLY_CONNECTED,IPV6]'.format(ni_name), { - 'protocol': 'DIRECTLY_CONNECTED', 'address_family': 'IPV6' - }) - ]) - - if include_interfaces: - expected_config.extend([ - ('/network_instance[{:s}]/interface[{:s}]'.format(ni_name, interface['name']), { - - }) - for interface in network_instance.get('interfaces', list()) - ]) - - if include_static_routes: - expected_config.extend([ - ('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { - 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], - 'next_hop_index': 0, 'metric': static_route['metric'] - }) - for static_route in network_instance.get('static_routes', list()) - ]) - - return expected_config -- GitLab From ffe63af338f70a638ec5df86c47be131ce117261 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 11 Jan 2024 18:14:24 +0000 Subject: [PATCH 019/941] Device component - GNMI OpenConfig: - Updated unitary tests --- src/device/tests/gnmi_openconfig/tools/check_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/check_config.py index 974acdeba..5258da80c 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/check_config.py @@ -25,10 +25,12 @@ from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_netwo LOGGER = logging.getLogger(__name__) def check_expected_config( - driver : GnmiOpenConfigDriver, resources_to_get : Dict[str], expected_config : List[Dict], + driver : GnmiOpenConfigDriver, resources_to_get : List[str], expected_config : List[Dict], func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, max_retries : int = 1, retry_delay : float = 0.5 ) -> List[Dict]: + LOGGER.info('expected_config = {:s}'.format(str(expected_config))) + num_retry = 0 return_data = None while num_retry < max_retries: -- GitLab From fce71b000b701d2d4e511092389eccb448d28987 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 12 Jan 2024 19:03:48 +0000 Subject: [PATCH 020/941] Device component - GNMI OpenConfig: - Corrected Component PORT retrieval - Corrected Static Route management - Corrected unitary tests - Added libyang to Dockerfile and corrected copy of clients - Corrected requirements.in --- src/device/Dockerfile | 21 +- src/device/requirements.in | 14 +- .../gnmi_openconfig/handlers/Component.py | 4 +- .../handlers/NetworkInstance.py | 17 +- .../handlers/NetworkInstanceStaticRoute.py | 6 +- .../storage/StorageEndpoints.py | 5 +- .../storage/StorageInterface.py | 11 +- .../storage/StorageNetworkInstance.py | 45 ++- .../tests/gnmi_openconfig/storage/Tools.py | 1 + .../test_unitary_gnmi_openconfig.py | 259 +++++++++--------- .../gnmi_openconfig/tools/check_updates.py | 5 +- .../{check_config.py => manage_config.py} | 30 +- .../tools/request_composers.py | 4 +- 13 files changed, 250 insertions(+), 172 deletions(-) rename src/device/tests/gnmi_openconfig/tools/{check_config.py => manage_config.py} (71%) diff --git a/src/device/Dockerfile b/src/device/Dockerfile index 656662552..2bcb5322a 100644 --- a/src/device/Dockerfile +++ b/src/device/Dockerfile @@ -53,6 +53,21 @@ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; +# Download, build and install libyang. Note that APT package is outdated +# - Ref: https://github.com/CESNET/libyang +# - Ref: https://github.com/CESNET/libyang-python/ +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install build-essential cmake libpcre2-dev python3-dev python3-cffi && \ + rm -rf /var/lib/apt/lists/* +RUN mkdir -p /var/libyang +RUN git clone https://github.com/CESNET/libyang.git /var/libyang +RUN mkdir -p /var/libyang/build +WORKDIR /var/libyang/build +RUN cmake -D CMAKE_BUILD_TYPE:String="Release" .. +RUN make +RUN make install +RUN ldconfig + # Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/device WORKDIR /var/teraflow/device @@ -62,9 +77,11 @@ RUN python3 -m pip install -r requirements.txt # Add component files into working directory WORKDIR /var/teraflow -COPY src/context/. context/ COPY src/device/. device/ -COPY src/monitoring/. monitoring/ +COPY src/context/__init__.py context/__init__.py +COPY src/context/client/. context/client/ +COPY src/monitoring/__init__.py monitoring/__init__.py +COPY src/monitoring/client/. monitoring/client/ # Start the service ENTRYPOINT ["python", "-m", "device.service"] diff --git a/src/device/requirements.in b/src/device/requirements.in index d8a33455e..20ed1e2dc 100644 --- a/src/device/requirements.in +++ b/src/device/requirements.in @@ -12,30 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. - anytree==2.8.0 APScheduler==3.10.1 bitarray==2.8.* cryptography==36.0.2 #fastcache==1.1.0 +ipaddress Jinja2==3.0.3 +libyang==2.8.0 +macaddress ncclient==0.6.13 p4runtime==1.3.0 pandas==1.5.* paramiko==2.9.2 +pyang==2.6.* +git+https://github.com/robshakir/pyangbind.git python-json-logger==2.0.2 #pytz==2021.3 #redis==4.1.2 requests==2.27.1 requests-mock==1.9.3 -xmltodict==0.12.0 tabulate -ipaddress -macaddress -yattag -pyang==2.6.* -git+https://github.com/robshakir/pyangbind.git websockets==10.4 +xmltodict==0.12.0 +yattag # pip's dependency resolver does not take into account installed packages. # p4runtime does not specify the version of grpcio/protobuf it needs, so it tries to install latest one diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py index 73728192f..9b92c6b83 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/Component.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/Component.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json, logging # libyang +import json, logging, re # libyang from typing import Any, Dict, List, Tuple from common.proto.kpi_sample_types_pb2 import KpiSampleType from ._Handler import _Handler @@ -55,7 +55,7 @@ class ComponentHandler(_Handler): # TODO: improve mapping between interface name and component name # By now, computed by time for the sake of saving time for the Hackfest. - interface_name = component_name.lower().replace('-port', '') + interface_name = re.sub(r'\-[pP][oO][rR][tT]', '', component_name) endpoint = {'uuid': interface_name, 'type': '-'} endpoint['sample_types'] = { diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py index b97612987..d8231b2a6 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstance.py @@ -134,18 +134,15 @@ class NetworkInstanceHandler(_Handler): for static_route in static_routes: static_route_prefix = static_route['prefix'] - next_hops = static_route.get('next-hops', {}).get('next-hop', []) - _next_hops = [ - { - 'index' : next_hop['index'], - 'gateway': next_hop['config']['next-hop'], - 'metric' : next_hop['config']['metric'], + next_hops = { + next_hop['index'] : { + 'next_hop': next_hop['config']['next-hop'], + 'metric' : next_hop['config']['metric'], } - for next_hop in next_hops - ] - _next_hops = sorted(_next_hops, key=operator.itemgetter('index')) + for next_hop in static_route.get('next-hops', {}).get('next-hop', []) + } - _static_route = {'prefix': static_route_prefix, 'next_hops': _next_hops} + _static_route = {'prefix': static_route_prefix, 'next_hops': next_hops} entry_static_route_key = '{:s}/static_routes[{:s}]'.format( entry_protocol_key, static_route_prefix ) diff --git a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py index 0343e3cba..03c04e316 100644 --- a/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py +++ b/src/device/service/drivers/gnmi_openconfig/handlers/NetworkInstanceStaticRoute.py @@ -39,8 +39,8 @@ class NetworkInstanceStaticRouteHandler(_Handler): str_data = json.dumps({}) return str_path, str_data - next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' - next_hop_index = get_int(resource_value, 'next_hop_index', 0) # 0 + next_hop = get_str(resource_value, 'next_hop' ) # '172.0.0.1' + next_hop_index = get_str(resource_value, 'next_hop_index') # AUTO_1_172-0-0-1 PATH_TMPL = '/network-instances/network-instance[name={:s}]/protocols/protocol[identifier={:s}][name={:s}]' str_path = PATH_TMPL.format(ni_name, identifier, name) @@ -74,7 +74,7 @@ class NetworkInstanceStaticRouteHandler(_Handler): yang_ni_pr_sr.create_path('config/prefix', prefix) yang_ni_pr_sr_nhs : libyang.DContainer = yang_ni_pr_sr.create_path('next-hops') - yang_ni_pr_sr_nh_path = 'next-hop[index="{:d}"]'.format(next_hop_index) + yang_ni_pr_sr_nh_path = 'next-hop[index="{:s}"]'.format(next_hop_index) yang_ni_pr_sr_nh : libyang.DContainer = yang_ni_pr_sr_nhs.create_path(yang_ni_pr_sr_nh_path) yang_ni_pr_sr_nh.create_path('config/index', next_hop_index) yang_ni_pr_sr_nh.create_path('config/next-hop', next_hop) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py index 815a1b0ad..d2596b732 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageEndpoints.py @@ -39,13 +39,16 @@ class Endpoints: for _, field_names in Endpoints.STRUCT: field_names = set(field_names) - item.update({k:v for k,v in resource_value if k in field_names}) + item.update({k:v for k,v in resource_value.items() if k in field_names}) item['sample_types'] = { sample_type_id : sample_type_path.format(ep_uuid) for sample_type_id, sample_type_path in ENDPOINT_PACKET_SAMPLE_TYPES.items() } + def get(self, ep_uuid : str) -> Dict: + return self._items.get(ep_uuid) + def remove(self, ep_uuid : str) -> None: self._items.pop(ep_uuid, None) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py index a0391e92f..0933433cb 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageInterface.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageInterface.py @@ -37,7 +37,10 @@ class Interfaces: item['name'] = if_name for _, field_names in Interfaces.STRUCT: field_names = set(field_names) - item.update({k:v for k,v in resource_value if k in field_names}) + item.update({k:v for k,v in resource_value.items() if k in field_names}) + + def get(self, if_name : str) -> Dict: + return self._items.get(if_name) def remove(self, if_name : str) -> None: self._items.pop(if_name, None) @@ -57,6 +60,9 @@ class SubInterfaces: item = self._items.setdefault((if_name, subif_index), dict()) item['index'] = subif_index + def get(self, if_name : str, subif_index : int) -> Dict: + return self._items.get((if_name, subif_index)) + def remove(self, if_name : str, subif_index : int) -> None: self._items.pop((if_name, subif_index), None) @@ -77,6 +83,9 @@ class IPv4Addresses: item['origin'] = resource_value.get('origin') item['prefix'] = resource_value.get('prefix') + def get(self, if_name : str, subif_index : int, ipv4_address : str) -> Dict: + return self._items.get((if_name, subif_index, ipv4_address)) + def remove(self, if_name : str, subif_index : int, ipv4_address : str) -> None: self._items.pop((if_name, subif_index, ipv4_address), None) diff --git a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py index fa3364883..ba437ef9d 100644 --- a/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py +++ b/src/device/tests/gnmi_openconfig/storage/StorageNetworkInstance.py @@ -37,6 +37,9 @@ class NetworkInstances: item['name'] = ni_name item['type'] = resource_value.get('type') + def get(self, ni_name : str) -> Dict: + return self._items.get(ni_name) + def remove(self, ni_name : str) -> None: self._items.pop(ni_name, None) @@ -45,17 +48,24 @@ class NetworkInstances: class Interfaces: STRUCT : List[Tuple[str, List[str]]] = [ - ('/network_instance[{:s}]/interface[{:s}]', []), + ('/network_instance[{:s}]/interface[{:s}.{:d}]', ['name', 'id', 'if_name', 'sif_index']), ] def __init__(self) -> None: self._items : Dict[Tuple[str, str], Dict] = dict() - def add(self, ni_name : str, if_name : str) -> None: - self._items.setdefault((ni_name, if_name), dict()) + def add(self, ni_name : str, if_name : str, sif_index : int) -> None: + item = self._items.setdefault((ni_name, if_name, sif_index), dict()) + item['name' ] = ni_name + item['id' ] = '{:s}.{:d}'.format(if_name, sif_index) + item['if_name' ] = if_name + item['sif_index'] = sif_index + + def get(self, ni_name : str, if_name : str, sif_index : int) -> Dict: + return self._items.get((ni_name, if_name, sif_index)) - def remove(self, ni_name : str, if_name : str) -> None: - self._items.pop((ni_name, if_name), None) + def remove(self, ni_name : str, if_name : str, sif_index : int) -> None: + self._items.pop((ni_name, if_name, sif_index), None) def compose_resources(self) -> List[Dict]: return compose_resources(self._items, Interfaces.STRUCT) @@ -73,6 +83,9 @@ class Protocols: item['id' ] = protocol item['name'] = protocol + def get(self, ni_name : str, protocol : str) -> Dict: + return self._items.get((ni_name, protocol)) + def remove(self, ni_name : str, protocol : str) -> None: self._items.pop((ni_name, protocol), None) @@ -84,18 +97,16 @@ class StaticRoutes: ('/network_instance[{:s}]/protocol[{:s}]/static_routes[{:s}]', ['prefix', 'next_hops']), ] - #('/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, static_route['prefix']), { - # 'name': ni_name, 'prefix': static_route['prefix'], 'next_hop': static_route['gateway'], - # 'next_hop_index': 0, 'metric': static_route['metric'] - #}) - def __init__(self) -> None: self._items : Dict[Tuple[str, str, str], Dict] = dict() def add(self, ni_name : str, protocol : str, prefix : str, resource_value : Dict) -> None: item = self._items.setdefault((ni_name, protocol, prefix), dict()) item['prefix' ] = prefix - item['next_hops'] = sorted(resource_value.get('next_hops')) + item['next_hops'] = resource_value.get('next_hops') + + def get(self, ni_name : str, protocol : str, prefix : str) -> Dict: + return self._items.get((ni_name, protocol, prefix)) def remove(self, ni_name : str, protocol : str, prefix : str) -> None: self._items.pop((ni_name, protocol, prefix), None) @@ -116,6 +127,9 @@ class Tables: item['protocol' ] = protocol item['address_family'] = address_family + def get(self, ni_name : str, protocol : str, address_family : str) -> Dict: + return self._items.get((ni_name, protocol, address_family)) + def remove(self, ni_name : str, protocol : str, address_family : str) -> None: self._items.pop((ni_name, protocol, address_family), None) @@ -136,6 +150,9 @@ class Vlans: item['name' ] = resource_value.get('name') item['members'] = sorted(resource_value.get('members')) + def get(self, ni_name : str, vlan_id : int) -> Dict: + return self._items.get((ni_name, vlan_id)) + def remove(self, ni_name : str, vlan_id : int) -> None: self._items.pop((ni_name, vlan_id), None) @@ -160,7 +177,11 @@ class StorageNetworkInstance: match = RE_RESKEY_INTERFACE.match(resource_key) if match is not None: - self.interfaces.add(match.group(1), match.group(2)) + if_id = match.group(2) + if_id_parts = if_id.split('.') + if_name = if_id_parts[0] + sif_index = 0 if len(if_id_parts) == 1 else int(if_id_parts[1]) + self.interfaces.add(match.group(1), if_name, sif_index) continue match = RE_RESKEY_PROTOCOL.match(resource_key) diff --git a/src/device/tests/gnmi_openconfig/storage/Tools.py b/src/device/tests/gnmi_openconfig/storage/Tools.py index 4da48af46..c9dab12e6 100644 --- a/src/device/tests/gnmi_openconfig/storage/Tools.py +++ b/src/device/tests/gnmi_openconfig/storage/Tools.py @@ -21,6 +21,7 @@ def compose_resources( for resource_key_fields, resource_value_data in storage.items(): for resource_key_template, resource_key_field_names in config_struct: + if isinstance(resource_key_fields, (str, int, float, bool)): resource_key_fields = (resource_key_fields,) resource_key = resource_key_template.format(*resource_key_fields) resource_value = { field_name : resource_value_data[field_name] diff --git a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py index 3970e65a6..a601e1f23 100644 --- a/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py +++ b/src/device/tests/gnmi_openconfig/test_unitary_gnmi_openconfig.py @@ -17,11 +17,15 @@ os.environ['DEVICE_EMULATED_ONLY'] = 'YES' # pylint: disable=wrong-import-position import logging, pytest, time -from typing import Dict +from typing import Dict, List +from device.service.driver_api._Driver import ( + RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, + RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES +) from device.service.drivers.gnmi_openconfig.GnmiOpenConfigDriver import GnmiOpenConfigDriver from .storage.Storage import Storage -from .tools.check_config import ( - check_config_endpoints, check_config_interfaces, check_config_network_instances +from .tools.manage_config import ( + check_config_endpoints, check_config_interfaces, check_config_network_instances, del_config, get_config, set_config ) from .tools.check_updates import check_updates from .tools.request_composers import ( @@ -69,26 +73,26 @@ NETWORK_INSTANCES = [ 'name': 'test-l3-svc', 'type': 'L3VRF', 'interfaces': [ - {'name': 'Ethernet1', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, - {'name': 'Ethernet10', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet1', 'index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + {'name': 'Ethernet10', 'index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, ], 'static_routes': [ - {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, - {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, + {'prefix': '172.0.0.0/24', 'next_hop': '172.16.0.2', 'metric': 1}, + {'prefix': '172.2.0.0/24', 'next_hop': '172.16.0.3', 'metric': 1}, ] }, - { - 'name': 'test-l2-svc', - 'type': 'L2VSI', - 'interfaces': [ - {'name': 'Ethernet2', 'subif_index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, - {'name': 'Ethernet4', 'subif_index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, - ], - 'static_routes': [ - {'prefix': '172.0.0.0/24', 'gateway': '172.16.0.2', 'metric': 1}, - {'prefix': '172.2.0.0/24', 'gateway': '172.16.0.3', 'metric': 1}, - ] - } + #{ + # 'name': 'test-l2-svc', + # 'type': 'L2VSI', + # 'interfaces': [ + # {'name': 'Ethernet2', 'index': 0, 'ipv4_addr': '192.168.1.1', 'ipv4_prefix': 24, 'enabled': True}, + # {'name': 'Ethernet4', 'index': 0, 'ipv4_addr': '192.168.10.1', 'ipv4_prefix': 24, 'enabled': True}, + # ], + # 'static_routes': [ + # {'prefix': '172.0.0.0/24', 'next_hop': '172.16.0.2', 'metric': 1}, + # {'prefix': '172.2.0.0/24', 'next_hop': '172.16.0.3', 'metric': 1}, + # ] + #} ] @@ -98,7 +102,7 @@ def test_get_endpoints( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_endpoints(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_ENDPOINTS]) storage.endpoints.populate(results_getconfig) check_config_endpoints(driver, storage) @@ -107,7 +111,7 @@ def test_get_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_interfaces(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_INTERFACES]) storage.interfaces.populate(results_getconfig) check_config_interfaces(driver, storage) @@ -116,7 +120,7 @@ def test_get_network_instances( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: - results_getconfig = check_config_network_instances(driver, storage) + results_getconfig = get_config(driver, [RESOURCE_NETWORK_INSTANCES]) storage.network_instances.populate(results_getconfig) check_config_network_instances(driver, storage) @@ -136,17 +140,18 @@ def test_set_network_instances( resources_to_set.append(network_instance(ni_name, ni_type)) ni_names.append(ni_name) storage.network_instances.network_instances.add(ni_name, {'type': ni_type}) - - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + storage.network_instances.protocols.add(ni_name, 'DIRECTLY_CONNECTED') + storage.network_instances.tables.add(ni_name, 'DIRECTLY_CONNECTED', 'IPV4') + storage.network_instances.tables.add(ni_name, 'DIRECTLY_CONNECTED', 'IPV6') + + results_setconfig = set_config(driver, resources_to_set) check_updates(results_setconfig, '/network_instance[{:s}]', ni_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_set_interfaces( +def test_add_interfaces_to_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: @@ -154,33 +159,24 @@ def test_set_interfaces( check_config_network_instances(driver, storage) resources_to_set = list() - if_names = list() + ni_if_names = list() for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - ipv4_address = ni_if['ipv4_addr'] - ipv4_prefix = ni_if['ipv4_prefix'] - enabled = ni_if['enabled'] - resources_to_set.append(interface( - if_name, subif_index, ipv4_address, ipv4_prefix, enabled - )) - if_names.append(ni_name) - storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { - 'origin' : 'STATIC', 'prefix': ipv4_prefix - }) + if_name = ni_if['name' ] + subif_index = ni_if['index'] + resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.add(ni_name, if_name, subif_index) - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/interface[{:s}]', if_names) + results_setconfig = set_config(driver, resources_to_set) + check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_add_interfaces_to_network_instance( +def test_set_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: @@ -188,23 +184,30 @@ def test_add_interfaces_to_network_instance( check_config_network_instances(driver, storage) resources_to_set = list() - ni_if_names = list() + if_names = list() for ni in NETWORK_INSTANCES: - ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - resources_to_set.append(network_instance_interface(ni_name, if_name, subif_index)) - ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.add(ni_name, if_name) # TODO: add subif_index + if_name = ni_if['name' ] + subif_index = ni_if['index' ] + ipv4_address = ni_if['ipv4_addr' ] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled' ] + resources_to_set.append(interface( + if_name, subif_index, ipv4_address, ipv4_prefix, enabled + )) + if_names.append(if_name) + storage.interfaces.ipv4_addresses.add(if_name, subif_index, ipv4_address, { + 'origin' : 'STATIC', 'prefix': ipv4_prefix + }) + default_vlan = storage.network_instances.vlans.get('default', 1) + default_vlan_members : List[str] = default_vlan.setdefault('members', list()) + if if_name in default_vlan_members: default_vlan_members.remove(if_name) - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) - check_updates(results_setconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + results_setconfig = set_config(driver, resources_to_set) + check_updates(results_setconfig, '/interface[{:s}]', if_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_set_network_instance_static_routes( @@ -219,24 +222,28 @@ def test_set_network_instance_static_routes( for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_sr in ni.get('static_routes', list()): - ni_sr_prefix = ni_sr['prefix' ] - ni_sr_gateway = ni_sr['gateway'] - ni_sr_metric = ni_sr['metric' ] - resources_to_set.append( - network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) - ) + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_next_hop = ni_sr['next_hop'] + ni_sr_metric = ni_sr['metric' ] + ni_sr_next_hop_index = 'AUTO_{:d}_{:s}'.format(ni_sr_metric, '-'.join(ni_sr_next_hop.split('.'))) + resources_to_set.append(network_instance_static_route( + ni_name, ni_sr_prefix, ni_sr_next_hop_index, ni_sr_next_hop, metric=ni_sr_metric + )) ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + storage.network_instances.protocols.add(ni_name, 'STATIC') storage.network_instances.protocol_static.add(ni_name, 'STATIC', ni_sr_prefix, { - 'prefix': ni_sr_prefix, + 'prefix': ni_sr_prefix, 'next_hops': { + ni_sr_next_hop_index: {'next_hop': ni_sr_next_hop, 'metric': ni_sr_metric} + } }) + storage.network_instances.tables.add(ni_name, 'STATIC', 'IPV4') + storage.network_instances.tables.add(ni_name, 'STATIC', 'IPV6') - LOGGER.info('resources_to_set = {:s}'.format(str(resources_to_set))) - results_setconfig = driver.SetConfig(resources_to_set) - LOGGER.info('results_setconfig = {:s}'.format(str(results_setconfig))) + results_setconfig = set_config(driver, resources_to_set) check_updates(results_setconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_del_network_instance_static_routes( @@ -251,79 +258,80 @@ def test_del_network_instance_static_routes( for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_sr in ni.get('static_routes', list()): - ni_sr_prefix = ni_sr['prefix' ] - ni_sr_gateway = ni_sr['gateway'] - ni_sr_metric = ni_sr['metric' ] - resources_to_delete.append( - network_instance_static_route(ni_name, ni_sr_prefix, ni_sr_gateway, metric=ni_sr_metric) - ) + ni_sr_prefix = ni_sr['prefix' ] + ni_sr_next_hop = ni_sr['next_hop'] + ni_sr_metric = ni_sr['metric' ] + ni_sr_next_hop_index = 'AUTO_{:d}_{:s}'.format(ni_sr_metric, '-'.join(ni_sr_next_hop.split('.'))) + resources_to_delete.append(network_instance_static_route( + ni_name, ni_sr_prefix, ni_sr_next_hop_index, ni_sr_next_hop, metric=ni_sr_metric + )) ni_sr_prefixes.append((ni_name, ni_sr_prefix)) + + storage.network_instances.protocols.remove(ni_name, 'STATIC') storage.network_instances.protocol_static.remove(ni_name, 'STATIC', ni_sr_prefix) + storage.network_instances.tables.remove(ni_name, 'STATIC', 'IPV4') + storage.network_instances.tables.remove(ni_name, 'STATIC', 'IPV6') - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + results_deleteconfig = del_config(driver, resources_to_delete) check_updates(results_deleteconfig, '/network_instance[{:s}]/static_route[{:s}]', ni_sr_prefixes) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_del_interfaces_from_network_instance( +def test_del_interfaces( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() - ni_if_names = list() + if_names = list() for ni in NETWORK_INSTANCES: - ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) - ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) - storage.network_instances.interfaces.remove(ni_name, if_name) # TODO: add subif_index + if_name = ni_if['name' ] + subif_index = ni_if['index' ] + ipv4_address = ni_if['ipv4_addr' ] + ipv4_prefix = ni_if['ipv4_prefix'] + enabled = ni_if['enabled' ] + resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) + if_names.append(if_name) + storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) + default_vlan = storage.network_instances.vlans.get('default', 1) + default_vlan_members : List[str] = default_vlan.setdefault('members', list()) + if if_name not in default_vlan_members: default_vlan_members.append(if_name) - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) - - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + results_deleteconfig = del_config(driver, resources_to_delete) + check_updates(results_deleteconfig, '/interface[{:s}]', if_names) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) -def test_del_interfaces( + +def test_del_interfaces_from_network_instance( driver : GnmiOpenConfigDriver, # pylint: disable=redefined-outer-name storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() - if_names = list() + ni_if_names = list() for ni in NETWORK_INSTANCES: ni_name = ni['name'] for ni_if in ni.get('interfaces', list()): - if_name = ni_if['if_name'] - subif_index = ni_if['sif_index'] - ipv4_address = ni_if['ipv4_addr'] - ipv4_prefix = ni_if['ipv4_prefix'] - enabled = ni_if['enabled'] - resources_to_delete.append(interface(if_name, subif_index, ipv4_address, ipv4_prefix, enabled)) - if_names.append(ni_name) - storage.interfaces.ipv4_addresses.remove(if_name, subif_index, ipv4_address) - - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) - check_updates(results_deleteconfig, '/interface[{:s}]', if_names) + if_name = ni_if['name' ] + subif_index = ni_if['index'] + resources_to_delete.append(network_instance_interface(ni_name, if_name, subif_index)) + ni_if_names.append((ni_name, '{:s}.{:d}'.format(if_name, subif_index))) + storage.network_instances.interfaces.remove(ni_name, if_name, subif_index) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + results_deleteconfig = del_config(driver, resources_to_delete) + check_updates(results_deleteconfig, '/network_instance[{:s}]/interface[{:s}]', ni_if_names) + + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + #check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) def test_del_network_instances( @@ -331,7 +339,7 @@ def test_del_network_instances( storage : Storage, # pylint: disable=redefined-outer-name ) -> None: check_config_interfaces(driver, storage) - check_config_network_instances(driver, storage) + #check_config_network_instances(driver, storage) resources_to_delete = list() ni_names = list() @@ -341,11 +349,12 @@ def test_del_network_instances( resources_to_delete.append(network_instance(ni_name, ni_type)) ni_names.append(ni_name) storage.network_instances.network_instances.remove(ni_name) + storage.network_instances.protocols.remove(ni_name, 'DIRECTLY_CONNECTED') + storage.network_instances.tables.remove(ni_name, 'DIRECTLY_CONNECTED', 'IPV4') + storage.network_instances.tables.remove(ni_name, 'DIRECTLY_CONNECTED', 'IPV6') - LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) - results_deleteconfig = driver.DeleteConfig(resources_to_delete) - LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + results_deleteconfig = del_config(driver, resources_to_delete) check_updates(results_deleteconfig, '/network_instance[{:s}]', ni_names) - check_config_interfaces(driver, storage, max_retries=5) - check_config_network_instances(driver, storage, max_retries=5) + check_config_interfaces(driver, storage, max_retries=10, retry_delay=2.0) + check_config_network_instances(driver, storage, max_retries=10, retry_delay=2.0) diff --git a/src/device/tests/gnmi_openconfig/tools/check_updates.py b/src/device/tests/gnmi_openconfig/tools/check_updates.py index 7f31844cf..a9e2a1be9 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_updates.py +++ b/src/device/tests/gnmi_openconfig/tools/check_updates.py @@ -17,5 +17,6 @@ from typing import Iterable, List, Tuple def check_updates(results : Iterable[Tuple[str, bool]], format_str : str, item_ids : List[Tuple]) -> None: results = set(results) assert len(results) == len(item_ids) - for item_id in item_ids: - assert (format_str.format(*item_id), True) in results + for item_id_fields in item_ids: + if isinstance(item_id_fields, (str, int, float, bool)): item_id_fields = (item_id_fields,) + assert (format_str.format(*item_id_fields), True) in results diff --git a/src/device/tests/gnmi_openconfig/tools/check_config.py b/src/device/tests/gnmi_openconfig/tools/manage_config.py similarity index 71% rename from src/device/tests/gnmi_openconfig/tools/check_config.py rename to src/device/tests/gnmi_openconfig/tools/manage_config.py index 5258da80c..72d6a09d3 100644 --- a/src/device/tests/gnmi_openconfig/tools/check_config.py +++ b/src/device/tests/gnmi_openconfig/tools/manage_config.py @@ -13,7 +13,7 @@ # limitations under the License. import copy, deepdiff, logging, time -from typing import Callable, Dict, List, Tuple +from typing import Callable, Dict, List, Tuple, Union from device.service.driver_api._Driver import ( RESOURCE_ENDPOINTS, RESOURCE_INTERFACES, RESOURCE_NETWORK_INSTANCES, RESOURCE_ROUTING_POLICIES, RESOURCE_SERVICES @@ -24,6 +24,28 @@ from .result_config_adapters import adapt_endpoint, adapt_interface, adapt_netwo LOGGER = logging.getLogger(__name__) +def get_config(driver : GnmiOpenConfigDriver, resources_to_get : List[str]) -> List[Tuple[str, Dict]]: + LOGGER.info('[get_config] resources_to_get = {:s}'.format(str(resources_to_get))) + results_getconfig = driver.GetConfig(resources_to_get) + LOGGER.info('[get_config] results_getconfig = {:s}'.format(str(results_getconfig))) + return results_getconfig + +def set_config( + driver : GnmiOpenConfigDriver, resources_to_set : List[Tuple[str, Dict]] +) -> List[Tuple[str, Union[bool, Exception]]]: + LOGGER.info('[set_config] resources_to_set = {:s}'.format(str(resources_to_set))) + results_setconfig = driver.SetConfig(resources_to_set) + LOGGER.info('[set_config] results_setconfig = {:s}'.format(str(results_setconfig))) + return results_setconfig + +def del_config( + driver : GnmiOpenConfigDriver, resources_to_delete : List[Tuple[str, Dict]] +) -> List[Tuple[str, Union[bool, Exception]]]: + LOGGER.info('resources_to_delete = {:s}'.format(str(resources_to_delete))) + results_deleteconfig = driver.DeleteConfig(resources_to_delete) + LOGGER.info('results_deleteconfig = {:s}'.format(str(results_deleteconfig))) + return results_deleteconfig + def check_expected_config( driver : GnmiOpenConfigDriver, resources_to_get : List[str], expected_config : List[Dict], func_adapt_returned_config : Callable[[Tuple[str, Dict]], Tuple[str, Dict]] = lambda x: x, @@ -34,9 +56,7 @@ def check_expected_config( num_retry = 0 return_data = None while num_retry < max_retries: - LOGGER.info('resources_to_get = {:s}'.format(str(resources_to_get))) - results_getconfig = driver.GetConfig(resources_to_get) - LOGGER.info('results_getconfig = {:s}'.format(str(results_getconfig))) + results_getconfig = get_config(driver, resources_to_get) return_data = copy.deepcopy(results_getconfig) results_getconfig = [ @@ -49,7 +69,7 @@ def check_expected_config( if num_diffs == 0: break # let the device take some time to reconfigure time.sleep(retry_delay) - num_retry -= 1 + num_retry += 1 if num_diffs > 0: LOGGER.error('Differences[{:d}]:\n{:s}'.format(num_diffs, str(diff_data.pretty()))) assert num_diffs == 0 diff --git a/src/device/tests/gnmi_openconfig/tools/request_composers.py b/src/device/tests/gnmi_openconfig/tools/request_composers.py index faa8425c8..be0587101 100644 --- a/src/device/tests/gnmi_openconfig/tools/request_composers.py +++ b/src/device/tests/gnmi_openconfig/tools/request_composers.py @@ -29,10 +29,10 @@ def network_instance(ni_name, ni_type) -> Tuple[str, Dict]: } return str_path, str_data -def network_instance_static_route(ni_name, prefix, next_hop, next_hop_index=0, metric=1) -> Tuple[str, Dict]: +def network_instance_static_route(ni_name, prefix, next_hop_index, next_hop, metric=1) -> Tuple[str, Dict]: str_path = '/network_instance[{:s}]/static_route[{:s}]'.format(ni_name, prefix) str_data = { - 'name': ni_name, 'prefix': prefix, 'next_hop': next_hop, 'next_hop_index': next_hop_index, 'metric': metric + 'name': ni_name, 'prefix': prefix, 'next_hop_index': next_hop_index, 'next_hop': next_hop, 'metric': metric } return str_path, str_data -- GitLab From c6f443a5d825bfda0bdaa2a34db283d30a018da9 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Mon, 15 Jan 2024 13:48:14 +0000 Subject: [PATCH 021/941] Device component - GNMI OpenConfig: - Added TODO.txt --- .../service/drivers/gnmi_openconfig/TODO.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/device/service/drivers/gnmi_openconfig/TODO.txt diff --git a/src/device/service/drivers/gnmi_openconfig/TODO.txt b/src/device/service/drivers/gnmi_openconfig/TODO.txt new file mode 100644 index 000000000..ba8ff1c2c --- /dev/null +++ b/src/device/service/drivers/gnmi_openconfig/TODO.txt @@ -0,0 +1,15 @@ +- update parse() @ InterfaceCounter.py +- update compose() @ NetworkInstance.py +- update compose() @ NetworkInstanceInterface.py +- implement parse() @ NetworkInstanceInterface.py +- update compose() @ NetworkInstanceStaticRoute.py +- implement parse() @ NetworkInstanceStaticRoute.py +- Fix MonitoringThread.py + + +there is an error removing static routes that makes unitary tests to crash +uncomment commented check_config_network_instance and validate + +- implement L2 VPN with BGP +- implement L3 VPN with BGP +- test static routes with ping -- GitLab From c4b613b68b5cc253fb652c3e47fa1f103cef96aa Mon Sep 17 00:00:00 2001 From: armingol Date: Wed, 14 Feb 2024 10:54:44 +0100 Subject: [PATCH 022/941] first version --- manifests/sliceservice.yaml | 2 +- src/common/tools/descriptor/Loader.py | 5 ++- src/common/tools/descriptor/Tools.py | 11 +++-- .../drivers/openconfig/templates/Inventory.py | 33 ++++++++++++++- .../drivers/openconfig/templates/Tools.py | 5 +++ src/slice/service/SliceServiceServicerImpl.py | 41 +++++++++++-------- .../service/slice_grouper/SliceGrouper.py | 28 ++++++------- src/webui/service/main/routes.py | 4 +- 8 files changed, 90 insertions(+), 39 deletions(-) diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index e7e5c1604..61f5b1d21 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "INFO" + value: "DEBUG" - name: SLICE_GROUPING value: "DISABLE" envFrom: diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 916a73d30..9e536f935 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -104,7 +104,7 @@ class DescriptorLoader: self.__devices = self.__descriptors.get('devices' , []) self.__links = self.__descriptors.get('links' , []) self.__services = self.__descriptors.get('services' , []) - self.__slices = self.__descriptors.get('slices' , []) + self.__slices = self.__descriptors.get('data', []) #Coge de la file el campo slices self.__connections = self.__descriptors.get('connections', []) self.__contexts_add = None @@ -194,7 +194,7 @@ class DescriptorLoader: _slices = {} for slice_ in self.__slices: context_uuid = slice_['slice_id']['context_id']['context_uuid']['uuid'] - _slices.setdefault(context_uuid, []).append(slice_) + _slices.setdefault(context_uuid, []).append(slice_) #no tenemos context_uuid en este formato, lo meto a mano? return _slices @property @@ -215,6 +215,7 @@ class DescriptorLoader: # Format CustomConfigRules in Devices, Services and Slices provided in JSON format self.__devices = [format_device_custom_config_rules (device ) for device in self.__devices ] self.__services = [format_service_custom_config_rules(service) for service in self.__services] + LOGGERS.INFO(self.__slices) self.__slices = [format_slice_custom_config_rules (slice_ ) for slice_ in self.__slices ] # Context and Topology require to create the entity first, and add devices, links, services, diff --git a/src/common/tools/descriptor/Tools.py b/src/common/tools/descriptor/Tools.py index f03c635b8..1811f77d9 100644 --- a/src/common/tools/descriptor/Tools.py +++ b/src/common/tools/descriptor/Tools.py @@ -15,6 +15,7 @@ import copy, json from typing import Dict, List, Optional, Tuple, Union +#context es la db, al inicio esta vacía def get_descriptors_add_contexts(contexts : List[Dict]) -> List[Dict]: contexts_add = copy.deepcopy(contexts) for context in contexts_add: @@ -52,7 +53,7 @@ def get_descriptors_add_slices(slices : List[Dict]) -> List[Dict]: TypeResourceValue = Union[str, int, bool, float, dict, list] def format_custom_config_rules(config_rules : List[Dict]) -> List[Dict]: for config_rule in config_rules: - if 'custom' not in config_rule: continue + # if 'custom' not in config_rule: continue #suponemos que siempre son custom, quitamos esta linea custom_resource_value : TypeResourceValue = config_rule['custom']['resource_value'] if isinstance(custom_resource_value, (dict, list)): custom_resource_value = json.dumps(custom_resource_value, sort_keys=True, indent=0) @@ -71,10 +72,14 @@ def format_service_custom_config_rules(service : Dict) -> Dict: service['service_config']['config_rules'] = config_rules return service +#UTILIZA LA FUNCION FORMAT_CUSTOM_CONFIG_RULES +#cambio def format_slice_custom_config_rules(slice_ : Dict) -> Dict: - config_rules = slice_.get('slice_config', {}).get('config_rules', []) + #donde cojo los config_rules + #las config_rules parecen estar en ACs? + config_rules = slice_.get('sdps', []) config_rules = format_custom_config_rules(config_rules) - slice_['slice_config']['config_rules'] = config_rules + slice_['sdps']['sdp']['attachment-circuits'] = config_rules return slice_ def split_devices_by_rules(devices : List[Dict]) -> Tuple[List[Dict], List[Dict]]: diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index 2ae67ba47..01b417739 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -15,7 +15,7 @@ import logging, lxml.etree as ET from typing import Any, Dict, List, Tuple from .Namespace import NAMESPACES -from .Tools import add_value_from_tag +from .Tools import add_value_from_tag, add_int_from_tag LOGGER = logging.getLogger(__name__) @@ -56,6 +56,8 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] LOGGER.debug("InventoryPrueba") parent_types = {} + #Initialized count to 0 for index + count = 0 for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) inventory = {} @@ -64,6 +66,7 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: inventory['class'] = '' inventory['attributes'] = {} component_reference = [] + component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) if component_name is None or component_name.text is None: continue @@ -84,6 +87,34 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue + ##Added (after the checking of the name and the class) + #Physical index- Index of the component in the array + + add_int_from_tag(inventory['attributes'], 'physical-index', count) + count +=1 + + ##Added + #FRU + if inventory['class'] == 'FRU': + component_isfru = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) + add_value_from_tag(inventory['attributes'], 'isfru', component_isfru) + ##ID + component_id = xml_component.find('ocp:state/ocp:id', namespaces=NAMESPACES) + if not component_id is None: + add_value_from_tag(inventory['attributes'], 'id', component_id) + + ##OPER_STATUS + component_oper_status = xml_component.find('ocp:state/ocp:oper-status', namespaces=NAMESPACES) + if not component_oper_status is None: + add_value_from_tag(inventory['attributes'], 'oper-status', component_oper_status) + + ##MODEL_ID + component_model_id = xml_component.find('ocp:state/ocp:entity-id', namespaces=NAMESPACES) + if not component_model_id is None: + add_value_from_tag(inventory['attributes'], 'model-id', component_model_id) + + ## + component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: add_value_from_tag(inventory['attributes'], 'empty', component_empty) diff --git a/src/device/service/drivers/openconfig/templates/Tools.py b/src/device/service/drivers/openconfig/templates/Tools.py index 79bebef51..78e61e0ae 100644 --- a/src/device/service/drivers/openconfig/templates/Tools.py +++ b/src/device/service/drivers/openconfig/templates/Tools.py @@ -26,6 +26,11 @@ def add_value_from_tag(target : Dict, field_name: str, field_value : ET.Element, if cast is not None: field_value = cast(field_value) target[field_name] = field_value +def add_int_from_tag(target : Dict, field_name: str, field_value : int, cast=None) -> None: + if field_value is None: return + if cast is not None: field_value = cast(field_value) + target[field_name] = field_value + def add_value_from_collection(target : Dict, field_name: str, field_value : Collection) -> None: if field_value is None or len(field_value) == 0: return target[field_name] = field_value diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index cbe2dd5c7..52552a6ed 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +#agrupar slices agrupar recursos para no hacer mas configs + from typing import Optional import grpc, json, logging #, deepdiff from common.proto.context_pb2 import ( @@ -30,33 +32,34 @@ from interdomain.client.InterdomainClient import InterdomainClient from service.client.ServiceClient import ServiceClient from .slice_grouper.SliceGrouper import SliceGrouper -LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) #crea un objeto de registro con el nombre del modulo actual METRICS_POOL = MetricsPool('Slice', 'RPC') -class SliceServiceServicerImpl(SliceServiceServicer): +class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio gRPC definido por SliceServiceServicer def __init__(self): LOGGER.debug('Creating Servicer...') - self._slice_grouper = SliceGrouper() + self._slice_grouper = SliceGrouper() #crea una instancia de slicegrouper LOGGER.debug('Servicer Created') def create_update(self, request : Slice) -> SliceId: # Set slice status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the slice is # being modified. context_client = ContextClient() - slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) + slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) # se obtiene la slice con el sliceId de la req - slice_rw = Slice() + slice_rw = Slice() #crea nueva slice desde la slice de la req slice_rw.CopyFrom(request if slice_ro is None else slice_ro) - if len(request.name) > 0: slice_rw.name = request.name + if len(request.name) > 0: slice_rw.name = request.name #actualizamos el nombre y estado de la slice rw slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member +#copiamos endpoints, reglas y configuraciones de la req a la slice copy_endpoint_ids(request.slice_endpoint_ids, slice_rw.slice_endpoint_ids ) # pylint: disable=no-member copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member - slice_id_with_uuids = context_client.SetSlice(slice_rw) + slice_id_with_uuids = context_client.SetSlice(slice_rw) #actualizar o crear la slice en la db if len(slice_rw.slice_endpoint_ids) < 2: # pylint: disable=no-member # unable to identify the kind of slice; just update endpoints, constraints and config rules @@ -65,8 +68,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): reply = context_client.SetSlice(slice_rw) context_client.close() return reply + #si tiene menos de 2 endpoints se omite la actualizacion y se retorna el sliceid - slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) #obtenemos la slice actualizada #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) @@ -74,8 +78,8 @@ class SliceServiceServicerImpl(SliceServiceServicer): #changes = deepdiff.DeepDiff(json_current_slice, json_updated_slice) #LOGGER.info('changes = {:s}'.format(str(changes))) - if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): - interdomain_client = InterdomainClient() + if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): #si la slice es interdominio + interdomain_client = InterdomainClient() #que es interdomain client? slice_id = interdomain_client.RequestSlice(slice_with_uuids) slice_ = context_client.GetSlice(slice_id) slice_active = Slice() @@ -97,10 +101,10 @@ class SliceServiceServicerImpl(SliceServiceServicer): service_client = ServiceClient() try: - _service = context_client.GetService(service_id) + _service = context_client.GetService(service_id) #obtener info de un servicio si existe except: # pylint: disable=bare-except # pylint: disable=no-member - service_request = Service() + service_request = Service() # sino se crea un nuevo servicio service_request.service_id.CopyFrom(service_id) service_request.service_type = ServiceTypeEnum.SERVICETYPE_UNKNOWN service_request.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED @@ -109,6 +113,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): service_request = Service() service_request.CopyFrom(_service) +#actualiza el servicio con la info de la slice # pylint: disable=no-member copy_endpoint_ids(request.slice_endpoint_ids, service_request.service_endpoint_ids) copy_constraints(request.slice_constraints, service_request.service_constraints) @@ -162,11 +167,11 @@ class SliceServiceServicerImpl(SliceServiceServicer): slice_active.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_ACTIVE # pylint: disable=no-member context_client.SetSlice(slice_active) - service_client.close() - context_client.close() + service_client.close() #liberar recursos, que es realmente? + context_client.close() #db teraflow return slice_id - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) #agrega funcionalidades de metrica y seguridad def CreateSlice(self, request : Slice, context : grpc.ServicerContext) -> SliceId: #try: # slice_ = context_client.GetSlice(request.slice_id) @@ -196,7 +201,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteSlice(self, request : SliceId, context : grpc.ServicerContext) -> Empty: - context_client = ContextClient() + context_client = ContextClient() #coge la info de una slice try: _slice = context_client.GetSlice(request) except: # pylint: disable=bare-except @@ -205,9 +210,11 @@ class SliceServiceServicerImpl(SliceServiceServicer): _slice_rw = Slice() _slice_rw.CopyFrom(_slice) + #cambia el status _slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(_slice_rw) - +#elimina la slice considerando si es interdominio o no, y desagrupa la slice eliminada + #elimina los servicios asociados a la slice if is_inter_domain(context_client, _slice.slice_endpoint_ids): interdomain_client = InterdomainClient() slice_id = interdomain_client.DeleteSlice(request) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index 2f1a79181..d59531a1b 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -14,7 +14,7 @@ import logging, pandas, threading from typing import Dict, Optional, Tuple -from sklearn.cluster import KMeans +from sklearn.cluster import KMeans #algoritmo de agrupamiento de scikit-learn (biblio de aprendizaje automatico) from common.proto.context_pb2 import Slice from common.tools.grpc.Tools import grpc_message_to_json_string from .Constants import SLICE_GROUPS @@ -27,30 +27,30 @@ LOGGER = logging.getLogger(__name__) class SliceGrouper: def __init__(self) -> None: - self._lock = threading.Lock() - self._is_enabled = is_slice_grouping_enabled() + self._lock = threading.Lock() #controla el acceso concurrente + self._is_enabled = is_slice_grouping_enabled() #esta habilitado el agrupamiento de slices? LOGGER.info('Slice Grouping: {:s}'.format('ENABLED' if self._is_enabled else 'DISABLED')) if not self._is_enabled: return - metrics_exporter = MetricsExporter() + metrics_exporter = MetricsExporter() #instancia de la clase metrics_exporter.create_table() - self._slice_groups = create_slice_groups(SLICE_GROUPS) + self._slice_groups = create_slice_groups(SLICE_GROUPS) #grupos de slices # Initialize and fit K-Means with the pre-defined clusters we want, i.e., one per slice group - df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) - k_means = KMeans(n_clusters=df_groups.shape[0]) + df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) #data frame con info de los grupos + k_means = KMeans(n_clusters=df_groups.shape[0]) #modelo utilizado para el agrupamiento k_means.fit(df_groups[['availability', 'capacity_gbps']]) df_groups['label'] = k_means.predict(df_groups[['availability', 'capacity_gbps']]) self._k_means = k_means self._df_groups = df_groups - self._group_mapping : Dict[str, Dict] = { + self._group_mapping : Dict[str, Dict] = { #Dict = dictionary group['name']:{k:v for k,v in group.items() if k != 'name'} - for group in list(df_groups.to_dict('records')) + for group in list(df_groups.to_dict('records')) #mapeo de nombres de grupo a sus atributos } - label_to_group = {} + label_to_group = {} #mapeo de etiquetas a nombres de grupo for group_name,group_attrs in self._group_mapping.items(): label = group_attrs['label'] availability = group_attrs['availability'] @@ -60,7 +60,7 @@ class SliceGrouper: label_to_group[label] = group_name self._label_to_group = label_to_group - def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: + def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: #selecciona un grupo para una slice with self._lock: grouping_parameters = get_slice_grouping_parameters(slice_obj) LOGGER.debug('[_select_group] grouping_parameters={:s}'.format(str(grouping_parameters))) @@ -78,16 +78,16 @@ class SliceGrouper: return group_name, availability, capacity_gbps @property - def is_enabled(self): return self._is_enabled + def is_enabled(self): return self._is_enabled #indica si el agrupamiento de slices esta habilitado - def group(self, slice_obj : Slice) -> bool: + def group(self, slice_obj : Slice) -> bool: #determina el grupo al que debe pertenecer la slice LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[group] selected_group={:s}'.format(str(selected_group))) if selected_group is None: return False return add_slice_to_group(slice_obj, selected_group) - def ungroup(self, slice_obj : Slice) -> bool: + def ungroup(self, slice_obj : Slice) -> bool: # desagrupa la slice de un grupo LOGGER.debug('[ungroup] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[ungroup] selected_group={:s}'.format(str(selected_group))) diff --git a/src/webui/service/main/routes.py b/src/webui/service/main/routes.py index 75f036bef..eba758ff3 100644 --- a/src/webui/service/main/routes.py +++ b/src/webui/service/main/routes.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64, json, logging #, re +import base64, json, logging +import traceback #, re from flask import jsonify, redirect, render_template, Blueprint, flash, session, url_for, request from common.proto.context_pb2 import ContextList, Empty, TopologyId, TopologyList from common.tools.descriptor.Loader import DescriptorLoader, compose_notifications @@ -113,6 +114,7 @@ def home(): except Exception as e: # pylint: disable=broad-except LOGGER.exception('Descriptor load failed') flash(f'Descriptor load failed: `{str(e)}`', 'danger') + traceback.print_exc() # Agregar esta línea para imprimir el traceback completo finally: context_client.close() device_client.close() -- GitLab From 0b89701b198e73126e8419c571ea314eb21b26c5 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Thu, 22 Feb 2024 12:16:07 +0000 Subject: [PATCH 023/941] Dataplane-in-a-box: - Updated Arista cEOS image version - Updated README.md --- dataplane-in-a-box/README.md | 1 + dataplane-in-a-box/arista.clab.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 45e5dc5e0..603b24114 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -46,6 +46,7 @@ docker exec -it clab-arista-wan1 bash ## Access cEOS CLI ```bash docker exec -it clab-arista-wan1 Cli +docker exec -it clab-arista-wan2 Cli ``` ## Configure ContainerLab clients diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 4f3b77129..3a92c7e9b 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -25,7 +25,7 @@ topology: arista_ceos: kind: arista_ceos #image: ceos:4.30.4M - image: ceos:4.31.1F + image: ceos:4.31.2F linux: kind: linux image: ghcr.io/hellt/network-multitool:latest -- GitLab From 5696254239b649a49c6155a618f7b54a928c786f Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Fri, 1 Mar 2024 18:27:51 +0000 Subject: [PATCH 024/941] Dataplane-in-a-box: - Updated ContainerLab scenario - Updated TFS descriptors - Updated scripts - Updated README.md --- dataplane-in-a-box/README.md | 6 + dataplane-in-a-box/arista.clab.yml | 26 +++- dataplane-in-a-box/clab-cli-dc1.sh | 3 + dataplane-in-a-box/clab-cli-dc2.sh | 3 + .../{ceos-cli-wan1.sh => clab-cli-wan1.sh} | 0 .../{ceos-cli-wan2.sh => clab-cli-wan2.sh} | 0 dataplane-in-a-box/clab-cli-wan3.sh | 3 + dataplane-in-a-box/clab-load-image.sh | 19 --- dataplane-in-a-box/clab-pull-images.sh | 18 --- dataplane-in-a-box/links.json | 136 ------------------ .../{topology.json => tfs-01-topo-nodes.json} | 59 ++------ dataplane-in-a-box/tfs-02-topo-links.json | 63 ++++++++ dataplane-in-a-box/tfs-03-dc2dc-l2svc.json | 17 +++ ...3-service.json => tfs-04-dc2dc-l3svc.json} | 18 +-- 14 files changed, 139 insertions(+), 232 deletions(-) create mode 100755 dataplane-in-a-box/clab-cli-dc1.sh create mode 100755 dataplane-in-a-box/clab-cli-dc2.sh rename dataplane-in-a-box/{ceos-cli-wan1.sh => clab-cli-wan1.sh} (100%) rename dataplane-in-a-box/{ceos-cli-wan2.sh => clab-cli-wan2.sh} (100%) create mode 100755 dataplane-in-a-box/clab-cli-wan3.sh delete mode 100755 dataplane-in-a-box/clab-load-image.sh delete mode 100755 dataplane-in-a-box/clab-pull-images.sh delete mode 100644 dataplane-in-a-box/links.json rename dataplane-in-a-box/{topology.json => tfs-01-topo-nodes.json} (50%) create mode 100644 dataplane-in-a-box/tfs-02-topo-links.json create mode 100644 dataplane-in-a-box/tfs-03-dc2dc-l2svc.json rename dataplane-in-a-box/{dc-2-dc-l3-service.json => tfs-04-dc2dc-l3svc.json} (72%) diff --git a/dataplane-in-a-box/README.md b/dataplane-in-a-box/README.md index 603b24114..4dd22dec3 100644 --- a/dataplane-in-a-box/README.md +++ b/dataplane-in-a-box/README.md @@ -19,6 +19,12 @@ source dataplane-in-a-box/deploy_specs.sh sudo bash -c "$(curl -sL https://get.containerlab.dev)" -- -v 0.48.4 ``` +## Download Arista cEOS image and create Docker image +```bash +cd ~/tfs-ctrl/dataplane-in-a-box +docker import arista/cEOS64-lab-4.31.2F.tar ceos:4.31.2F +``` + ## Deploy scenario ```bash cd ~/tfs-ctrl/dataplane-in-a-box diff --git a/dataplane-in-a-box/arista.clab.yml b/dataplane-in-a-box/arista.clab.yml index 3a92c7e9b..2865100bd 100644 --- a/dataplane-in-a-box/arista.clab.yml +++ b/dataplane-in-a-box/arista.clab.yml @@ -35,21 +35,37 @@ topology: kind: arista_ceos mgmt-ipv4: 172.20.20.101 ports: [6001:6030] + wan2: kind: arista_ceos mgmt-ipv4: 172.20.20.102 ports: [6002:6030] - client1: + wan3: + kind: arista_ceos + mgmt-ipv4: 172.20.20.103 + ports: [6003:6030] + + dc1: kind: linux mgmt-ipv4: 172.20.20.201 ports: [2201:22] - client2: + exec: + - ip link set address 00:c1:ab:00:00:01 dev eth1 + - ip address add 192.168.1.10/24 dev eth1 + - ip route add 192.168.2.0/24 via 192.168.1.1 + + dc2: kind: linux mgmt-ipv4: 172.20.20.202 ports: [2202:22] + exec: + - ip link set address 00:c1:ab:00:00:02 dev eth1 + - ip address add 192.168.2.10/24 dev eth1 + - ip route add 192.168.1.0/24 via 192.168.2.1 links: - - endpoints: ["wan1:eth1", "wan2:eth1"] - - endpoints: ["client1:eth1", "wan1:eth10"] - - endpoints: ["client2:eth1", "wan2:eth10"] + - endpoints: ["wan1:eth2", "wan2:eth1"] + - endpoints: ["wan2:eth3", "wan3:eth2"] + - endpoints: ["wan1:eth10", "dc1:eth1"] + - endpoints: ["wan3:eth10", "dc2:eth1"] diff --git a/dataplane-in-a-box/clab-cli-dc1.sh b/dataplane-in-a-box/clab-cli-dc1.sh new file mode 100755 index 000000000..7d793f035 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-dc1.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-dc1 bash diff --git a/dataplane-in-a-box/clab-cli-dc2.sh b/dataplane-in-a-box/clab-cli-dc2.sh new file mode 100755 index 000000000..311d6dae5 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-dc2.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-dc2 bash diff --git a/dataplane-in-a-box/ceos-cli-wan1.sh b/dataplane-in-a-box/clab-cli-wan1.sh similarity index 100% rename from dataplane-in-a-box/ceos-cli-wan1.sh rename to dataplane-in-a-box/clab-cli-wan1.sh diff --git a/dataplane-in-a-box/ceos-cli-wan2.sh b/dataplane-in-a-box/clab-cli-wan2.sh similarity index 100% rename from dataplane-in-a-box/ceos-cli-wan2.sh rename to dataplane-in-a-box/clab-cli-wan2.sh diff --git a/dataplane-in-a-box/clab-cli-wan3.sh b/dataplane-in-a-box/clab-cli-wan3.sh new file mode 100755 index 000000000..c931ac940 --- /dev/null +++ b/dataplane-in-a-box/clab-cli-wan3.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it clab-arista-wan2 Cli diff --git a/dataplane-in-a-box/clab-load-image.sh b/dataplane-in-a-box/clab-load-image.sh deleted file mode 100755 index 87e666422..000000000 --- a/dataplane-in-a-box/clab-load-image.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download image from Arista account > software downloads - -cd /home/tfs/tfs-ctrl/dataplane-in-a-box -docker import cEOS64-lab-4.30.4M.tar ceos:4.30.4M diff --git a/dataplane-in-a-box/clab-pull-images.sh b/dataplane-in-a-box/clab-pull-images.sh deleted file mode 100755 index 8f2805c6b..000000000 --- a/dataplane-in-a-box/clab-pull-images.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker pull ghcr.io/hellt/network-multitool:latest -#docker pull ghcr.io/nokia/srlinux:23.7.2 -#docker pull netreplica/docker-sonic-vs:20220111 diff --git a/dataplane-in-a-box/links.json b/dataplane-in-a-box/links.json deleted file mode 100644 index 832a24fdd..000000000 --- a/dataplane-in-a-box/links.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "contexts": [ - {"context_id": {"context_uuid": {"uuid": "admin"}}} - ], - "topologies": [ - {"topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, "topology_uuid": {"uuid": "admin"}}} - ], - "devices": [ - { - "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": true - }}} - ]} - } - ], - "links": [ - { - "link_id": {"link_uuid": {"uuid": "DC1/eth1==WAN1/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/2==DC1/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "DC1"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "WAN1/ethernet-1/1==WAN2/ethernet-1/1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/1==WAN1/ethernet-1/1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN1"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} - ] - }, - - { - "link_id": {"link_uuid": {"uuid": "DC2/eth1==WAN2/ethernet-1/2"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}}, - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} - ] - }, - { - "link_id": {"link_uuid": {"uuid": "WAN2/ethernet-1/2==DC2/eth1"}}, - "link_endpoint_ids": [ - {"device_id": {"device_uuid": {"uuid": "WAN2"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, - {"device_id": {"device_uuid": {"uuid": "DC2"}}, "endpoint_uuid": {"uuid": "eth1"}} - ] - } - ] -} diff --git a/dataplane-in-a-box/topology.json b/dataplane-in-a-box/tfs-01-topo-nodes.json similarity index 50% rename from dataplane-in-a-box/topology.json rename to dataplane-in-a-box/tfs-01-topo-nodes.json index 42752235d..3b5e42b99 100644 --- a/dataplane-in-a-box/topology.json +++ b/dataplane-in-a-box/tfs-01-topo-nodes.json @@ -7,85 +7,54 @@ ], "devices": [ { - "device_id": {"device_uuid": {"uuid": "DC1"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc1"}}, "device_type": "emu-datacenter", "device_drivers": [0], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "DC2"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "dc2"}}, "device_type": "emu-datacenter", "device_drivers": [0], "device_config": {"config_rules": [ {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} + {"uuid": "eth1", "type": "copper"}, {"uuid": "int", "type": "copper"} ]}}} ]} }, { - "device_id": {"device_uuid": {"uuid": "DC3"}}, "device_type": "emu-datacenter", "device_drivers": [0], + "device_id": {"device_uuid": {"uuid": "wan1"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "DC4"}}, "device_type": "emu-datacenter", "device_drivers": [0], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "127.0.0.1"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "0"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": {"endpoints": [ - {"uuid": "eth1", "type": "copper"}, {"uuid": "eth2", "type": "copper"}, {"uuid": "int", "type": "copper"} - ]}}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6001"}}, - {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { - "username": "admin", "password": "admin", "use_tls": false - }}} - ]} - }, - { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], - "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6002"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.101"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "WAN1"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "wan2"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6003"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.102"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} }, { - "device_id": {"device_uuid": {"uuid": "WAN2"}}, "device_type": "packet-router", "device_drivers": [8], + "device_id": {"device_uuid": {"uuid": "wan3"}}, "device_type": "packet-router", "device_drivers": [8], "device_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "10.1.7.200"}}, - {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6004"}}, + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.20.20.103"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "6030"}}, {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { "username": "admin", "password": "admin", "use_tls": false }}} ]} } - ], - "links": [] + ] } diff --git a/dataplane-in-a-box/tfs-02-topo-links.json b/dataplane-in-a-box/tfs-02-topo-links.json new file mode 100644 index 000000000..b9070dd9d --- /dev/null +++ b/dataplane-in-a-box/tfs-02-topo-links.json @@ -0,0 +1,63 @@ +{ + "links": [ + { + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/2==wan2/ethernet-1/1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/1==wan1/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/1"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "wan2/ethernet-1/3==wan3/ethernet-1/2"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}}, + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/2==wan2/ethernet-1/3"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/2"}}, + {"device_id": {"device_uuid": {"uuid": "wan2"}}, "endpoint_uuid": {"uuid": "ethernet-1/3"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "dc1/eth1==wan1/ethernet-1/10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan1/ethernet-1/10==dc1/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan1"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + }, + + { + "link_id": {"link_uuid": {"uuid": "dc2/eth1==wan3/ethernet-1/10"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}}, + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}} + ] + }, + { + "link_id": {"link_uuid": {"uuid": "wan3/ethernet-1/10==dc2/eth1"}}, + "link_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "wan3"}}, "endpoint_uuid": {"uuid": "ethernet-1/10"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "eth1"}} + ] + } + ] +} diff --git a/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json new file mode 100644 index 000000000..8d10e5f4b --- /dev/null +++ b/dataplane-in-a-box/tfs-03-dc2dc-l2svc.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "service_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l2svc"} + }, + "service_type": 2, + "service_status": {"service_status": 1}, + "service_endpoint_ids": [ + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} + ], + "service_constraints": [], + "service_config": {"config_rules": []} + } + ] +} diff --git a/dataplane-in-a-box/dc-2-dc-l3-service.json b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json similarity index 72% rename from dataplane-in-a-box/dc-2-dc-l3-service.json rename to dataplane-in-a-box/tfs-04-dc2dc-l3svc.json index cb9ef972e..b21cba0da 100644 --- a/dataplane-in-a-box/dc-2-dc-l3-service.json +++ b/dataplane-in-a-box/tfs-04-dc2dc-l3svc.json @@ -2,33 +2,33 @@ "services": [ { "service_id": { - "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc-2-dc-l3-svc"} + "context_id": {"context_uuid": {"uuid": "admin"}}, "service_uuid": {"uuid": "dc2dc-l3svc"} }, "service_type": 1, "service_status": {"service_status": 1}, "service_endpoint_ids": [ - {"device_id":{"device_uuid":{"uuid":"DC1"}},"endpoint_uuid":{"uuid":"int"}}, - {"device_id":{"device_uuid":{"uuid":"DC2"}},"endpoint_uuid":{"uuid":"int"}} + {"device_id": {"device_uuid": {"uuid": "dc1"}}, "endpoint_uuid": {"uuid": "int"}}, + {"device_id": {"device_uuid": {"uuid": "dc2"}}, "endpoint_uuid": {"uuid": "int"}} ], "service_constraints": [], "service_config": {"config_rules": [ - {"action": 1, "custom": {"resource_key": "/device[SRL1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/settings", "resource_value": { "static_routes": [{"prefix": "172.16.2.0/24", "next_hop": "172.0.0.2"}] }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/1]/settings", "resource_value": { "ipv4_address": "172.0.0.1", "ipv4_prefix": 30, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL1]/endpoint[ethernet-1/2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan1]/endpoint[ethernet-1/2]/settings", "resource_value": { "ipv4_address": "172.16.1.1", "ipv4_prefix": 24, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/settings", "resource_value": { "static_routes": [{"prefix": "172.16.1.0/24", "next_hop": "172.0.0.1"}] }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/1]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/1]/settings", "resource_value": { "ipv4_address": "172.0.0.2", "ipv4_prefix": 30, "sub_interface_index": 0 }}}, - {"action": 1, "custom": {"resource_key": "/device[SRL2]/endpoint[ethernet-1/2]/settings", "resource_value": { + {"action": 1, "custom": {"resource_key": "/device[wan3]/endpoint[ethernet-1/2]/settings", "resource_value": { "ipv4_address": "172.16.2.1", "ipv4_prefix": 24, "sub_interface_index": 0 }}} ]} -- GitLab From 7def79c1209173d0be26bf9dc7e493ad8e544fda Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 15:01:11 +0100 Subject: [PATCH 025/941] Slices with IETF format --- src/common/tools/descriptor/Loader.py | 97 +++++++++++++++++++++------ src/common/tools/descriptor/Tools.py | 13 +--- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 24f45aeb6..b2eb75045 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -105,9 +105,69 @@ class DescriptorLoader: self.__devices = self.__descriptors.get('devices' , []) self.__links = self.__descriptors.get('links' , []) self.__services = self.__descriptors.get('services' , []) - self.__slices = self.__descriptors.get('data', []) #Coge de la file el campo slices + self.__slices = self.__descriptors.get('slices' , []) + self.__slices = self.__descriptors.get('ietf-network-slice-service:network-slice-services', {}) self.__connections = self.__descriptors.get('connections', []) + if self.__slices: + json_out = {"slices": [ + { + "slice_id": { + "context_id": {"context_uuid": {"uuid": "admin"}}, + "slice_uuid": {} + }, + "name": {}, + "slice_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "/settings", "resource_value": { + "address_families": ["IPV4"], "bgp_as": 65000, "bgp_route_target": "65000:333", "mtu": 1512 + }}} + ]}, + "slice_constraints": [ + {"sla_capacity": {"capacity_gbps": 20.0}}, + {"sla_availability": {"availability": 20.0, "num_disjoint_paths": 1, "all_active": True}}, + {"sla_isolation": {"isolation_level": [0]}} + ], + "slice_endpoint_ids": [ + + ], + "slice_status": {"slice_status": 1} + } + ]} + + for slice_service in self.__slices["slice-service"]: + for slice in json_out["slices"]: + slice["slice_id"]["slice_uuid"] = { "uuid": slice_service["id"]} + slice["name"] = slice_service["description"] + sdp = slice_service["sdps"]["sdp"] + for elemento in sdp: + attcircuits = elemento["attachment-circuits"]["attachment-circuit"] + for attcircuit in attcircuits: + resource_key = "/device[{sdp_id}]/endpoint[{endpoint_id}]/settings".format(sdp_id = elemento["id"], endpoint_id = attcircuit["ac-tp-id"]) + + for tag in attcircuit['ac-tags']['ac-tag']: + if tag.get('tag-type') == 'ietf-nss:vlan-id': + vlan_id = tag.get('value') + else: + vlan_id = 0 + + slice["slice_config"]["config_rules"].append( {"action": 1, "custom": {"resource_key": resource_key, "resource_value": { + "router_id": elemento.get("node-id",[]), "sub_interface_index": 0, "vlan_id": vlan_id + }}}) + slice["slice_endpoint_ids"].append({ + "device_id": {"device_uuid": {"uuid": elemento["id"]}}, + "endpoint_uuid": {"uuid": attcircuit["ac-tp-id"]}, + "topology_id": {"context_id": {"context_uuid": {"uuid": "admin"}}, + "topology_uuid": {"uuid": "admin"}} + }) + slice["slice_constraints"].append({"endpoint_location": { + "endpoint_id": {"device_id": {"device_uuid": {"uuid": elemento["id"]}}, "endpoint_uuid": {"uuid": attcircuit["ac-tp-id"]}}, + "location": {"region": "4"} + }}) + + # Convertir a JSON de salida + #json_output = json.dumps(json_out, indent=2) + self.__slices = json_out.get('slices' , []) + self.__contexts_add = None self.__topologies_add = None self.__devices_add = None @@ -216,7 +276,6 @@ class DescriptorLoader: # Format CustomConfigRules in Devices, Services and Slices provided in JSON format self.__devices = [format_device_custom_config_rules (device ) for device in self.__devices ] self.__services = [format_service_custom_config_rules(service) for service in self.__services] - LOGGERS.INFO(self.__slices) self.__slices = [format_slice_custom_config_rules (slice_ ) for slice_ in self.__slices ] # Context and Topology require to create the entity first, and add devices, links, services, @@ -237,8 +296,7 @@ class DescriptorLoader: self.__ctx_cli.connect() self._process_descr('context', 'add', self.__ctx_cli.SetContext, Context, self.__contexts_add ) self._process_descr('topology', 'add', self.__ctx_cli.SetTopology, Topology, self.__topologies_add) - self._process_descr('controller', 'add', self.__ctx_cli.SetDevice, Device, controllers ) - self._process_descr('device', 'add', self.__ctx_cli.SetDevice, Device, network_devices ) + self._process_descr('device', 'add', self.__ctx_cli.SetDevice, Device, self.__devices ) self._process_descr('link', 'add', self.__ctx_cli.SetLink, Link, self.__links ) self._process_descr('service', 'add', self.__ctx_cli.SetService, Service, self.__services ) self._process_descr('slice', 'add', self.__ctx_cli.SetSlice, Slice, self.__slices ) @@ -266,29 +324,24 @@ class DescriptorLoader: self.__services_add = get_descriptors_add_services(self.__services) self.__slices_add = get_descriptors_add_slices(self.__slices) - controllers_add, network_devices_add = split_controllers_and_network_devices(self.__devices_add) - self.__ctx_cli.connect() self.__dev_cli.connect() self.__svc_cli.connect() self.__slc_cli.connect() - self._process_descr('context', 'add', self.__ctx_cli.SetContext, Context, self.__contexts_add ) - self._process_descr('topology', 'add', self.__ctx_cli.SetTopology, Topology, self.__topologies_add) - self._process_descr('controller', 'add', self.__dev_cli.AddDevice, Device, controllers_add ) - self._process_descr('device', 'add', self.__dev_cli.AddDevice, Device, network_devices_add ) - self._process_descr('device', 'config', self.__dev_cli.ConfigureDevice, Device, self.__devices_config) - self._process_descr('link', 'add', self.__ctx_cli.SetLink, Link, self.__links ) - self._process_descr('service', 'add', self.__svc_cli.CreateService, Service, self.__services_add ) - self._process_descr('service', 'update', self.__svc_cli.UpdateService, Service, self.__services ) - self._process_descr('slice', 'add', self.__slc_cli.CreateSlice, Slice, self.__slices_add ) - self._process_descr('slice', 'update', self.__slc_cli.UpdateSlice, Slice, self.__slices ) - - # By default the Context component automatically assigns devices and links to topologies based on their - # endpoints, and assigns topologies, services, and slices to contexts based on their identifiers. - - # The following statement is useless; up to now, any use case requires assigning a topology, service, or - # slice to a different context. + self._process_descr('context', 'add', self.__ctx_cli.SetContext, Context, self.__contexts_add ) + self._process_descr('topology', 'add', self.__ctx_cli.SetTopology, Topology, self.__topologies_add) + self._process_descr('device', 'add', self.__dev_cli.AddDevice, Device, self.__devices_add ) + self._process_descr('device', 'config', self.__dev_cli.ConfigureDevice, Device, self.__devices_config) + self._process_descr('link', 'add', self.__ctx_cli.SetLink, Link, self.__links ) + self._process_descr('service', 'add', self.__svc_cli.CreateService, Service, self.__services_add ) + self._process_descr('service', 'update', self.__svc_cli.UpdateService, Service, self.__services ) + self._process_descr('slice', 'add', self.__slc_cli.CreateSlice, Slice, self.__slices_add ) + self._process_descr('slice', 'update', self.__slc_cli.UpdateSlice, Slice, self.__slices ) + + # Update context and topology is useless: + # - devices and links are assigned to topologies automatically by Context component + # - topologies, services, and slices are assigned to contexts automatically by Context component #self._process_descr('context', 'update', self.__ctx_cli.SetContext, Context, self.__contexts ) # In some cases, it might be needed to assign devices and links to multiple topologies; the diff --git a/src/common/tools/descriptor/Tools.py b/src/common/tools/descriptor/Tools.py index 11e79211f..ad1c402a0 100644 --- a/src/common/tools/descriptor/Tools.py +++ b/src/common/tools/descriptor/Tools.py @@ -16,7 +16,6 @@ import copy, json from typing import Dict, List, Optional, Tuple, Union from common.DeviceTypes import DeviceTypeEnum -#context es la db, al inicio esta vacía def get_descriptors_add_contexts(contexts : List[Dict]) -> List[Dict]: contexts_add = copy.deepcopy(contexts) for context in contexts_add: @@ -54,13 +53,11 @@ def get_descriptors_add_slices(slices : List[Dict]) -> List[Dict]: TypeResourceValue = Union[str, int, bool, float, dict, list] def format_custom_config_rules(config_rules : List[Dict]) -> List[Dict]: for config_rule in config_rules: - # if 'custom' not in config_rule: continue #suponemos que siempre son custom, quitamos esta linea + if 'custom' not in config_rule: continue custom_resource_value : TypeResourceValue = config_rule['custom']['resource_value'] if isinstance(custom_resource_value, (dict, list)): custom_resource_value = json.dumps(custom_resource_value, sort_keys=True, indent=0) config_rule['custom']['resource_value'] = custom_resource_value - elif not isinstance(custom_resource_value, str): - config_rule['custom']['resource_value'] = str(custom_resource_value) return config_rules def format_device_custom_config_rules(device : Dict) -> Dict: @@ -75,14 +72,10 @@ def format_service_custom_config_rules(service : Dict) -> Dict: service['service_config']['config_rules'] = config_rules return service -#UTILIZA LA FUNCION FORMAT_CUSTOM_CONFIG_RULES -#cambio def format_slice_custom_config_rules(slice_ : Dict) -> Dict: - #donde cojo los config_rules - #las config_rules parecen estar en ACs? - config_rules = slice_.get('sdps', []) + config_rules = slice_.get('slice_config', {}).get('config_rules', []) config_rules = format_custom_config_rules(config_rules) - slice_['sdps']['sdp']['attachment-circuits'] = config_rules + slice_['slice_config']['config_rules'] = config_rules return slice_ def split_devices_by_rules(devices : List[Dict]) -> Tuple[List[Dict], List[Dict]]: -- GitLab From 02d28cbc1cc70a1356edc13a8b6319159d9c97b4 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 17:33:28 +0100 Subject: [PATCH 026/941] code cleanup --- manifests/sliceservice.yaml | 2 +- .../drivers/openconfig/templates/Inventory.py | 33 +-------------- .../drivers/openconfig/templates/Tools.py | 5 --- src/slice/service/SliceServiceServicerImpl.py | 42 ++++++++----------- .../service/slice_grouper/SliceGrouper.py | 32 +++++++------- src/webui/service/main/routes.py | 4 +- 6 files changed, 36 insertions(+), 82 deletions(-) diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index 61f5b1d21..e7e5c1604 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -36,7 +36,7 @@ spec: - containerPort: 9192 env: - name: LOG_LEVEL - value: "DEBUG" + value: "INFO" - name: SLICE_GROUPING value: "DISABLE" envFrom: diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index e45958538..65562bc5b 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -15,7 +15,7 @@ import logging, lxml.etree as ET from typing import Any, Dict, List, Tuple from .Namespace import NAMESPACES -from .Tools import add_value_from_tag, add_int_from_tag +from .Tools import add_value_from_tag LOGGER = logging.getLogger(__name__) @@ -55,8 +55,6 @@ XPATH_PORTS = "//ocp:components/ocp:component" def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: response = [] parent_types = {} - #Initialized count to 0 for index - count = 0 for xml_component in xml_data.xpath(XPATH_PORTS, namespaces=NAMESPACES): LOGGER.info('xml_component inventario = {:s}'.format(str(ET.tostring(xml_component)))) inventory = {} @@ -65,7 +63,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: inventory['class'] = '' inventory['attributes'] = {} component_reference = [] - component_name = xml_component.find('ocp:name', namespaces=NAMESPACES) if component_name is None or component_name.text is None: continue @@ -85,34 +82,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory, 'class', component_type) if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue - - ##Added (after the checking of the name and the class) - #Physical index- Index of the component in the array - - add_int_from_tag(inventory['attributes'], 'physical-index', count) - count +=1 - - ##Added - #FRU - if inventory['class'] == 'FRU': - component_isfru = xml_component.find('ocp:state/ocp:type', namespaces=NAMESPACES) - add_value_from_tag(inventory['attributes'], 'isfru', component_isfru) - ##ID - component_id = xml_component.find('ocp:state/ocp:id', namespaces=NAMESPACES) - if not component_id is None: - add_value_from_tag(inventory['attributes'], 'id', component_id) - - ##OPER_STATUS - component_oper_status = xml_component.find('ocp:state/ocp:oper-status', namespaces=NAMESPACES) - if not component_oper_status is None: - add_value_from_tag(inventory['attributes'], 'oper-status', component_oper_status) - - ##MODEL_ID - component_model_id = xml_component.find('ocp:state/ocp:entity-id', namespaces=NAMESPACES) - if not component_model_id is None: - add_value_from_tag(inventory['attributes'], 'model-id', component_model_id) - - ## component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: diff --git a/src/device/service/drivers/openconfig/templates/Tools.py b/src/device/service/drivers/openconfig/templates/Tools.py index 78e61e0ae..79bebef51 100644 --- a/src/device/service/drivers/openconfig/templates/Tools.py +++ b/src/device/service/drivers/openconfig/templates/Tools.py @@ -26,11 +26,6 @@ def add_value_from_tag(target : Dict, field_name: str, field_value : ET.Element, if cast is not None: field_value = cast(field_value) target[field_name] = field_value -def add_int_from_tag(target : Dict, field_name: str, field_value : int, cast=None) -> None: - if field_value is None: return - if cast is not None: field_value = cast(field_value) - target[field_name] = field_value - def add_value_from_collection(target : Dict, field_name: str, field_value : Collection) -> None: if field_value is None or len(field_value) == 0: return target[field_name] = field_value diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index 52552a6ed..8a834f352 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -#agrupar slices agrupar recursos para no hacer mas configs - from typing import Optional import grpc, json, logging #, deepdiff from common.proto.context_pb2 import ( @@ -32,14 +30,13 @@ from interdomain.client.InterdomainClient import InterdomainClient from service.client.ServiceClient import ServiceClient from .slice_grouper.SliceGrouper import SliceGrouper -LOGGER = logging.getLogger(__name__) #crea un objeto de registro con el nombre del modulo actual - +LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('Slice', 'RPC') -class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio gRPC definido por SliceServiceServicer +class SliceServiceServicerImpl(SliceServiceServicer): def __init__(self): LOGGER.debug('Creating Servicer...') - self._slice_grouper = SliceGrouper() #crea una instancia de slicegrouper + self._slice_grouper = SliceGrouper() LOGGER.debug('Servicer Created') def create_update(self, request : Slice) -> SliceId: @@ -48,9 +45,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g context_client = ContextClient() slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) # se obtiene la slice con el sliceId de la req - slice_rw = Slice() #crea nueva slice desde la slice de la req + slice_rw = Slice() slice_rw.CopyFrom(request if slice_ro is None else slice_ro) - if len(request.name) > 0: slice_rw.name = request.name #actualizamos el nombre y estado de la slice rw + if len(request.name) > 0: slice_rw.name = request.name slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member @@ -59,7 +56,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member - slice_id_with_uuids = context_client.SetSlice(slice_rw) #actualizar o crear la slice en la db + slice_id_with_uuids = context_client.SetSlice(slice_rw) if len(slice_rw.slice_endpoint_ids) < 2: # pylint: disable=no-member # unable to identify the kind of slice; just update endpoints, constraints and config rules @@ -68,18 +65,17 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g reply = context_client.SetSlice(slice_rw) context_client.close() return reply - #si tiene menos de 2 endpoints se omite la actualizacion y se retorna el sliceid - - slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) #obtenemos la slice actualizada - + + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) + #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) #LOGGER.info('json_updated_slice = {:s}'.format(str(json_updated_slice))) #changes = deepdiff.DeepDiff(json_current_slice, json_updated_slice) #LOGGER.info('changes = {:s}'.format(str(changes))) - if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): #si la slice es interdominio - interdomain_client = InterdomainClient() #que es interdomain client? + if is_inter_domain(context_client, slice_with_uuids.slice_endpoint_ids): + interdomain_client = InterdomainClient() slice_id = interdomain_client.RequestSlice(slice_with_uuids) slice_ = context_client.GetSlice(slice_id) slice_active = Slice() @@ -101,10 +97,10 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g service_client = ServiceClient() try: - _service = context_client.GetService(service_id) #obtener info de un servicio si existe + _service = context_client.GetService(service_id) except: # pylint: disable=bare-except # pylint: disable=no-member - service_request = Service() # sino se crea un nuevo servicio + service_request = Service() service_request.service_id.CopyFrom(service_id) service_request.service_type = ServiceTypeEnum.SERVICETYPE_UNKNOWN service_request.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PLANNED @@ -113,7 +109,6 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g service_request = Service() service_request.CopyFrom(_service) -#actualiza el servicio con la info de la slice # pylint: disable=no-member copy_endpoint_ids(request.slice_endpoint_ids, service_request.service_endpoint_ids) copy_constraints(request.slice_constraints, service_request.service_constraints) @@ -167,11 +162,11 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g slice_active.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_ACTIVE # pylint: disable=no-member context_client.SetSlice(slice_active) - service_client.close() #liberar recursos, que es realmente? - context_client.close() #db teraflow + service_client.close() + context_client.close() return slice_id - @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) #agrega funcionalidades de metrica y seguridad + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def CreateSlice(self, request : Slice, context : grpc.ServicerContext) -> SliceId: #try: # slice_ = context_client.GetSlice(request.slice_id) @@ -201,7 +196,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def DeleteSlice(self, request : SliceId, context : grpc.ServicerContext) -> Empty: - context_client = ContextClient() #coge la info de una slice + context_client = ContextClient() try: _slice = context_client.GetSlice(request) except: # pylint: disable=bare-except @@ -210,11 +205,8 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Implementa el servicio g _slice_rw = Slice() _slice_rw.CopyFrom(_slice) - #cambia el status _slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(_slice_rw) -#elimina la slice considerando si es interdominio o no, y desagrupa la slice eliminada - #elimina los servicios asociados a la slice if is_inter_domain(context_client, _slice.slice_endpoint_ids): interdomain_client = InterdomainClient() slice_id = interdomain_client.DeleteSlice(request) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index d59531a1b..11aa9bb58 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -14,7 +14,7 @@ import logging, pandas, threading from typing import Dict, Optional, Tuple -from sklearn.cluster import KMeans #algoritmo de agrupamiento de scikit-learn (biblio de aprendizaje automatico) +from sklearn.cluster import KMeans from common.proto.context_pb2 import Slice from common.tools.grpc.Tools import grpc_message_to_json_string from .Constants import SLICE_GROUPS @@ -27,30 +27,30 @@ LOGGER = logging.getLogger(__name__) class SliceGrouper: def __init__(self) -> None: - self._lock = threading.Lock() #controla el acceso concurrente - self._is_enabled = is_slice_grouping_enabled() #esta habilitado el agrupamiento de slices? + self._lock = threading.Lock() + self._is_enabled = is_slice_grouping_enabled() LOGGER.info('Slice Grouping: {:s}'.format('ENABLED' if self._is_enabled else 'DISABLED')) if not self._is_enabled: return - metrics_exporter = MetricsExporter() #instancia de la clase + metrics_exporter = MetricsExporter() metrics_exporter.create_table() - self._slice_groups = create_slice_groups(SLICE_GROUPS) #grupos de slices + self._slice_groups = create_slice_groups(SLICE_GROUPS) # Initialize and fit K-Means with the pre-defined clusters we want, i.e., one per slice group - df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) #data frame con info de los grupos - k_means = KMeans(n_clusters=df_groups.shape[0]) #modelo utilizado para el agrupamiento + df_groups = pandas.DataFrame(SLICE_GROUPS, columns=['name', 'availability', 'capacity_gbps']) + k_means = KMeans(n_clusters=df_groups.shape[0]) k_means.fit(df_groups[['availability', 'capacity_gbps']]) df_groups['label'] = k_means.predict(df_groups[['availability', 'capacity_gbps']]) self._k_means = k_means self._df_groups = df_groups - self._group_mapping : Dict[str, Dict] = { #Dict = dictionary + self._group_mapping : Dict[str, Dict] = { group['name']:{k:v for k,v in group.items() if k != 'name'} - for group in list(df_groups.to_dict('records')) #mapeo de nombres de grupo a sus atributos - } + for group in list(df_groups.to_dict('records')) + } - label_to_group = {} #mapeo de etiquetas a nombres de grupo + label_to_group = {} for group_name,group_attrs in self._group_mapping.items(): label = group_attrs['label'] availability = group_attrs['availability'] @@ -60,7 +60,7 @@ class SliceGrouper: label_to_group[label] = group_name self._label_to_group = label_to_group - def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: #selecciona un grupo para una slice + def _select_group(self, slice_obj : Slice) -> Optional[Tuple[str, float, float]]: with self._lock: grouping_parameters = get_slice_grouping_parameters(slice_obj) LOGGER.debug('[_select_group] grouping_parameters={:s}'.format(str(grouping_parameters))) @@ -78,16 +78,16 @@ class SliceGrouper: return group_name, availability, capacity_gbps @property - def is_enabled(self): return self._is_enabled #indica si el agrupamiento de slices esta habilitado - - def group(self, slice_obj : Slice) -> bool: #determina el grupo al que debe pertenecer la slice + def is_enabled(self): return self._is_enabled + + def group(self, slice_obj : Slice) -> bool: LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[group] selected_group={:s}'.format(str(selected_group))) if selected_group is None: return False return add_slice_to_group(slice_obj, selected_group) - def ungroup(self, slice_obj : Slice) -> bool: # desagrupa la slice de un grupo + def ungroup(self, slice_obj : Slice) -> bool: LOGGER.debug('[ungroup] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) LOGGER.debug('[ungroup] selected_group={:s}'.format(str(selected_group))) diff --git a/src/webui/service/main/routes.py b/src/webui/service/main/routes.py index eba758ff3..75f036bef 100644 --- a/src/webui/service/main/routes.py +++ b/src/webui/service/main/routes.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64, json, logging -import traceback #, re +import base64, json, logging #, re from flask import jsonify, redirect, render_template, Blueprint, flash, session, url_for, request from common.proto.context_pb2 import ContextList, Empty, TopologyId, TopologyList from common.tools.descriptor.Loader import DescriptorLoader, compose_notifications @@ -114,7 +113,6 @@ def home(): except Exception as e: # pylint: disable=broad-except LOGGER.exception('Descriptor load failed') flash(f'Descriptor load failed: `{str(e)}`', 'danger') - traceback.print_exc() # Agregar esta línea para imprimir el traceback completo finally: context_client.close() device_client.close() -- GitLab From 008be1bab8dcafdf7fe95231135546ee00f602d3 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 17:37:57 +0100 Subject: [PATCH 027/941] code cleanup --- .../service/drivers/openconfig/templates/Inventory.py | 1 - src/slice/service/SliceServiceServicerImpl.py | 9 +++++---- src/slice/service/slice_grouper/SliceGrouper.py | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index 65562bc5b..e2999c579 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -82,7 +82,6 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory, 'class', component_type) if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue - component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: add_value_from_tag(inventory['attributes'], 'empty', component_empty) diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index 8a834f352..cbe2dd5c7 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -31,6 +31,7 @@ from service.client.ServiceClient import ServiceClient from .slice_grouper.SliceGrouper import SliceGrouper LOGGER = logging.getLogger(__name__) + METRICS_POOL = MetricsPool('Slice', 'RPC') class SliceServiceServicerImpl(SliceServiceServicer): @@ -43,7 +44,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Set slice status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the slice is # being modified. context_client = ContextClient() - slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) # se obtiene la slice con el sliceId de la req + slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) slice_rw = Slice() slice_rw.CopyFrom(request if slice_ro is None else slice_ro) @@ -51,7 +52,6 @@ class SliceServiceServicerImpl(SliceServiceServicer): slice_rw.slice_owner.CopyFrom(request.slice_owner) # pylint: disable=no-member slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_PLANNED # pylint: disable=no-member -#copiamos endpoints, reglas y configuraciones de la req a la slice copy_endpoint_ids(request.slice_endpoint_ids, slice_rw.slice_endpoint_ids ) # pylint: disable=no-member copy_constraints (request.slice_constraints, slice_rw.slice_constraints ) # pylint: disable=no-member copy_config_rules(request.slice_config.config_rules, slice_rw.slice_config.config_rules) # pylint: disable=no-member @@ -65,9 +65,9 @@ class SliceServiceServicerImpl(SliceServiceServicer): reply = context_client.SetSlice(slice_rw) context_client.close() return reply - + slice_with_uuids = context_client.GetSlice(slice_id_with_uuids) - + #LOGGER.info('json_current_slice = {:s}'.format(str(json_current_slice))) #json_updated_slice = grpc_message_to_json(request) #LOGGER.info('json_updated_slice = {:s}'.format(str(json_updated_slice))) @@ -207,6 +207,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): _slice_rw.CopyFrom(_slice) _slice_rw.slice_status.slice_status = SliceStatusEnum.SLICESTATUS_DEINIT # pylint: disable=no-member context_client.SetSlice(_slice_rw) + if is_inter_domain(context_client, _slice.slice_endpoint_ids): interdomain_client = InterdomainClient() slice_id = interdomain_client.DeleteSlice(request) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index 11aa9bb58..66d293e1e 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -48,7 +48,7 @@ class SliceGrouper: self._group_mapping : Dict[str, Dict] = { group['name']:{k:v for k,v in group.items() if k != 'name'} for group in list(df_groups.to_dict('records')) - } + } label_to_group = {} for group_name,group_attrs in self._group_mapping.items(): @@ -79,7 +79,6 @@ class SliceGrouper: @property def is_enabled(self): return self._is_enabled - def group(self, slice_obj : Slice) -> bool: LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) -- GitLab From 75ddb3d9e93e9b8656af1e4fd71d32461f1542b1 Mon Sep 17 00:00:00 2001 From: armingol Date: Thu, 14 Mar 2024 17:40:41 +0100 Subject: [PATCH 028/941] code cleanup --- src/device/service/drivers/openconfig/templates/Inventory.py | 1 + src/slice/service/slice_grouper/SliceGrouper.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/device/service/drivers/openconfig/templates/Inventory.py b/src/device/service/drivers/openconfig/templates/Inventory.py index e2999c579..916af0478 100644 --- a/src/device/service/drivers/openconfig/templates/Inventory.py +++ b/src/device/service/drivers/openconfig/templates/Inventory.py @@ -82,6 +82,7 @@ def parse(xml_data : ET.Element) -> List[Tuple[str, Dict[str, Any]]]: add_value_from_tag(inventory, 'class', component_type) if inventory['class'] == 'CPU' or inventory['class'] == 'STORAGE': continue + component_empty = xml_component.find('ocp:state/ocp:empty', namespaces=NAMESPACES) if not component_empty is None: add_value_from_tag(inventory['attributes'], 'empty', component_empty) diff --git a/src/slice/service/slice_grouper/SliceGrouper.py b/src/slice/service/slice_grouper/SliceGrouper.py index 66d293e1e..2f1a79181 100644 --- a/src/slice/service/slice_grouper/SliceGrouper.py +++ b/src/slice/service/slice_grouper/SliceGrouper.py @@ -79,6 +79,7 @@ class SliceGrouper: @property def is_enabled(self): return self._is_enabled + def group(self, slice_obj : Slice) -> bool: LOGGER.debug('[group] slice_obj={:s}'.format(grpc_message_to_json_string(slice_obj))) selected_group = self._select_group(slice_obj) -- GitLab From 87e30be2bab5855e2a0c65dd73fc0f0324f3ce92 Mon Sep 17 00:00:00 2001 From: armingol Date: Mon, 18 Mar 2024 14:43:41 +0100 Subject: [PATCH 029/941] code cleanup --- src/common/tools/descriptor/Loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index b2eb75045..5875094d8 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -255,7 +255,7 @@ class DescriptorLoader: _slices = {} for slice_ in self.__slices: context_uuid = slice_['slice_id']['context_id']['context_uuid']['uuid'] - _slices.setdefault(context_uuid, []).append(slice_) #no tenemos context_uuid en este formato, lo meto a mano? + _slices.setdefault(context_uuid, []).append(slice_) return _slices @property -- GitLab From 5f563662ab123c7732cf8f15ccb3b40b65d038ee Mon Sep 17 00:00:00 2001 From: armingol Date: Mon, 18 Mar 2024 14:45:12 +0100 Subject: [PATCH 030/941] code cleanup --- src/common/tools/descriptor/Loader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/common/tools/descriptor/Loader.py b/src/common/tools/descriptor/Loader.py index 5875094d8..92390f63e 100644 --- a/src/common/tools/descriptor/Loader.py +++ b/src/common/tools/descriptor/Loader.py @@ -163,9 +163,6 @@ class DescriptorLoader: "endpoint_id": {"device_id": {"device_uuid": {"uuid": elemento["id"]}}, "endpoint_uuid": {"uuid": attcircuit["ac-tp-id"]}}, "location": {"region": "4"} }}) - - # Convertir a JSON de salida - #json_output = json.dumps(json_out, indent=2) self.__slices = json_out.get('slices' , []) self.__contexts_add = None -- GitLab From 654eb4328bdb90c9b267cad11683385e02d191f6 Mon Sep 17 00:00:00 2001 From: carcel Date: Wed, 20 Mar 2024 10:14:21 +0000 Subject: [PATCH 031/941] Morpheus Client Extension --- manifests/nginx_ingress_http.yaml | 7 + proto/context.proto | 1 + src/common/DeviceTypes.py | 1 + src/common/tools/object_factory/Device.py | 11 + src/common/type_checkers/Assertions.py | 1 + .../data/sql_hash_join_full_scan_tests.sql | 2 +- .../database/models/enums/DeviceDriver.py | 1 + src/device/service/drivers/__init__.py | 12 + .../drivers/ietf_l2vpn/TfsDebugApiClient.py | 2 + .../drivers/smartnic/SmartnicDriver.py | 122 ++++ src/device/service/drivers/smartnic/Tools.py | 174 +++++ .../service/drivers/smartnic/__init__.py | 20 + src/nbi/service/__main__.py | 2 + .../nbi_plugins/agent_probes/Resources.py | 236 +++++++ .../nbi_plugins/agent_probes/Tools.py | 118 ++++ .../nbi_plugins/agent_probes/__init__.py | 69 ++ .../data/agent_probes_configuration_rule.json | 17 + src/nbi/tests/data/agent_probes_device.json | 27 + src/policy/Dockerfile | 67 +- .../java/org/etsi/tfs/policy/Serializer.java | 4 + .../context/model/DeviceDriverEnum.java | 3 +- src/policy/src/main/proto/acl.proto | 70 +- src/policy/src/main/proto/context.proto | 616 +++++++++++++++++- .../src/main/proto/context_policy.proto | 29 +- src/policy/src/main/proto/device.proto | 35 +- .../src/main/proto/kpi_sample_types.proto | 43 +- src/policy/src/main/proto/monitoring.proto | 175 ++++- src/policy/src/main/proto/policy.proto | 114 +++- src/policy/src/main/proto/policy_action.proto | 43 +- .../src/main/proto/policy_condition.proto | 44 +- src/policy/src/main/proto/service.proto | 26 +- .../service_handler_api/FilterFields.py | 1 + .../service/service_handlers/__init__.py | 6 + src/webui/service/device/forms.py | 1 + src/webui/service/device/routes.py | 2 + src/webui/service/templates/device/add.html | 1 + src/ztp/Dockerfile | 68 +- src/ztp/src/main/proto/acl.proto | 70 +- src/ztp/src/main/proto/context.proto | 616 +++++++++++++++++- src/ztp/src/main/proto/device.proto | 35 +- src/ztp/src/main/proto/kpi_sample_types.proto | 43 +- src/ztp/src/main/proto/monitoring.proto | 175 ++++- src/ztp/src/main/proto/ztp.proto | 70 +- 43 files changed, 3160 insertions(+), 20 deletions(-) create mode 100644 src/device/service/drivers/smartnic/SmartnicDriver.py create mode 100644 src/device/service/drivers/smartnic/Tools.py create mode 100644 src/device/service/drivers/smartnic/__init__.py create mode 100644 src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py create mode 100644 src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py create mode 100644 src/nbi/service/rest_server/nbi_plugins/agent_probes/__init__.py create mode 100644 src/nbi/tests/data/agent_probes_configuration_rule.json create mode 100644 src/nbi/tests/data/agent_probes_device.json mode change 120000 => 100644 src/policy/Dockerfile mode change 120000 => 100644 src/policy/src/main/proto/acl.proto mode change 120000 => 100644 src/policy/src/main/proto/context.proto mode change 120000 => 100644 src/policy/src/main/proto/context_policy.proto mode change 120000 => 100644 src/policy/src/main/proto/device.proto mode change 120000 => 100644 src/policy/src/main/proto/kpi_sample_types.proto mode change 120000 => 100644 src/policy/src/main/proto/monitoring.proto mode change 120000 => 100644 src/policy/src/main/proto/policy.proto mode change 120000 => 100644 src/policy/src/main/proto/policy_action.proto mode change 120000 => 100644 src/policy/src/main/proto/policy_condition.proto mode change 120000 => 100644 src/policy/src/main/proto/service.proto mode change 120000 => 100644 src/ztp/Dockerfile mode change 120000 => 100644 src/ztp/src/main/proto/acl.proto mode change 120000 => 100644 src/ztp/src/main/proto/context.proto mode change 120000 => 100644 src/ztp/src/main/proto/device.proto mode change 120000 => 100644 src/ztp/src/main/proto/kpi_sample_types.proto mode change 120000 => 100644 src/ztp/src/main/proto/monitoring.proto mode change 120000 => 100644 src/ztp/src/main/proto/ztp.proto diff --git a/manifests/nginx_ingress_http.yaml b/manifests/nginx_ingress_http.yaml index e8e8a80e4..336f164b4 100644 --- a/manifests/nginx_ingress_http.yaml +++ b/manifests/nginx_ingress_http.yaml @@ -50,6 +50,13 @@ spec: name: nbiservice port: number: 8080 + - path: /()(agent-probes/.*) + pathType: Prefix + backend: + service: + name: nbiservice + port: + number: 8080 - path: /()(bmw/.*) pathType: Prefix backend: diff --git a/proto/context.proto b/proto/context.proto index 5085cad33..856caa4f9 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -204,6 +204,7 @@ enum DeviceDriverEnum { DEVICEDRIVER_GNMI_OPENCONFIG = 8; DEVICEDRIVER_FLEXSCALE = 9; DEVICEDRIVER_IETF_ACTN = 10; + DEVICEDRIVER_SMARTNIC = 12; } enum DeviceOperationalStatusEnum { diff --git a/src/common/DeviceTypes.py b/src/common/DeviceTypes.py index 72b3e21fd..5ed4b6681 100644 --- a/src/common/DeviceTypes.py +++ b/src/common/DeviceTypes.py @@ -47,6 +47,7 @@ class DeviceTypeEnum(Enum): PACKET_ROUTER = 'packet-router' PACKET_SWITCH = 'packet-switch' XR_CONSTELLATION = 'xr-constellation' + SMARTNIC = 'smartnic' # ETSI TeraFlowSDN controller TERAFLOWSDN_CONTROLLER = 'teraflowsdn' diff --git a/src/common/tools/object_factory/Device.py b/src/common/tools/object_factory/Device.py index 76959232a..b3182e302 100644 --- a/src/common/tools/object_factory/Device.py +++ b/src/common/tools/object_factory/Device.py @@ -49,6 +49,9 @@ DEVICE_TFS_DRIVERS = [DeviceDriverEnum.DEVICEDRIVER_IETF_L2VPN] DEVICE_IETF_ACTN_TYPE = DeviceTypeEnum.OPEN_LINE_SYSTEM.value DEVICE_IETF_ACTN_DRIVERS = [DeviceDriverEnum.DEVICEDRIVER_IETF_ACTN] +DEVICE_SMARTNIC_TYPE = DeviceTypeEnum.SMARTNIC.value +DEVICE_SMARTNIC_DRIVERS = [DeviceDriverEnum.DEVICEDRIVER_SMARTNIC] + def json_device_id(device_uuid : str): return {'device_uuid': {'uuid': device_uuid}} @@ -148,6 +151,14 @@ def json_device_ietf_actn_disabled( device_uuid, DEVICE_IETF_ACTN_TYPE, DEVICE_DISABLED, name=name, endpoints=endpoints, config_rules=config_rules, drivers=drivers) +def json_device_smartnic_disabled( + device_uuid : str, name : Optional[str] = None, endpoints : List[Dict] = [], config_rules : List[Dict] = [], + drivers : List[Dict] = DEVICE_SMARTNIC_DRIVERS + ): + return json_device( + device_uuid, DEVICE_SMARTNIC_TYPE, DEVICE_DISABLED, name=name, endpoints=endpoints, config_rules=config_rules, + drivers=drivers) + def json_device_connect_rules(address : str, port : int, settings : Dict = {}) -> List[Dict]: return [ json_config_rule_set('_connect/address', address), diff --git a/src/common/type_checkers/Assertions.py b/src/common/type_checkers/Assertions.py index 87d8e54ee..06bcd4482 100644 --- a/src/common/type_checkers/Assertions.py +++ b/src/common/type_checkers/Assertions.py @@ -48,6 +48,7 @@ def validate_device_driver_enum(message): 'DEVICEDRIVER_GNMI_OPENCONFIG', 'DEVICEDRIVER_FLEXSCALE', 'DEVICEDRIVER_IETF_ACTN', + 'DEVICEDRIVER_SMARTNIC' ] def validate_device_operational_status_enum(message): diff --git a/src/context/data/sql_hash_join_full_scan_tests.sql b/src/context/data/sql_hash_join_full_scan_tests.sql index ebead1be6..34330f916 100644 --- a/src/context/data/sql_hash_join_full_scan_tests.sql +++ b/src/context/data/sql_hash_join_full_scan_tests.sql @@ -9,7 +9,7 @@ CREATE DATABASE tests; USE tests; CREATE TYPE public.orm_deviceoperationalstatusenum AS ENUM ('UNDEFINED', 'DISABLED', 'ENABLED'); -CREATE TYPE public.orm_devicedriverenum AS ENUM ('UNDEFINED', 'OPENCONFIG', 'TRANSPORT_API', 'P4', 'IETF_NETWORK_TOPOLOGY', 'ONF_TR_352', 'XR', 'IETF_L2VPN'); +CREATE TYPE public.orm_devicedriverenum AS ENUM ('UNDEFINED', 'OPENCONFIG', 'TRANSPORT_API', 'P4', 'IETF_NETWORK_TOPOLOGY', 'ONF_TR_352', 'XR', 'IETF_L2VPN', 'SMARTNIC'); CREATE TYPE public.configrulekindenum AS ENUM ('CUSTOM', 'ACL'); CREATE TYPE public.orm_configactionenum AS ENUM ('UNDEFINED', 'SET', 'DELETE'); diff --git a/src/context/service/database/models/enums/DeviceDriver.py b/src/context/service/database/models/enums/DeviceDriver.py index 8e15bf058..559291965 100644 --- a/src/context/service/database/models/enums/DeviceDriver.py +++ b/src/context/service/database/models/enums/DeviceDriver.py @@ -33,6 +33,7 @@ class ORM_DeviceDriverEnum(enum.Enum): GNMI_OPENCONFIG = DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG FLEXSCALE = DeviceDriverEnum.DEVICEDRIVER_FLEXSCALE IETF_ACTN = DeviceDriverEnum.DEVICEDRIVER_IETF_ACTN + SMARTNIC = DeviceDriverEnum.DEVICEDRIVER_SMARTNIC grpc_to_enum__device_driver = functools.partial( grpc_to_enum, DeviceDriverEnum, ORM_DeviceDriverEnum) diff --git a/src/device/service/drivers/__init__.py b/src/device/service/drivers/__init__.py index 27c61f89f..1534655db 100644 --- a/src/device/service/drivers/__init__.py +++ b/src/device/service/drivers/__init__.py @@ -136,6 +136,18 @@ if LOAD_ALL_DEVICE_DRIVERS: FilterFieldEnum.DRIVER : DeviceDriverEnum.DEVICEDRIVER_P4, } ])) + +if LOAD_ALL_DEVICE_DRIVERS: + from .smartnic.SmartnicDriver import SmartnicDriver # pylint: disable=wrong-import-position + DRIVERS.append( + (SmartnicDriver, [ + { + # Real P4 Switch, specifying P4 Driver => use P4Driver + FilterFieldEnum.DEVICE_TYPE: DeviceTypeEnum.SMARTNIC, + FilterFieldEnum.DRIVER : DeviceDriverEnum.DEVICEDRIVER_SMARTNIC, + } + ])) + if LOAD_ALL_DEVICE_DRIVERS: from .microwave.IETFApiDriver import IETFApiDriver # pylint: disable=wrong-import-position diff --git a/src/device/service/drivers/ietf_l2vpn/TfsDebugApiClient.py b/src/device/service/drivers/ietf_l2vpn/TfsDebugApiClient.py index 06c55c5dc..ee2d1ab6e 100644 --- a/src/device/service/drivers/ietf_l2vpn/TfsDebugApiClient.py +++ b/src/device/service/drivers/ietf_l2vpn/TfsDebugApiClient.py @@ -45,6 +45,8 @@ MAPPING_DRIVER = { 'DEVICEDRIVER_IETF_L2VPN' : 7, 'DEVICEDRIVER_GNMI_OPENCONFIG' : 8, 'DEVICEDRIVER_FLEXSCALE' : 9, + 'DEVICEDRIVER_IETF_ACTN' : 10, + 'DEVICEDRIVER_SMARTNIC' : 12 } MSG_ERROR = 'Could not retrieve devices in remote TeraFlowSDN instance({:s}). status_code={:s} reply={:s}' diff --git a/src/device/service/drivers/smartnic/SmartnicDriver.py b/src/device/service/drivers/smartnic/SmartnicDriver.py new file mode 100644 index 000000000..4bad52db4 --- /dev/null +++ b/src/device/service/drivers/smartnic/SmartnicDriver.py @@ -0,0 +1,122 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging, requests, threading +from requests.auth import HTTPBasicAuth +from typing import Any, Iterator, List, Optional, Tuple, Union +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method +from common.type_checkers.Checkers import chk_string, chk_type +from device.service.driver_api._Driver import _Driver +from . import ALL_RESOURCE_KEYS +from .Tools import create_connectivity_service, find_key, config_getter, delete_connectivity_service + +LOGGER = logging.getLogger(__name__) + +DRIVER_NAME = 'smartnic' +METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME}) + +class SmartnicDriver(_Driver): + def __init__(self, address: str, port: int, **settings) -> None: + super().__init__(DRIVER_NAME, address, port, **settings) + self.__lock = threading.Lock() + self.__started = threading.Event() + self.__terminate = threading.Event() + username = self.settings.get('username') + password = self.settings.get('password') + #self.__auth = HTTPBasicAuth(username, password) if username is not None and password is not None else None + scheme = self.settings.get('scheme', 'http') + self.__tapi_root = '{:s}://{:s}:{:d}'.format(scheme, self.address, int(self.port)) + self.__timeout = int(self.settings.get('timeout', 120)) + + def Connect(self) -> bool: + url = self.__tapi_root + with self.__lock: + if self.__started.is_set(): return True + try: + requests.get(url, timeout=self.__timeout, verify=False) + #requests.get(url, timeout=self.__timeout, verify=False, auth=self.__auth) + except requests.exceptions.Timeout: + LOGGER.exception('Timeout connecting {:s}'.format(str(self.__tapi_root))) + return False + except Exception: # pylint: disable=broad-except + LOGGER.exception('Exception connecting {:s}'.format(str(self.__tapi_root))) + return False + else: + self.__started.set() + return True + + def Disconnect(self) -> bool: + with self.__lock: + self.__terminate.set() + return True + + @metered_subclass_method(METRICS_POOL) + def GetInitialConfig(self) -> List[Tuple[str, Any]]: + with self.__lock: + return [] + + @metered_subclass_method(METRICS_POOL) + def GetConfig(self, resource_keys : List[str] = []) -> List[Tuple[str, Union[Any, None, Exception]]]: + chk_type('resources', resource_keys, list) + results = [] + with self.__lock: + if len(resource_keys) == 0: resource_keys = ALL_RESOURCE_KEYS + for i, resource_key in enumerate(resource_keys): + str_resource_name = 'resource_key[#{:d}]'.format(i) + chk_string(str_resource_name, resource_key, allow_empty=False) + results.extend(config_getter( + self.__tapi_root, resource_key, timeout=self.__timeout)) + return results + + @metered_subclass_method(METRICS_POOL) + def SetConfig(self, resources: List[Tuple[str, Any]]) -> List[Union[bool, Exception]]: + results = [] + if len(resources) == 0: + return results + with self.__lock: + for resource in resources: + LOGGER.info('resource = {:s}'.format(str(resource))) + config_rules = find_key(resource, 'config_rules') + data = create_connectivity_service( + self.__tapi_root, config_rules, timeout=self.__timeout) + results.extend(data) + return results + + @metered_subclass_method(METRICS_POOL) + def DeleteConfig(self, resources: List[Tuple[str, Any]]) -> List[Union[bool, Exception]]: + results = [] + if len(resources) == 0: return results + with self.__lock: + for resource in resources: + LOGGER.info('resource = {:s}'.format(str(resource))) + config_rules = find_key(resource, 'config_rules') + results.extend(delete_connectivity_service( + self.__tapi_root, config_rules, timeout=self.__timeout)) + return results + + @metered_subclass_method(METRICS_POOL) + def SubscribeState(self, subscriptions : List[Tuple[str, float, float]]) -> List[Union[bool, Exception]]: + # TODO: TAPI does not support monitoring by now + return [False for _ in subscriptions] + + @metered_subclass_method(METRICS_POOL) + def UnsubscribeState(self, subscriptions : List[Tuple[str, float, float]]) -> List[Union[bool, Exception]]: + # TODO: TAPI does not support monitoring by now + return [False for _ in subscriptions] + + def GetState( + self, blocking=False, terminate : Optional[threading.Event] = None + ) -> Iterator[Tuple[float, str, Any]]: + # TODO: TAPI does not support monitoring by now + return [] diff --git a/src/device/service/drivers/smartnic/Tools.py b/src/device/service/drivers/smartnic/Tools.py new file mode 100644 index 000000000..54961bbdd --- /dev/null +++ b/src/device/service/drivers/smartnic/Tools.py @@ -0,0 +1,174 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json, logging, operator, requests +from requests.auth import HTTPBasicAuth +from typing import Optional +#from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_SERVICES, RESOURCE_INTERFACES + + +import logging +from typing import Any, Dict, Optional, Tuple +from common.proto.kpi_sample_types_pb2 import KpiSampleType +from common.type_checkers.Checkers import chk_attribute, chk_string, chk_type +from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_INTERFACES + +LOGGER = logging.getLogger(__name__) + + +SPECIAL_RESOURCE_MAPPINGS = { + RESOURCE_ENDPOINTS : '/endpoints', + RESOURCE_INTERFACES : '/interfaces' +} + +HTTP_OK_CODES = { + 200, # OK + 201, # Created + 202, # Accepted + 204, # No Content +} + +def find_key(resource, key): + return json.loads(resource[1])[key] + + +def config_getter( + root_url : str, resource_key : str, auth : Optional[HTTPBasicAuth] = None, timeout : Optional[int] = None +): + url = '{:s}/manage-probe/ports'.format(root_url) + result = [] + try: + response = requests.get(url, timeout=timeout, verify=False) + except requests.exceptions.Timeout: + LOGGER.exception('Timeout connecting {:s}'.format(url)) + return result + except Exception as e: # pylint: disable=broad-except + LOGGER.exception('Exception retrieving {:s}'.format(resource_key)) + result.append((resource_key, e)) + return result + return response + + # try: + # context = json.loads(response.content) + # except Exception as e: # pylint: disable=broad-except + # LOGGER.warning('Unable to decode reply: {:s}'.format(str(response.content))) + # result.append((resource_key, e)) + # return result + + + +def create_connectivity_service( + root_url, config_rules, timeout : Optional[int] = None, auth : Optional[HTTPBasicAuth] = None +): + + url = '{:s}/configure'.format(root_url) + headers = {'content-type': 'application/json'} + results = [] + try: + LOGGER.info('Configuring Smartnic rules') + response = requests.post( + url=url, data=json.dumps(config_rules), timeout=timeout, headers=headers, verify=False) + LOGGER.info('SmartNIC Probes response: {:s}'.format(str(response))) + except Exception as e: # pylint: disable=broad-except + LOGGER.exception('Exception creating ConfigRule') + results.append(e) + else: + if response.status_code not in HTTP_OK_CODES: + msg = 'Could not create ConfigRule status_code={:s} reply={:s}' + LOGGER.error(msg.format(str(response.status_code), str(response))) + results.append(response.status_code in HTTP_OK_CODES) + return results + +def delete_connectivity_service(root_url, config_rules, timeout : Optional[int] = None, auth : Optional[HTTPBasicAuth] = None +): + url = '{:s}/configure'.format(root_url) + results = [] + try: + response = requests.delete(url=url, timeout=timeout, verify=False) + except Exception as e: # pylint: disable=broad-except + LOGGER.exception('Exception deleting ConfigRule') + results.append(e) + else: + if response.status_code not in HTTP_OK_CODES: + msg = 'Could not delete ConfigRule status_code={:s} reply={:s}' + LOGGER.error(msg.format(str(response.status_code), str(response))) + results.append(response.status_code in HTTP_OK_CODES) + return results + +def process_optional_string_field( + endpoint_data : Dict[str, Any], field_name : str, endpoint_resource_value : Dict[str, Any] +) -> None: + field_value = chk_attribute(field_name, endpoint_data, 'endpoint_data', default=None) + if field_value is None: return + chk_string('endpoint_data.{:s}'.format(field_name), field_value) + if len(field_value) > 0: endpoint_resource_value[field_name] = field_value + +def compose_resource_endpoint(endpoint_data : Dict[str, Any]) -> Optional[Tuple[str, Dict]]: + try: + # Check type of endpoint_data + chk_type('endpoint_data', endpoint_data, dict) + + # Check endpoint UUID (mandatory) + endpoint_uuid = chk_attribute('uuid', endpoint_data, 'endpoint_data') + chk_string('endpoint_data.uuid', endpoint_uuid, min_length=1) + endpoint_resource_path = SPECIAL_RESOURCE_MAPPINGS.get(RESOURCE_ENDPOINTS) + endpoint_resource_key = '{:s}/endpoint[{:s}]'.format(endpoint_resource_path, endpoint_uuid) + endpoint_resource_value = {'uuid': endpoint_uuid} + + # Check endpoint optional string fields + process_optional_string_field(endpoint_data, 'name', endpoint_resource_value) + process_optional_string_field(endpoint_data, 'type', endpoint_resource_value) + process_optional_string_field(endpoint_data, 'context_uuid', endpoint_resource_value) + process_optional_string_field(endpoint_data, 'topology_uuid', endpoint_resource_value) + + # Check endpoint sample types (optional) + endpoint_sample_types = chk_attribute('sample_types', endpoint_data, 'endpoint_data', default=[]) + chk_type('endpoint_data.sample_types', endpoint_sample_types, list) + sample_types = {} + sample_type_errors = [] + for i,endpoint_sample_type in enumerate(endpoint_sample_types): + field_name = 'endpoint_data.sample_types[{:d}]'.format(i) + try: + chk_type(field_name, endpoint_sample_type, (int, str)) + if isinstance(endpoint_sample_type, int): + metric_name = KpiSampleType.Name(endpoint_sample_type) + metric_id = endpoint_sample_type + elif isinstance(endpoint_sample_type, str): + metric_id = KpiSampleType.Value(endpoint_sample_type) + metric_name = endpoint_sample_type + else: + str_type = str(type(endpoint_sample_type)) + raise Exception('Bad format: {:s}'.format(str_type)) # pylint: disable=broad-exception-raised + except Exception as e: # pylint: disable=broad-exception-caught + MSG = 'Unsupported {:s}({:s}) : {:s}' + sample_type_errors.append(MSG.format(field_name, str(endpoint_sample_type), str(e))) + + metric_name = metric_name.lower().replace('kpisampletype_', '') + monitoring_resource_key = '{:s}/state/{:s}'.format(endpoint_resource_key, metric_name) + sample_types[metric_id] = monitoring_resource_key + + if len(sample_type_errors) > 0: + # pylint: disable=broad-exception-raised + raise Exception('Malformed Sample Types:\n{:s}'.format('\n'.join(sample_type_errors))) + + if len(sample_types) > 0: + endpoint_resource_value['sample_types'] = sample_types + + if 'location' in endpoint_data: + endpoint_resource_value['location'] = endpoint_data['location'] + + return endpoint_resource_key, endpoint_resource_value + except: # pylint: disable=bare-except + LOGGER.exception('Problem composing endpoint({:s})'.format(str(endpoint_data))) + return None diff --git a/src/device/service/drivers/smartnic/__init__.py b/src/device/service/drivers/smartnic/__init__.py new file mode 100644 index 000000000..bc88d00fa --- /dev/null +++ b/src/device/service/drivers/smartnic/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_SERVICES, RESOURCE_INTERFACES + +ALL_RESOURCE_KEYS = [ + RESOURCE_ENDPOINTS, + RESOURCE_SERVICES, + RESOURCE_INTERFACES +] diff --git a/src/nbi/service/__main__.py b/src/nbi/service/__main__.py index 8834e45a2..0b6fb18e3 100644 --- a/src/nbi/service/__main__.py +++ b/src/nbi/service/__main__.py @@ -20,6 +20,7 @@ from common.Settings import ( wait_for_environment_variables) from .NbiService import NbiService from .rest_server.RestServer import RestServer +from .rest_server.nbi_plugins.agent_probes import register_agent_probes from .rest_server.nbi_plugins.debug_api import register_debug_api from .rest_server.nbi_plugins.etsi_bwm import register_etsi_bwm_api from .rest_server.nbi_plugins.ietf_l2vpn import register_ietf_l2vpn @@ -62,6 +63,7 @@ def main(): grpc_service.start() rest_server = RestServer() + register_agent_probes(rest_server) register_debug_api(rest_server) register_etsi_bwm_api(rest_server) register_ietf_l2vpn(rest_server) # Registering L2VPN entrypoint diff --git a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py new file mode 100644 index 000000000..2b0a537cc --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py @@ -0,0 +1,236 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from flask.json import jsonify +from flask_restful import Resource, request +from common.proto.context_pb2 import Empty +from common.tools.grpc.Tools import grpc_message_to_json +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from service.client.ServiceClient import ServiceClient +from .Tools import ( + format_grpc_to_json, grpc_connection_id, grpc_context_id, grpc_device, grpc_device_id, grpc_link_id, grpc_policy_rule_id, + grpc_service_id, grpc_service, grpc_slice_id, grpc_topology_id) + +class _Resource(Resource): + def __init__(self) -> None: + super().__init__() + self.client = ContextClient() + self.device_client = DeviceClient() + self.service_client = ServiceClient() + +class ContextIds(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListContextIds(Empty())) + +class Contexts(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListContexts(Empty())) + +class DummyContexts(_Resource): + def get(self): + contexts = grpc_message_to_json(self.client.ListContexts(Empty()), use_integers_for_enums=True)['contexts'] + devices = grpc_message_to_json(self.client.ListDevices(Empty()), use_integers_for_enums=True)['devices'] + links = grpc_message_to_json(self.client.ListLinks(Empty()), use_integers_for_enums=True)['links'] + + topologies = list() + slices = list() + services = list() + connections = list() + + for context in contexts: + context_uuid = context['context_id']['context_uuid']['uuid'] + context_id = grpc_context_id(context_uuid) + + topologies.extend(grpc_message_to_json( + self.client.ListTopologies(context_id), + use_integers_for_enums=True + )['topologies']) + + slices.extend(grpc_message_to_json( + self.client.ListSlices(context_id), + use_integers_for_enums=True + )['slices']) + + context_services = grpc_message_to_json( + self.client.ListServices(context_id), + use_integers_for_enums=True + )['services'] + services.extend(context_services) + + for service in context_services: + service_uuid = service['service_id']['service_uuid']['uuid'] + service_id = grpc_service_id(context_uuid, service_uuid) + connections.extend(grpc_message_to_json( + self.client.ListConnections(service_id), + use_integers_for_enums=True + )['connections']) + + for device in devices: + for config_rule in device['device_config']['config_rules']: + if 'custom' not in config_rule: continue + resource_value = config_rule['custom']['resource_value'] + if not isinstance(resource_value, str): continue + try: + resource_value = json.loads(resource_value) + except: # pylint: disable=bare-except + pass + config_rule['custom']['resource_value'] = resource_value + + dummy_context = {'dummy_mode': True} + if len(contexts ) > 0: dummy_context['contexts' ] = contexts + if len(topologies ) > 0: dummy_context['topologies' ] = topologies + if len(devices ) > 0: dummy_context['devices' ] = devices + if len(links ) > 0: dummy_context['links' ] = links + if len(slices ) > 0: dummy_context['slices' ] = slices + if len(services ) > 0: dummy_context['services' ] = services + if len(connections) > 0: dummy_context['connections'] = connections + return jsonify(dummy_context) + +class Context(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.GetContext(grpc_context_id(context_uuid))) + +class TopologyIds(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.ListTopologyIds(grpc_context_id(context_uuid))) + +class Topologies(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.ListTopologies(grpc_context_id(context_uuid))) + +class Topology(_Resource): + def get(self, context_uuid : str, topology_uuid : str): + return format_grpc_to_json(self.client.GetTopology(grpc_topology_id(context_uuid, topology_uuid))) + +class ServiceIds(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.ListServiceIds(grpc_context_id(context_uuid))) + +class Services(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.ListServices(grpc_context_id(context_uuid))) + +class Service(_Resource): + def get(self, context_uuid : str, service_uuid : str): + return format_grpc_to_json(self.client.GetService(grpc_service_id(context_uuid, service_uuid))) + + def post(self, context_uuid : str, service_uuid : str): # pylint: disable=unused-argument + service = request.get_json()['services'][0] + return format_grpc_to_json(self.service_client.CreateService(grpc_service( + service_uuid = service['service_id']['service_uuid']['uuid'], + service_type = service['service_type'], + context_uuid = service['service_id']['context_id']['context_uuid']['uuid'], + ))) + + def put(self, context_uuid : str, service_uuid : str): # pylint: disable=unused-argument + service = request.get_json()['services'][0] + return format_grpc_to_json(self.service_client.UpdateService(grpc_service( + service_uuid = service['service_id']['service_uuid']['uuid'], + service_type = service['service_type'], + context_uuid = service['service_id']['context_id']['context_uuid']['uuid'], + status = service['service_status']['service_status'], + endpoint_ids = service['service_endpoint_ids'], + constraints = service['service_constraints'], + config_rules = service['service_config']['config_rules'] + ))) + + def delete(self, context_uuid : str, service_uuid : str): + return format_grpc_to_json(self.service_client.DeleteService(grpc_service_id( + context_uuid, service_uuid, + ))) + +class SliceIds(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.ListSliceIds(grpc_context_id(context_uuid))) + +class Slices(_Resource): + def get(self, context_uuid : str): + return format_grpc_to_json(self.client.ListSlices(grpc_context_id(context_uuid))) + +class Slice(_Resource): + def get(self, context_uuid : str, slice_uuid : str): + return format_grpc_to_json(self.client.GetSlice(grpc_slice_id(context_uuid, slice_uuid))) + +class DeviceIds(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListDeviceIds(Empty())) + +class Devices(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListDevices(Empty())) + +class Device(_Resource): + def get(self, device_uuid : str): + return format_grpc_to_json(self.client.GetDevice(grpc_device_id(device_uuid))) + + def post(self, device_uuid : str): # pylint: disable=unused-argument + device = request.get_json()['devices'][0] + return format_grpc_to_json(self.device_client.AddDevice(grpc_device( + device_uuid = device['device_id']['device_uuid']['uuid'], + device_type = device['device_type'], + config_rules = device['device_config']['config_rules'], + status = device['device_operational_status'], + drivers = device['device_drivers'], + endpoints = device['device_endpoints'] + ))) + + def put(self, device_uuid : str): # pylint: disable=unused-argument + device = request.get_json()['devices'][0] + return format_grpc_to_json(self.device_client.ConfigureDevice(grpc_device( + device_uuid = device['device_id']['device_uuid']['uuid'], + device_type = device['device_type'], + device_config = device['device_config']['config_rules'], + device_operational_status = device['device_operational_status'], + device_drivers = device['device_drivers'], + device_endpoints = device['device_endpoints'] + ))) + + +class LinkIds(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListLinkIds(Empty())) + +class Links(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListLinks(Empty())) + +class Link(_Resource): + def get(self, link_uuid : str): + return format_grpc_to_json(self.client.GetLink(grpc_link_id(link_uuid))) + +class ConnectionIds(_Resource): + def get(self, context_uuid : str, service_uuid : str): + return format_grpc_to_json(self.client.ListConnectionIds(grpc_service_id(context_uuid, service_uuid))) + +class Connections(_Resource): + def get(self, context_uuid : str, service_uuid : str): + return format_grpc_to_json(self.client.ListConnections(grpc_service_id(context_uuid, service_uuid))) + +class Connection(_Resource): + def get(self, connection_uuid : str): + return format_grpc_to_json(self.client.GetConnection(grpc_connection_id(connection_uuid))) + +class PolicyRuleIds(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListPolicyRuleIds(Empty())) + +class PolicyRules(_Resource): + def get(self): + return format_grpc_to_json(self.client.ListPolicyRules(Empty())) + +class PolicyRule(_Resource): + def get(self, policy_rule_uuid : str): + return format_grpc_to_json(self.client.GetPolicyRule(grpc_policy_rule_id(policy_rule_uuid))) diff --git a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py new file mode 100644 index 000000000..17b6dcdfd --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py @@ -0,0 +1,118 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask.json import jsonify +from common.proto.context_pb2 import ( + ConnectionId, ContextId, DeviceDriverEnum, Device, DeviceId, DeviceOperationalStatusEnum, LinkId, ServiceId, SliceId, TopologyId, Service, ServiceStatusEnum +) +from common.proto.policy_pb2 import PolicyRuleId +from common.tools.grpc.Tools import grpc_message_to_json +from common.tools.object_factory.Connection import json_connection_id +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.ConfigRule import json_config_rule +from common.tools.object_factory.Constraint import json_constraint_custom +from common.tools.object_factory.EndPoint import json_endpoint_id, json_endpoint +from common.tools.object_factory.Device import json_device_id, json_device +from common.tools.object_factory.Link import json_link_id +from common.tools.object_factory.PolicyRule import json_policyrule_id +from common.tools.object_factory.Service import json_service_id, json_service +from common.tools.object_factory.Slice import json_slice_id +from common.tools.object_factory.Topology import json_topology_id + + +def format_grpc_to_json(grpc_reply): + return jsonify(grpc_message_to_json(grpc_reply)) + +def grpc_connection_id(connection_uuid): + return ConnectionId(**json_connection_id(connection_uuid)) + +def grpc_context_id(context_uuid): + return ContextId(**json_context_id(context_uuid)) + +def grpc_device_id(device_uuid): + return DeviceId(**json_device_id(device_uuid)) + +def grpc_device( + device_uuid, device_type, config_rules=None, status=None, drivers=None, endpoints=None +): + json_config_rules = [ + json_config_rule( + config_rule['action'], + config_rule['custom']['resource_key'], + config_rule['custom']['resource_value'] + ) + for config_rule in config_rules + ] if config_rules else [] + json_status = status if status else DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_UNDEFINED + json_drivers = drivers if drivers else DeviceDriverEnum.DEVICEDRIVER_UNDEFINED + json_endpoints = [ + json_endpoint( + endpoint['device_id']['device_uuid']['uuid'], + endpoint['endpoint_uuid']['uuid'], + endpoint['endpoint_type'], + endpoint['topology_id'], + endpoint['kpi_sample_types'], + endpoint['location']['region'] + ) + for endpoint in endpoints + ] if endpoints else [] + return Device(**json_device( + device_uuid, device_type, json_config_rules, json_status, + json_drivers, json_endpoints)) + +def grpc_link_id(link_uuid): + return LinkId(**json_link_id(link_uuid)) + +def grpc_service_id(context_uuid, service_uuid): + return ServiceId(**json_service_id(service_uuid, context_id=json_context_id(context_uuid))) + +def grpc_service( + service_uuid, service_type, context_uuid, status=None, endpoint_ids=None, constraints=None, config_rules=None +): + json_context = json_context_id(context_uuid) + json_status = status if status else ServiceStatusEnum.SERVICESTATUS_PLANNED + json_endpoints_ids = [ + json_endpoint_id( + json_device_id(endpoint_id['device_id']['device_uuid']['uuid']), + endpoint_id['endpoint_uuid']['uuid'] + ) + for endpoint_id in endpoint_ids + ] if endpoint_ids else [] + json_constraints = [ + json_constraint_custom( + constraint['custom']['constraint_type'], + constraint['custom']['constraint_value'] + ) + for constraint in constraints + ] if constraints else [] + json_config_rules = [ + json_config_rule( + config_rule['action'], + config_rule['custom']['resource_key'], + config_rule['custom']['resource_value'] + ) + for config_rule in config_rules + ] if config_rules else [] + return Service(**json_service( + service_uuid, service_type, json_context, json_status, + json_endpoints_ids, json_constraints, json_config_rules)) + +def grpc_slice_id(context_uuid, slice_uuid): + return SliceId(**json_slice_id(slice_uuid, context_id=json_context_id(context_uuid))) + +def grpc_topology_id(context_uuid, topology_uuid): + return TopologyId(**json_topology_id(topology_uuid, context_id=json_context_id(context_uuid))) + +def grpc_policy_rule_id(policy_rule_uuid): + return PolicyRuleId(**json_policyrule_id(policy_rule_uuid)) diff --git a/src/nbi/service/rest_server/nbi_plugins/agent_probes/__init__.py b/src/nbi/service/rest_server/nbi_plugins/agent_probes/__init__.py new file mode 100644 index 000000000..0b85500fc --- /dev/null +++ b/src/nbi/service/rest_server/nbi_plugins/agent_probes/__init__.py @@ -0,0 +1,69 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nbi.service.rest_server.RestServer import RestServer +from .Resources import ( + Connection, ConnectionIds, Connections, + Context, ContextIds, Contexts, + Device, DeviceIds, Devices, + DummyContexts, + Link, LinkIds, Links, + PolicyRule, PolicyRuleIds, PolicyRules, + Service, ServiceIds, Services, + Slice, SliceIds, Slices, + Topologies, Topology, TopologyIds +) + +URL_PREFIX = '/agent-probes' + +# Use 'path' type since some identifiers might contain char '/' and Flask is unable to recognize them in 'string' type. +RESOURCES = [ + # (endpoint_name, resource_class, resource_url) + # ('api.context_ids', ContextIds, '/context_ids'), + # ('api.contexts', Contexts, '/contexts'), + # ('api.dummy_contexts', DummyContexts, '/dummy_contexts'), + # ('api.context', Context, '/context/'), + + # ('api.topology_ids', TopologyIds, '/context//topology_ids'), + # ('api.topologies', Topologies, '/context//topologies'), + # ('api.topology', Topology, '/context//topology/'), + + # ('api.service_ids', ServiceIds, '/context//service_ids'), + # ('api.services', Services, '/context//services'), + # ('api.service', Service, '/context//service/'), + + # ('api.slice_ids', SliceIds, '/context//slice_ids'), + # ('api.slices', Slices, '/context//slices'), + # ('api.slice', Slice, '/context//slice/'), + + ('api.smartnic.device_ids', DeviceIds, '/device_ids'), + ('api.smartnic.devices', Devices, '/devices'), + ('api.smartnic.device', Device, '/device/'), + + # ('api.link_ids', LinkIds, '/link_ids'), + # ('api.links', Links, '/links'), + # ('api.link', Link, '/link/'), + + # ('api.connection_ids', ConnectionIds, '/context//service//connection_ids'), + # ('api.connections', Connections, '/context//service//connections'), + # ('api.connection', Connection, '/connection/'), + + # ('api.policyrule_ids', PolicyRuleIds, '/policyrule_ids'), + # ('api.policyrules', PolicyRules, '/policyrules'), + # ('api.policyrule', PolicyRule, '/policyrule/'), +] + +def register_agent_probes(rest_server : RestServer): + for endpoint_name, resource_class, resource_url in RESOURCES: + rest_server.add_resource(resource_class, URL_PREFIX + resource_url, endpoint=endpoint_name) diff --git a/src/nbi/tests/data/agent_probes_configuration_rule.json b/src/nbi/tests/data/agent_probes_configuration_rule.json new file mode 100644 index 000000000..dbbf953ad --- /dev/null +++ b/src/nbi/tests/data/agent_probes_configuration_rule.json @@ -0,0 +1,17 @@ +{ + "devices": [ + { + "device_id": {"device_uuid":{"uuid":"smp-01"}}, + "device_type": "smartnic", + "device_config": {"config_rules": [ + {"action": 1, "custom": { + "resource_key": "config_rules", + "resource_value": {"pipeline_name":"pipeline_example","num_threads":2,"pipeline_batch_size":2048,"model_max_batch_size":4096,"input_file":"inputfile1.csv","output_file":"outputfile1.csv","nic_addr": "10.10.2.25", "gpu_addr":"11.6.15.2", "model_fea_length":8,"model_name":"modelname1","iterative":false,"server_url":"servertest.com","file_type":"csv","stages":{"FileSourceStage":{"stage_name":"FileSourceStage","FileSourceStage":{"fs_configuration":{"FsConf":{"mode":"FsConf"}},"filename":"file1.csv","file_type":"csv"}},"DeserializeStage":{"stage_name":"DeserializeStage","DeserializeStage":{"ds_configuration":{"DeserializeConf":{"mode":"DeserializeConf"}}}},"AbpPcapPreprocessingStage":{"stage_name":"AbpPcapPreprocessingStage","AbpPcapPreprocessingStage":{"apps_configuration":{"AppsConf":{"mode":"AppsConf"}}}},"MonitorStage":{"stage_name":"MonitorStage","MonitorStage":{"ms_configuration":{"MonitoringConf":{"mode":"MonitoringConf"}},"descriptions":"MetricMonitoring","unit":"kbps"}},"TritonInferenceStage":{"stage_name":"TritonInferenceStage","TritonInferenceStage":{"tis_configuration":{"TritonConf":{"mode":"TritonConf"}},"model_name":"Modeltest1","server_url":"servertest.com","force_convert_inputs":false}},"AddClassificationsStage":{"stage_name":"AddClassificationsStage","AddClassificationsStage":{"acs_configuration":{"ClassificationConf":{"mode":"ClassificationConf"}}}},"SerializeStage":{"stage_name":"SerializeStage","SerializeStage":{"ss_configuration":{"SerializeConf":{"mode":"SerializeConf"}}}},"WriteToFileStage":{"stage_name":"WriteToFileStage","WriteToFileStage":{"wfs_configuration":{"WFSConf":{"mode":"WFSConf"}},"wfs_filename":"file2.txt","overwrite":false}},"CustomStage":{"stage_name":"CustomStage","custom":{"name":{"field_name":"name","field_value":"test"}}}}} + }} + ]}, + "device_operational_status": 1, + "device_drivers": ["smartnic"], + "device_endpoints": [] + } + ] +} diff --git a/src/nbi/tests/data/agent_probes_device.json b/src/nbi/tests/data/agent_probes_device.json new file mode 100644 index 000000000..e9d931528 --- /dev/null +++ b/src/nbi/tests/data/agent_probes_device.json @@ -0,0 +1,27 @@ +{ + + "contexts": [ + {"context_id": {"context_uuid": {"uuid": "admin"}}} + ], + "topologies": [ + {"topology_id": {"topology_uuid": {"uuid": "admin"}, "context_id": {"context_uuid": {"uuid": "admin"}}}} + ], + "devices": [ + { + "device_id": {"device_uuid": {"uuid": "smp-01"}}, + "device_type": "smartnic", + "device_config": {"config_rules": [ + {"action": 1, "custom": {"resource_key": "_connect/address", "resource_value": "172.17.0.3"}}, + {"action": 1, "custom": {"resource_key": "_connect/port", "resource_value": "8000"}}, + {"action": 1, "custom": {"resource_key": "_connect/settings", "resource_value": { + "username": "admin", "password": "admin", "base_url": "/manage-probe", "timeout" : 120 + }}} + ]}, + "device_operational_status": 1, + "device_drivers": [12], + "device_endpoints": [] + } + + ] +} + \ No newline at end of file diff --git a/src/policy/Dockerfile b/src/policy/Dockerfile deleted file mode 120000 index eec732273..000000000 --- a/src/policy/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -src/main/docker/Dockerfile.multistage.jvm \ No newline at end of file diff --git a/src/policy/Dockerfile b/src/policy/Dockerfile new file mode 100644 index 000000000..2c6412d07 --- /dev/null +++ b/src/policy/Dockerfile @@ -0,0 +1,66 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Multi-stage Docker image build + +# Stage 1 +FROM maven:3-jdk-11 AS builder + +# Define working directory +WORKDIR /app + +# Copy every file in working directory, as defined in .dockerignore file +COPY ./pom.xml pom.xml +COPY ./src src/ +COPY ./target/generated-sources/ target/generated-sources/ +RUN mvn --errors --batch-mode package -Dmaven.test.skip=true + +# Stage 2 +FROM builder AS unit-test + +RUN mvn --errors --batch-mode -Pgenerate-consolidated-coverage verify + +# Stage 3 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 AS release + +ARG JAVA_PACKAGE=java-11-openjdk-headless +ARG RUN_JAVA_VERSION=1.3.8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' +# Install java and the run-java script +# Also set up permissions for user `1001` +RUN microdnf install curl ca-certificates ${JAVA_PACKAGE} \ + && microdnf update \ + && microdnf clean all \ + && mkdir /deployments \ + && chown 1001 /deployments \ + && chmod "g+rwX" /deployments \ + && chown 1001:root /deployments \ + && curl https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh -o /deployments/run-java.sh \ + && chown 1001 /deployments/run-java.sh \ + && chmod 540 /deployments/run-java.sh \ + && echo "securerandom.source=file:/dev/urandom" >> /etc/alternatives/jre/conf/security/java.security + +# Configure the JAVA_OPTIONS, you can add -XshowSettings:vm to also display the heap size. +ENV JAVA_OPTIONS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --from=builder --chown=1001 /app/target/quarkus-app/lib/ /deployments/lib/ +COPY --from=builder --chown=1001 /app/target/quarkus-app/*.jar /deployments/ +COPY --from=builder --chown=1001 /app/target/quarkus-app/app/ /deployments/app/ +COPY --from=builder --chown=1001 /app/target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +EXPOSE 6060 +USER 1001 + +ENTRYPOINT [ "/deployments/run-java.sh" ] diff --git a/src/policy/src/main/java/org/etsi/tfs/policy/Serializer.java b/src/policy/src/main/java/org/etsi/tfs/policy/Serializer.java index 570a7fb9e..018a08bdf 100644 --- a/src/policy/src/main/java/org/etsi/tfs/policy/Serializer.java +++ b/src/policy/src/main/java/org/etsi/tfs/policy/Serializer.java @@ -2304,6 +2304,8 @@ public class Serializer { return ContextOuterClass.DeviceDriverEnum.DEVICEDRIVER_FLEXSCALE; case IETF_ACTN: return ContextOuterClass.DeviceDriverEnum.DEVICEDRIVER_IETF_ACTN; + case SMARTNIC: + return ContextOuterClass.DeviceDriverEnum.DEVICEDRIVER_SMARTNIC; case UNDEFINED: default: return ContextOuterClass.DeviceDriverEnum.DEVICEDRIVER_UNDEFINED; @@ -2333,6 +2335,8 @@ public class Serializer { return DeviceDriverEnum.FLEXSCALE; case DEVICEDRIVER_IETF_ACTN: return DeviceDriverEnum.IETF_ACTN; + case DEVICEDRIVER_SMARTNIC: + return DeviceDriverEnum.SMARTNIC; case DEVICEDRIVER_UNDEFINED: case UNRECOGNIZED: default: diff --git a/src/policy/src/main/java/org/etsi/tfs/policy/context/model/DeviceDriverEnum.java b/src/policy/src/main/java/org/etsi/tfs/policy/context/model/DeviceDriverEnum.java index 72a1d7136..937752dc8 100644 --- a/src/policy/src/main/java/org/etsi/tfs/policy/context/model/DeviceDriverEnum.java +++ b/src/policy/src/main/java/org/etsi/tfs/policy/context/model/DeviceDriverEnum.java @@ -27,5 +27,6 @@ public enum DeviceDriverEnum { IETF_L2VPN, GNMI_OPENCONFIG, FLEXSCALE, - IETF_ACTN + IETF_ACTN, + SMARTNIC } diff --git a/src/policy/src/main/proto/acl.proto b/src/policy/src/main/proto/acl.proto deleted file mode 120000 index 158ae78eb..000000000 --- a/src/policy/src/main/proto/acl.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/acl.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/acl.proto b/src/policy/src/main/proto/acl.proto new file mode 100644 index 000000000..3dba735dc --- /dev/null +++ b/src/policy/src/main/proto/acl.proto @@ -0,0 +1,69 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package acl; + +enum AclRuleTypeEnum { + ACLRULETYPE_UNDEFINED = 0; + ACLRULETYPE_IPV4 = 1; + ACLRULETYPE_IPV6 = 2; + ACLRULETYPE_L2 = 3; + ACLRULETYPE_MPLS = 4; + ACLRULETYPE_MIXED = 5; +} + +enum AclForwardActionEnum { + ACLFORWARDINGACTION_UNDEFINED = 0; + ACLFORWARDINGACTION_DROP = 1; + ACLFORWARDINGACTION_ACCEPT = 2; + ACLFORWARDINGACTION_REJECT = 3; +} + +enum AclLogActionEnum { + ACLLOGACTION_UNDEFINED = 0; + ACLLOGACTION_NOLOG = 1; + ACLLOGACTION_SYSLOG = 2; +} + +message AclMatch { + uint32 dscp = 1; + uint32 protocol = 2; + string src_address = 3; + string dst_address = 4; + uint32 src_port = 5; + uint32 dst_port = 6; + uint32 start_mpls_label = 7; + uint32 end_mpls_label = 8; +} + +message AclAction { + AclForwardActionEnum forward_action = 1; + AclLogActionEnum log_action = 2; +} + +message AclEntry { + uint32 sequence_id = 1; + string description = 2; + AclMatch match = 3; + AclAction action = 4; +} + +message AclRuleSet { + string name = 1; + AclRuleTypeEnum type = 2; + string description = 3; + string user_id = 4; + repeated AclEntry entries = 5; +} diff --git a/src/policy/src/main/proto/context.proto b/src/policy/src/main/proto/context.proto deleted file mode 120000 index 7f33c4bc7..000000000 --- a/src/policy/src/main/proto/context.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/context.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/context.proto b/src/policy/src/main/proto/context.proto new file mode 100644 index 000000000..fce1e71ad --- /dev/null +++ b/src/policy/src/main/proto/context.proto @@ -0,0 +1,615 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package context; + +import "acl.proto"; +import "kpi_sample_types.proto"; + +service ContextService { + rpc ListContextIds (Empty ) returns ( ContextIdList ) {} + rpc ListContexts (Empty ) returns ( ContextList ) {} + rpc GetContext (ContextId ) returns ( Context ) {} + rpc SetContext (Context ) returns ( ContextId ) {} + rpc RemoveContext (ContextId ) returns ( Empty ) {} + rpc GetContextEvents (Empty ) returns (stream ContextEvent ) {} + + rpc ListTopologyIds (ContextId ) returns ( TopologyIdList ) {} + rpc ListTopologies (ContextId ) returns ( TopologyList ) {} + rpc GetTopology (TopologyId ) returns ( Topology ) {} + rpc GetTopologyDetails (TopologyId ) returns ( TopologyDetails ) {} + rpc SetTopology (Topology ) returns ( TopologyId ) {} + rpc RemoveTopology (TopologyId ) returns ( Empty ) {} + rpc GetTopologyEvents (Empty ) returns (stream TopologyEvent ) {} + + rpc ListDeviceIds (Empty ) returns ( DeviceIdList ) {} + rpc ListDevices (Empty ) returns ( DeviceList ) {} + rpc GetDevice (DeviceId ) returns ( Device ) {} + rpc SetDevice (Device ) returns ( DeviceId ) {} + rpc RemoveDevice (DeviceId ) returns ( Empty ) {} + rpc GetDeviceEvents (Empty ) returns (stream DeviceEvent ) {} + rpc SelectDevice (DeviceFilter ) returns ( DeviceList ) {} + rpc ListEndPointNames (EndPointIdList) returns ( EndPointNameList) {} + + rpc ListLinkIds (Empty ) returns ( LinkIdList ) {} + rpc ListLinks (Empty ) returns ( LinkList ) {} + rpc GetLink (LinkId ) returns ( Link ) {} + rpc SetLink (Link ) returns ( LinkId ) {} + rpc RemoveLink (LinkId ) returns ( Empty ) {} + rpc GetLinkEvents (Empty ) returns (stream LinkEvent ) {} + + rpc ListServiceIds (ContextId ) returns ( ServiceIdList ) {} + rpc ListServices (ContextId ) returns ( ServiceList ) {} + rpc GetService (ServiceId ) returns ( Service ) {} + rpc SetService (Service ) returns ( ServiceId ) {} + rpc UnsetService (Service ) returns ( ServiceId ) {} + rpc RemoveService (ServiceId ) returns ( Empty ) {} + rpc GetServiceEvents (Empty ) returns (stream ServiceEvent ) {} + rpc SelectService (ServiceFilter ) returns ( ServiceList ) {} + + rpc ListSliceIds (ContextId ) returns ( SliceIdList ) {} + rpc ListSlices (ContextId ) returns ( SliceList ) {} + rpc GetSlice (SliceId ) returns ( Slice ) {} + rpc SetSlice (Slice ) returns ( SliceId ) {} + rpc UnsetSlice (Slice ) returns ( SliceId ) {} + rpc RemoveSlice (SliceId ) returns ( Empty ) {} + rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} + rpc SelectSlice (SliceFilter ) returns ( SliceList ) {} + + rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} + rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} + rpc GetConnection (ConnectionId ) returns ( Connection ) {} + rpc SetConnection (Connection ) returns ( ConnectionId ) {} + rpc RemoveConnection (ConnectionId ) returns ( Empty ) {} + rpc GetConnectionEvents(Empty ) returns (stream ConnectionEvent ) {} +} + +// ----- Generic ------------------------------------------------------------------------------------------------------- +message Empty {} + +message Uuid { + string uuid = 1; +} + +enum EventTypeEnum { + EVENTTYPE_UNDEFINED = 0; + EVENTTYPE_CREATE = 1; + EVENTTYPE_UPDATE = 2; + EVENTTYPE_REMOVE = 3; +} + +message Timestamp { + double timestamp = 1; +} + +message Event { + Timestamp timestamp = 1; + EventTypeEnum event_type = 2; +} + +// ----- Context ------------------------------------------------------------------------------------------------------- +message ContextId { + Uuid context_uuid = 1; +} + +message Context { + ContextId context_id = 1; + string name = 2; + repeated TopologyId topology_ids = 3; + repeated ServiceId service_ids = 4; + repeated SliceId slice_ids = 5; + TeraFlowController controller = 6; +} + +message ContextIdList { + repeated ContextId context_ids = 1; +} + +message ContextList { + repeated Context contexts = 1; +} + +message ContextEvent { + Event event = 1; + ContextId context_id = 2; +} + + +// ----- Topology ------------------------------------------------------------------------------------------------------ +message TopologyId { + ContextId context_id = 1; + Uuid topology_uuid = 2; +} + +message Topology { + TopologyId topology_id = 1; + string name = 2; + repeated DeviceId device_ids = 3; + repeated LinkId link_ids = 4; +} + +message TopologyDetails { + TopologyId topology_id = 1; + string name = 2; + repeated Device devices = 3; + repeated Link links = 4; +} + +message TopologyIdList { + repeated TopologyId topology_ids = 1; +} + +message TopologyList { + repeated Topology topologies = 1; +} + +message TopologyEvent { + Event event = 1; + TopologyId topology_id = 2; +} + + +// ----- Device -------------------------------------------------------------------------------------------------------- +message DeviceId { + Uuid device_uuid = 1; +} + +message Device { + DeviceId device_id = 1; + string name = 2; + string device_type = 3; + DeviceConfig device_config = 4; + DeviceOperationalStatusEnum device_operational_status = 5; + repeated DeviceDriverEnum device_drivers = 6; + repeated EndPoint device_endpoints = 7; + repeated Component components = 8; // Used for inventory + DeviceId controller_id = 9; // Identifier of node controlling the actual device +} + +message Component { //Defined previously to this section - Tested OK + Uuid component_uuid = 1; + string name = 2; + string type = 3; + + map attributes = 4; // dict[attr.name => json.dumps(attr.value)] + string parent = 5; +} + +message DeviceConfig { + repeated ConfigRule config_rules = 1; +} + +enum DeviceDriverEnum { + DEVICEDRIVER_UNDEFINED = 0; // also used for emulated + DEVICEDRIVER_OPENCONFIG = 1; + DEVICEDRIVER_TRANSPORT_API = 2; + DEVICEDRIVER_P4 = 3; + DEVICEDRIVER_IETF_NETWORK_TOPOLOGY = 4; + DEVICEDRIVER_ONF_TR_532 = 5; + DEVICEDRIVER_XR = 6; + DEVICEDRIVER_IETF_L2VPN = 7; + DEVICEDRIVER_GNMI_OPENCONFIG = 8; + DEVICEDRIVER_FLEXSCALE = 9; + DEVICEDRIVER_IETF_ACTN = 10; + DEVICEDRIVER_SMARTNIC = 11; +} + +enum DeviceOperationalStatusEnum { + DEVICEOPERATIONALSTATUS_UNDEFINED = 0; + DEVICEOPERATIONALSTATUS_DISABLED = 1; + DEVICEOPERATIONALSTATUS_ENABLED = 2; +} + +message DeviceIdList { + repeated DeviceId device_ids = 1; +} + +message DeviceList { + repeated Device devices = 1; +} + +message DeviceFilter { + DeviceIdList device_ids = 1; + bool include_endpoints = 2; + bool include_config_rules = 3; + bool include_components = 4; +} + +message DeviceEvent { + Event event = 1; + DeviceId device_id = 2; + DeviceConfig device_config = 3; +} + + +// ----- Link ---------------------------------------------------------------------------------------------------------- +message LinkId { + Uuid link_uuid = 1; +} + +message LinkAttributes { + float total_capacity_gbps = 1; + float used_capacity_gbps = 2; +} + +message Link { + LinkId link_id = 1; + string name = 2; + repeated EndPointId link_endpoint_ids = 3; + LinkAttributes attributes = 4; +} + +message LinkIdList { + repeated LinkId link_ids = 1; +} + +message LinkList { + repeated Link links = 1; +} + +message LinkEvent { + Event event = 1; + LinkId link_id = 2; +} + + +// ----- Service ------------------------------------------------------------------------------------------------------- +message ServiceId { + ContextId context_id = 1; + Uuid service_uuid = 2; +} + +message Service { + ServiceId service_id = 1; + string name = 2; + ServiceTypeEnum service_type = 3; + repeated EndPointId service_endpoint_ids = 4; + repeated Constraint service_constraints = 5; + ServiceStatus service_status = 6; + ServiceConfig service_config = 7; + Timestamp timestamp = 8; +} + +enum ServiceTypeEnum { + SERVICETYPE_UNKNOWN = 0; + SERVICETYPE_L3NM = 1; + SERVICETYPE_L2NM = 2; + SERVICETYPE_TAPI_CONNECTIVITY_SERVICE = 3; + SERVICETYPE_TE = 4; + SERVICETYPE_E2E = 5; +} + +enum ServiceStatusEnum { + SERVICESTATUS_UNDEFINED = 0; + SERVICESTATUS_PLANNED = 1; + SERVICESTATUS_ACTIVE = 2; + SERVICESTATUS_UPDATING = 3; + SERVICESTATUS_PENDING_REMOVAL = 4; + SERVICESTATUS_SLA_VIOLATED = 5; +} + +message ServiceStatus { + ServiceStatusEnum service_status = 1; +} + +message ServiceConfig { + repeated ConfigRule config_rules = 1; +} + +message ServiceIdList { + repeated ServiceId service_ids = 1; +} + +message ServiceList { + repeated Service services = 1; +} + +message ServiceFilter { + ServiceIdList service_ids = 1; + bool include_endpoint_ids = 2; + bool include_constraints = 3; + bool include_config_rules = 4; +} + +message ServiceEvent { + Event event = 1; + ServiceId service_id = 2; +} + +// ----- Slice --------------------------------------------------------------------------------------------------------- +message SliceId { + ContextId context_id = 1; + Uuid slice_uuid = 2; +} + +message Slice { + SliceId slice_id = 1; + string name = 2; + repeated EndPointId slice_endpoint_ids = 3; + repeated Constraint slice_constraints = 4; + repeated ServiceId slice_service_ids = 5; + repeated SliceId slice_subslice_ids = 6; + SliceStatus slice_status = 7; + SliceConfig slice_config = 8; + SliceOwner slice_owner = 9; + Timestamp timestamp = 10; +} + +message SliceOwner { + Uuid owner_uuid = 1; + string owner_string = 2; +} + +enum SliceStatusEnum { + SLICESTATUS_UNDEFINED = 0; + SLICESTATUS_PLANNED = 1; + SLICESTATUS_INIT = 2; + SLICESTATUS_ACTIVE = 3; + SLICESTATUS_DEINIT = 4; + SLICESTATUS_SLA_VIOLATED = 5; +} + +message SliceStatus { + SliceStatusEnum slice_status = 1; +} + +message SliceConfig { + repeated ConfigRule config_rules = 1; +} + +message SliceIdList { + repeated SliceId slice_ids = 1; +} + +message SliceList { + repeated Slice slices = 1; +} + +message SliceFilter { + SliceIdList slice_ids = 1; + bool include_endpoint_ids = 2; + bool include_constraints = 3; + bool include_service_ids = 4; + bool include_subslice_ids = 5; + bool include_config_rules = 6; +} + +message SliceEvent { + Event event = 1; + SliceId slice_id = 2; +} + +// ----- Connection ---------------------------------------------------------------------------------------------------- +message ConnectionId { + Uuid connection_uuid = 1; +} + +message ConnectionSettings_L0 { + string lsp_symbolic_name = 1; +} + +message ConnectionSettings_L2 { + string src_mac_address = 1; + string dst_mac_address = 2; + uint32 ether_type = 3; + uint32 vlan_id = 4; + uint32 mpls_label = 5; + uint32 mpls_traffic_class = 6; +} + +message ConnectionSettings_L3 { + string src_ip_address = 1; + string dst_ip_address = 2; + uint32 dscp = 3; + uint32 protocol = 4; + uint32 ttl = 5; +} + +message ConnectionSettings_L4 { + uint32 src_port = 1; + uint32 dst_port = 2; + uint32 tcp_flags = 3; + uint32 ttl = 4; +} + +message ConnectionSettings { + ConnectionSettings_L0 l0 = 1; + ConnectionSettings_L2 l2 = 2; + ConnectionSettings_L3 l3 = 3; + ConnectionSettings_L4 l4 = 4; +} + +message Connection { + ConnectionId connection_id = 1; + ServiceId service_id = 2; + repeated EndPointId path_hops_endpoint_ids = 3; + repeated ServiceId sub_service_ids = 4; + ConnectionSettings settings = 5; +} + +message ConnectionIdList { + repeated ConnectionId connection_ids = 1; +} + +message ConnectionList { + repeated Connection connections = 1; +} + +message ConnectionEvent { + Event event = 1; + ConnectionId connection_id = 2; +} + + +// ----- Endpoint ------------------------------------------------------------------------------------------------------ +message EndPointId { + TopologyId topology_id = 1; + DeviceId device_id = 2; + Uuid endpoint_uuid = 3; +} + +message EndPoint { + EndPointId endpoint_id = 1; + string name = 2; + string endpoint_type = 3; + repeated kpi_sample_types.KpiSampleType kpi_sample_types = 4; + Location endpoint_location = 5; +} + +message EndPointName { + EndPointId endpoint_id = 1; + string device_name = 2; + string endpoint_name = 3; + string endpoint_type = 4; +} + +message EndPointIdList { + repeated EndPointId endpoint_ids = 1; +} + +message EndPointNameList { + repeated EndPointName endpoint_names = 1; +} + + +// ----- Configuration ------------------------------------------------------------------------------------------------- +enum ConfigActionEnum { + CONFIGACTION_UNDEFINED = 0; + CONFIGACTION_SET = 1; + CONFIGACTION_DELETE = 2; +} + +message ConfigRule_Custom { + string resource_key = 1; + string resource_value = 2; +} + +message ConfigRule_ACL { + EndPointId endpoint_id = 1; + acl.AclRuleSet rule_set = 2; +} + +message ConfigRule { + ConfigActionEnum action = 1; + oneof config_rule { + ConfigRule_Custom custom = 2; + ConfigRule_ACL acl = 3; + } +} + + +// ----- Constraint ---------------------------------------------------------------------------------------------------- +enum ConstraintActionEnum { + CONSTRAINTACTION_UNDEFINED = 0; + CONSTRAINTACTION_SET = 1; + CONSTRAINTACTION_DELETE = 2; +} + +message Constraint_Custom { + string constraint_type = 1; + string constraint_value = 2; +} + +message Constraint_Schedule { + float start_timestamp = 1; + float duration_days = 2; +} + +message GPS_Position { + float latitude = 1; + float longitude = 2; +} + +message Location { + oneof location { + string region = 1; + GPS_Position gps_position = 2; + } +} + +message Constraint_EndPointLocation { + EndPointId endpoint_id = 1; + Location location = 2; +} + +message Constraint_EndPointPriority { + EndPointId endpoint_id = 1; + uint32 priority = 2; +} + +message Constraint_SLA_Latency { + float e2e_latency_ms = 1; +} + +message Constraint_SLA_Capacity { + float capacity_gbps = 1; +} + +message Constraint_SLA_Availability { + uint32 num_disjoint_paths = 1; + bool all_active = 2; + float availability = 3; // 0.0 .. 100.0 percentage of availability +} + +enum IsolationLevelEnum { + NO_ISOLATION = 0; + PHYSICAL_ISOLATION = 1; + LOGICAL_ISOLATION = 2; + PROCESS_ISOLATION = 3; + PHYSICAL_MEMORY_ISOLATION = 4; + PHYSICAL_NETWORK_ISOLATION = 5; + VIRTUAL_RESOURCE_ISOLATION = 6; + NETWORK_FUNCTIONS_ISOLATION = 7; + SERVICE_ISOLATION = 8; +} + +message Constraint_SLA_Isolation_level { + repeated IsolationLevelEnum isolation_level = 1; +} + +message Constraint_Exclusions { + bool is_permanent = 1; + repeated DeviceId device_ids = 2; + repeated EndPointId endpoint_ids = 3; + repeated LinkId link_ids = 4; +} + +message Constraint { + ConstraintActionEnum action = 1; + oneof constraint { + Constraint_Custom custom = 2; + Constraint_Schedule schedule = 3; + Constraint_EndPointLocation endpoint_location = 4; + Constraint_EndPointPriority endpoint_priority = 5; + Constraint_SLA_Capacity sla_capacity = 6; + Constraint_SLA_Latency sla_latency = 7; + Constraint_SLA_Availability sla_availability = 8; + Constraint_SLA_Isolation_level sla_isolation = 9; + Constraint_Exclusions exclusions = 10; + } +} + + +// ----- Miscellaneous ------------------------------------------------------------------------------------------------- +message TeraFlowController { + ContextId context_id = 1; + string ip_address = 2; + uint32 port = 3; +} + +message AuthenticationResult { + ContextId context_id = 1; + bool authenticated = 2; +} diff --git a/src/policy/src/main/proto/context_policy.proto b/src/policy/src/main/proto/context_policy.proto deleted file mode 120000 index d41593dde..000000000 --- a/src/policy/src/main/proto/context_policy.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/context_policy.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/context_policy.proto b/src/policy/src/main/proto/context_policy.proto new file mode 100644 index 000000000..f6dae4830 --- /dev/null +++ b/src/policy/src/main/proto/context_policy.proto @@ -0,0 +1,28 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package context_policy; + +import "context.proto"; +import "policy.proto"; + +// created as a separate service to prevent import-loops in context and policy +service ContextPolicyService { + rpc ListPolicyRuleIds(context.Empty ) returns (policy.PolicyRuleIdList) {} + rpc ListPolicyRules (context.Empty ) returns (policy.PolicyRuleList ) {} + rpc GetPolicyRule (policy.PolicyRuleId ) returns (policy.PolicyRule ) {} + rpc SetPolicyRule (policy.PolicyRule ) returns (policy.PolicyRuleId ) {} + rpc RemovePolicyRule (policy.PolicyRuleId ) returns (context.Empty ) {} +} diff --git a/src/policy/src/main/proto/device.proto b/src/policy/src/main/proto/device.proto deleted file mode 120000 index ad6e7c47e..000000000 --- a/src/policy/src/main/proto/device.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/device.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/device.proto b/src/policy/src/main/proto/device.proto new file mode 100644 index 000000000..30e60079d --- /dev/null +++ b/src/policy/src/main/proto/device.proto @@ -0,0 +1,34 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package device; + +import "context.proto"; +import "monitoring.proto"; + +service DeviceService { + rpc AddDevice (context.Device ) returns (context.DeviceId ) {} + rpc ConfigureDevice (context.Device ) returns (context.DeviceId ) {} + rpc DeleteDevice (context.DeviceId ) returns (context.Empty ) {} + rpc GetInitialConfig(context.DeviceId ) returns (context.DeviceConfig) {} + rpc MonitorDeviceKpi(MonitoringSettings) returns (context.Empty ) {} +} + +message MonitoringSettings { + monitoring.KpiId kpi_id = 1; + monitoring.KpiDescriptor kpi_descriptor = 2; + float sampling_duration_s = 3; + float sampling_interval_s = 4; +} diff --git a/src/policy/src/main/proto/kpi_sample_types.proto b/src/policy/src/main/proto/kpi_sample_types.proto deleted file mode 120000 index 98e748bbf..000000000 --- a/src/policy/src/main/proto/kpi_sample_types.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/kpi_sample_types.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/kpi_sample_types.proto b/src/policy/src/main/proto/kpi_sample_types.proto new file mode 100644 index 000000000..5b234a4e3 --- /dev/null +++ b/src/policy/src/main/proto/kpi_sample_types.proto @@ -0,0 +1,42 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package kpi_sample_types; + +enum KpiSampleType { + KPISAMPLETYPE_UNKNOWN = 0; + + KPISAMPLETYPE_PACKETS_TRANSMITTED = 101; + KPISAMPLETYPE_PACKETS_RECEIVED = 102; + KPISAMPLETYPE_PACKETS_DROPPED = 103; + KPISAMPLETYPE_BYTES_TRANSMITTED = 201; + KPISAMPLETYPE_BYTES_RECEIVED = 202; + KPISAMPLETYPE_BYTES_DROPPED = 203; + + KPISAMPLETYPE_LINK_TOTAL_CAPACITY_GBPS = 301; + KPISAMPLETYPE_LINK_USED_CAPACITY_GBPS = 302; + + KPISAMPLETYPE_ML_CONFIDENCE = 401; //. can be used by both optical and L3 without any issue + + KPISAMPLETYPE_OPTICAL_SECURITY_STATUS = 501; //. can be used by both optical and L3 without any issue + + KPISAMPLETYPE_L3_UNIQUE_ATTACK_CONNS = 601; + KPISAMPLETYPE_L3_TOTAL_DROPPED_PACKTS = 602; + KPISAMPLETYPE_L3_UNIQUE_ATTACKERS = 603; + KPISAMPLETYPE_L3_UNIQUE_COMPROMISED_CLIENTS = 604; + KPISAMPLETYPE_L3_SECURITY_STATUS_CRYPTO = 605; + + KPISAMPLETYPE_SERVICE_LATENCY_MS = 701; +} diff --git a/src/policy/src/main/proto/monitoring.proto b/src/policy/src/main/proto/monitoring.proto deleted file mode 120000 index aceaa7328..000000000 --- a/src/policy/src/main/proto/monitoring.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/monitoring.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/monitoring.proto b/src/policy/src/main/proto/monitoring.proto new file mode 100644 index 000000000..45ba48b02 --- /dev/null +++ b/src/policy/src/main/proto/monitoring.proto @@ -0,0 +1,174 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package monitoring; + +import "context.proto"; +import "kpi_sample_types.proto"; + +service MonitoringService { + rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final + rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final + rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final + rpc IncludeKpi (Kpi ) returns (context.Empty ) {} // Stable and final + rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {} // Stable and final + rpc QueryKpiData (KpiQuery ) returns (RawKpiTable ) {} // Not implemented + rpc SetKpiSubscription (SubsDescriptor ) returns (stream SubsResponse ) {} // Stable not final + rpc GetSubsDescriptor (SubscriptionID ) returns (SubsDescriptor ) {} // Stable and final + rpc GetSubscriptions (context.Empty ) returns (SubsList ) {} // Stable and final + rpc DeleteSubscription (SubscriptionID ) returns (context.Empty ) {} // Stable and final + rpc SetKpiAlarm (AlarmDescriptor ) returns (AlarmID ) {} // Stable not final + rpc GetAlarms (context.Empty ) returns (AlarmList ) {} // Stable and final + rpc GetAlarmDescriptor (AlarmID ) returns (AlarmDescriptor ) {} // Stable and final + rpc GetAlarmResponseStream(AlarmSubscription ) returns (stream AlarmResponse) {} // Not Stable not final + rpc DeleteAlarm (AlarmID ) returns (context.Empty ) {} // Stable and final + rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {} // Stable not final + rpc GetInstantKpi (KpiId ) returns (Kpi ) {} // Stable not final +} + +message KpiDescriptor { + KpiId kpi_id = 1; + string kpi_description = 2; + repeated KpiId kpi_id_list = 3; + kpi_sample_types.KpiSampleType kpi_sample_type = 4; + context.DeviceId device_id = 5; + context.EndPointId endpoint_id = 6; + context.ServiceId service_id = 7; + context.SliceId slice_id = 8; + context.ConnectionId connection_id = 9; + context.LinkId link_id = 10; +} + +message MonitorKpiRequest { + KpiId kpi_id = 1; + float monitoring_window_s = 2; + float sampling_rate_s = 3; + // Pending add field to reflect Available Device Protocols +} + +message KpiQuery { + repeated KpiId kpi_ids = 1; + float monitoring_window_s = 2; + uint32 last_n_samples = 3; // used when you want something like "get the last N many samples + context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time" + context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time" +} + + +message RawKpi { // cell + context.Timestamp timestamp = 1; + KpiValue kpi_value = 2; +} + +message RawKpiList { // column + KpiId kpi_id = 1; + repeated RawKpi raw_kpis = 2; +} + +message RawKpiTable { // table + repeated RawKpiList raw_kpi_lists = 1; +} + +message KpiId { + context.Uuid kpi_id = 1; +} + +message Kpi { + KpiId kpi_id = 1; + context.Timestamp timestamp = 2; + KpiValue kpi_value = 3; +} + +message KpiValueRange { + KpiValue kpiMinValue = 1; + KpiValue kpiMaxValue = 2; + bool inRange = 3; // by default True + bool includeMinValue = 4; // False is outside the interval + bool includeMaxValue = 5; // False is outside the interval +} + +message KpiValue { + oneof value { + int32 int32Val = 1; + uint32 uint32Val = 2; + int64 int64Val = 3; + uint64 uint64Val = 4; + float floatVal = 5; + string stringVal = 6; + bool boolVal = 7; + } +} + + +message KpiList { + repeated Kpi kpi = 1; +} + +message KpiDescriptorList { + repeated KpiDescriptor kpi_descriptor_list = 1; +} + +message SubsDescriptor{ + SubscriptionID subs_id = 1; + KpiId kpi_id = 2; + float sampling_duration_s = 3; + float sampling_interval_s = 4; + context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time" + context.Timestamp end_timestamp = 6; // used when you want something like "get the samples until X date/time" + // Pending add field to reflect Available Device Protocols +} + +message SubscriptionID { + context.Uuid subs_id = 1; +} + +message SubsResponse { + SubscriptionID subs_id = 1; + KpiList kpi_list = 2; +} + +message SubsList { + repeated SubsDescriptor subs_descriptor = 1; +} + +message AlarmDescriptor { + AlarmID alarm_id = 1; + string alarm_description = 2; + string name = 3; + KpiId kpi_id = 4; + KpiValueRange kpi_value_range = 5; + context.Timestamp timestamp = 6; +} + +message AlarmID{ + context.Uuid alarm_id = 1; +} + +message AlarmSubscription{ + AlarmID alarm_id = 1; + float subscription_timeout_s = 2; + float subscription_frequency_ms = 3; +} + +message AlarmResponse { + AlarmID alarm_id = 1; + string text = 2; + KpiList kpi_list = 3; +} + +message AlarmList { + repeated AlarmDescriptor alarm_descriptor = 1; +} diff --git a/src/policy/src/main/proto/policy.proto b/src/policy/src/main/proto/policy.proto deleted file mode 120000 index df455f961..000000000 --- a/src/policy/src/main/proto/policy.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/policy.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/policy.proto b/src/policy/src/main/proto/policy.proto new file mode 100644 index 000000000..a6f160150 --- /dev/null +++ b/src/policy/src/main/proto/policy.proto @@ -0,0 +1,113 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package policy; + +import "context.proto"; +import "policy_condition.proto"; +import "policy_action.proto"; + +service PolicyService { + rpc PolicyAddService (PolicyRuleService) returns (PolicyRuleState) {} + rpc PolicyAddDevice (PolicyRuleDevice) returns (PolicyRuleState) {} + rpc PolicyUpdateService (PolicyRuleService) returns (PolicyRuleState) {} + rpc PolicyUpdateDevice (PolicyRuleDevice) returns (PolicyRuleState) {} + rpc PolicyDelete (PolicyRuleId) returns (PolicyRuleState) {} + rpc GetPolicyService (PolicyRuleId) returns (PolicyRuleService) {} + rpc GetPolicyDevice (PolicyRuleId) returns (PolicyRuleDevice) {} + rpc GetPolicyByServiceId (context.ServiceId) returns (PolicyRuleServiceList) {} +} + +enum PolicyRuleStateEnum { + POLICY_UNDEFINED = 0; // Undefined rule state + POLICY_FAILED = 1; // Rule failed + POLICY_INSERTED = 2; // Rule is just inserted + POLICY_VALIDATED = 3; // Rule content is correct + POLICY_PROVISIONED = 4; // Rule subscribed to Monitoring + POLICY_ACTIVE = 5; // Rule is currently active (alarm is just thrown by Monitoring) + POLICY_ENFORCED = 6; // Rule action is successfully enforced + POLICY_INEFFECTIVE = 7; // The applied rule action did not work as expected + POLICY_EFFECTIVE = 8; // The applied rule action did work as expected + POLICY_UPDATED = 9; // Operator requires a policy to change + POLICY_REMOVED = 10; // Operator requires to remove a policy +} + +message PolicyRuleId { + context.Uuid uuid = 1; +} + +message PolicyRuleState { + PolicyRuleStateEnum policyRuleState = 1; + string policyRuleStateMessage = 2; +} + +// Basic policy rule attributes +message PolicyRuleBasic { + PolicyRuleId policyRuleId = 1; + PolicyRuleState policyRuleState = 2; //policy.proto:58:12: Explicit 'optional' labels are disallowed in the Proto3 syntax. To define 'optional' fields in Proto3, simply remove the 'optional' label, as fields are 'optional' by default. + uint32 priority = 3; + + // Event-Condition-Action (ECA) model + repeated PolicyRuleCondition conditionList = 4; // When these policy conditions are met, an event is automatically thrown + BooleanOperator booleanOperator = 5; // Evaluation operator to be used + repeated PolicyRuleAction actionList = 6; // One or more actions should be applied +} + +// Service-oriented policy rule +message PolicyRuleService { + // Basic policy rule attributes + PolicyRuleBasic policyRuleBasic = 1; + + // Affected service and (some of) its device(s) + context.ServiceId serviceId = 2; + repeated context.DeviceId deviceList = 3; // List of devices this service is traversing (not exhaustive) +} + +// Device-oriented policy rule +message PolicyRuleDevice { + // Basic policy rule attributes + PolicyRuleBasic policyRuleBasic = 1; + + // Affected device(s) + repeated context.DeviceId deviceList = 2; +} + +// Wrapper policy rule object +message PolicyRule { + oneof policy_rule { + PolicyRuleService service = 1; + PolicyRuleDevice device = 2; + } +} + +// A list of policy rule IDs +message PolicyRuleIdList { + repeated PolicyRuleId policyRuleIdList = 1; +} + +// A list of service-oriented policy rules +message PolicyRuleServiceList { + repeated PolicyRuleService policyRuleServiceList = 1; +} + +// A list of device-oriented policy rules +message PolicyRuleDeviceList { + repeated PolicyRuleDevice policyRuleDeviceList = 1; +} + +// A list of policy rules +message PolicyRuleList { + repeated PolicyRule policyRules = 1; +} diff --git a/src/policy/src/main/proto/policy_action.proto b/src/policy/src/main/proto/policy_action.proto deleted file mode 120000 index 63dcef3d2..000000000 --- a/src/policy/src/main/proto/policy_action.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/policy_action.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/policy_action.proto b/src/policy/src/main/proto/policy_action.proto new file mode 100644 index 000000000..d547e9779 --- /dev/null +++ b/src/policy/src/main/proto/policy_action.proto @@ -0,0 +1,42 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package policy; + +// Action +message PolicyRuleAction { + PolicyRuleActionEnum action = 1; + repeated PolicyRuleActionConfig action_config = 2; +} + +enum PolicyRuleActionEnum { + POLICYRULE_ACTION_NO_ACTION = 0; + POLICYRULE_ACTION_SET_DEVICE_STATUS = 1; + POLICYRULE_ACTION_ADD_SERVICE_CONFIGRULE = 2; + POLICYRULE_ACTION_ADD_SERVICE_CONSTRAINT = 3; + POLICY_RULE_ACTION_CALL_SERVICE_RPC = 4; + POLICY_RULE_ACTION_RECALCULATE_PATH = 5; +} + +// Action configuration +message PolicyRuleActionConfig { + string action_key = 1; + string action_value = 2; +} + +// message PolicyRuleAction { +// PolicyRuleActionEnum action = 1; +// repeated string parameters = 2; +// } diff --git a/src/policy/src/main/proto/policy_condition.proto b/src/policy/src/main/proto/policy_condition.proto deleted file mode 120000 index 31f7d9d10..000000000 --- a/src/policy/src/main/proto/policy_condition.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/policy_condition.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/policy_condition.proto b/src/policy/src/main/proto/policy_condition.proto new file mode 100644 index 000000000..2037af93c --- /dev/null +++ b/src/policy/src/main/proto/policy_condition.proto @@ -0,0 +1,43 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package policy; + +import "monitoring.proto"; + +// Condition +message PolicyRuleCondition { + monitoring.KpiId kpiId = 1; + NumericalOperator numericalOperator = 2; + monitoring.KpiValue kpiValue = 3; +} + +// Operator to be used when comparing Kpis with condition values +enum NumericalOperator { + POLICYRULE_CONDITION_NUMERICAL_UNDEFINED = 0; // Kpi numerical operator undefined + POLICYRULE_CONDITION_NUMERICAL_EQUAL = 1; // Kpi is equal with value + POLICYRULE_CONDITION_NUMERICAL_NOT_EQUAL = 2; // Kpi is not equal with value + POLICYRULE_CONDITION_NUMERICAL_LESS_THAN = 3; // Kpi is less than value + POLICYRULE_CONDITION_NUMERICAL_LESS_THAN_EQUAL = 4; // Kpi is less than or equal with value + POLICYRULE_CONDITION_NUMERICAL_GREATER_THAN = 5; // Kpi is greater than value + POLICYRULE_CONDITION_NUMERICAL_GREATER_THAN_EQUAL = 6; // Kpi is less than or equal with value +} + +// Operator to be used when evaluating each condition +enum BooleanOperator { + POLICYRULE_CONDITION_BOOLEAN_UNDEFINED = 0; // Boolean operator undefined + POLICYRULE_CONDITION_BOOLEAN_AND = 1; // Boolean AND operator + POLICYRULE_CONDITION_BOOLEAN_OR = 2; // Boolean OR operator +} \ No newline at end of file diff --git a/src/policy/src/main/proto/service.proto b/src/policy/src/main/proto/service.proto deleted file mode 120000 index 5ca543da0..000000000 --- a/src/policy/src/main/proto/service.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/service.proto \ No newline at end of file diff --git a/src/policy/src/main/proto/service.proto b/src/policy/src/main/proto/service.proto new file mode 100644 index 000000000..658859e3c --- /dev/null +++ b/src/policy/src/main/proto/service.proto @@ -0,0 +1,25 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package service; + +import "context.proto"; + +service ServiceService { + rpc CreateService (context.Service ) returns (context.ServiceId) {} + rpc UpdateService (context.Service ) returns (context.ServiceId) {} + rpc DeleteService (context.ServiceId) returns (context.Empty ) {} + rpc RecomputeConnections(context.Service ) returns (context.Empty ) {} +} diff --git a/src/service/service/service_handler_api/FilterFields.py b/src/service/service/service_handler_api/FilterFields.py index e771e24f1..e985fe292 100644 --- a/src/service/service/service_handler_api/FilterFields.py +++ b/src/service/service/service_handler_api/FilterFields.py @@ -40,6 +40,7 @@ DEVICE_DRIVER_VALUES = { DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG, DeviceDriverEnum.DEVICEDRIVER_FLEXSCALE, DeviceDriverEnum.DEVICEDRIVER_IETF_ACTN, + DeviceDriverEnum.DEVICEDRIVER_SMARTNIC } # Map allowed filter fields to allowed values per Filter field. If no restriction (free text) None is specified diff --git a/src/service/service/service_handlers/__init__.py b/src/service/service/service_handlers/__init__.py index eaf8f715a..61376c0d2 100644 --- a/src/service/service/service_handlers/__init__.py +++ b/src/service/service/service_handlers/__init__.py @@ -94,6 +94,12 @@ SERVICE_HANDLERS = [ FilterFieldEnum.DEVICE_DRIVER : [DeviceDriverEnum.DEVICEDRIVER_IETF_L2VPN], } ]), + # (SMARTNIC_ServiceHandler, [ + # { + # FilterFieldEnum.SERVICE_TYPE : ServiceTypeEnum.SERVICETYPE_L2NM, + # FilterFieldEnum.DEVICE_DRIVER : [DeviceDriverEnum.DEVICEDRIVER_SMARTNIC], + # } + # ]), (E2EOrchestratorServiceHandler, [ { FilterFieldEnum.SERVICE_TYPE : ServiceTypeEnum.SERVICETYPE_E2E, diff --git a/src/webui/service/device/forms.py b/src/webui/service/device/forms.py index 4c04bbfe1..af5ac4564 100644 --- a/src/webui/service/device/forms.py +++ b/src/webui/service/device/forms.py @@ -33,6 +33,7 @@ class AddDeviceForm(FlaskForm): device_drivers_gnmi_openconfig = BooleanField('GNMI OPENCONFIG') device_drivers_flexscale = BooleanField('FLEXSCALE') device_drivers_ietf_actn = BooleanField('IETF ACTN') + device_drivers_smartnic = BooleanField('SMARTNIC') device_config_address = StringField('connect/address',default='127.0.0.1',validators=[DataRequired(), Length(min=5)]) device_config_port = StringField('connect/port',default='0',validators=[DataRequired(), Length(min=1)]) diff --git a/src/webui/service/device/routes.py b/src/webui/service/device/routes.py index 8b8bc236a..8aaaafccf 100644 --- a/src/webui/service/device/routes.py +++ b/src/webui/service/device/routes.py @@ -127,6 +127,8 @@ def add(): device_drivers.append(DeviceDriverEnum.DEVICEDRIVER_GNMI_OPENCONFIG) if form.device_drivers_flexscale.data: device_drivers.append(DeviceDriverEnum.DEVICEDRIVER_FLEXSCALE) + if form.device_drivers_smartnic.data: + device_drivers.append(DeviceDriverEnum.DEVICEDRIVER_SMARTNIC) if form.device_drivers_ietf_actn.data: device_drivers.append(DeviceDriverEnum.DEVICEDRIVER_IETF_ACTN) device_obj.device_drivers.extend(device_drivers) # pylint: disable=no-member diff --git a/src/webui/service/templates/device/add.html b/src/webui/service/templates/device/add.html index c4d7f1685..484bc30bf 100644 --- a/src/webui/service/templates/device/add.html +++ b/src/webui/service/templates/device/add.html @@ -95,6 +95,7 @@
{{ form.device_drivers_flexscale }} {{ form.device_drivers_flexscale.label(class="col-sm-3 col-form-label") }} {{ form.device_drivers_ietf_actn }} {{ form.device_drivers_ietf_actn.label(class="col-sm-3 col-form-label") }} + {{ form.device_drivers_smartnic }} {{ form.device_drivers_smartnic_actn.label(class="col-sm-3 col-form-label") }} {% endif %} diff --git a/src/ztp/Dockerfile b/src/ztp/Dockerfile deleted file mode 120000 index eec732273..000000000 --- a/src/ztp/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -src/main/docker/Dockerfile.multistage.jvm \ No newline at end of file diff --git a/src/ztp/Dockerfile b/src/ztp/Dockerfile new file mode 100644 index 000000000..43fef96b4 --- /dev/null +++ b/src/ztp/Dockerfile @@ -0,0 +1,67 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Multi-stage Docker image build + +# Stage 1 +FROM maven:3-jdk-11 AS builder + +# Define working directory +WORKDIR /app + +# Copy every file in working directory, as defined in .dockerignore file +COPY ./pom.xml pom.xml +COPY ./src src/ +COPY ./target/generated-sources/ target/generated-sources/ +RUN mvn --errors --batch-mode package -Dmaven.test.skip=true + +# Stage 2 +FROM builder AS unit-test + +RUN mvn --errors --batch-mode -Pgenerate-consolidated-coverage verify + +# Stage 3 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 AS release + +ARG JAVA_PACKAGE=java-11-openjdk-headless +ARG RUN_JAVA_VERSION=1.3.8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' +# Install java and the run-java script +# Also set up permissions for user `1001` +RUN microdnf install curl ca-certificates ${JAVA_PACKAGE} \ + && microdnf update \ + && microdnf clean all \ + && mkdir /deployments \ + && chown 1001 /deployments \ + && chmod "g+rwX" /deployments \ + && chown 1001:root /deployments \ + && curl https://repo1.maven.org/maven2/io/fabric8/run-java-sh/${RUN_JAVA_VERSION}/run-java-sh-${RUN_JAVA_VERSION}-sh.sh -o /deployments/run-java.sh \ + && chown 1001 /deployments/run-java.sh \ + && chmod 540 /deployments/run-java.sh \ + && echo "securerandom.source=file:/dev/urandom" >> /etc/alternatives/jre/conf/security/java.security + +# Configure the JAVA_OPTIONS, you can add -XshowSettings:vm to also display the heap size. +ENV JAVA_OPTIONS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --from=builder --chown=1001 /app/target/quarkus-app/lib/ /deployments/lib/ +COPY --from=builder --chown=1001 /app/target/quarkus-app/*.jar /deployments/ +COPY --from=builder --chown=1001 /app/target/quarkus-app/app/ /deployments/app/ +COPY --from=builder --chown=1001 /app/target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +EXPOSE 5050 +USER 1001 + +ENTRYPOINT [ "/deployments/run-java.sh" ] + diff --git a/src/ztp/src/main/proto/acl.proto b/src/ztp/src/main/proto/acl.proto deleted file mode 120000 index 158ae78eb..000000000 --- a/src/ztp/src/main/proto/acl.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/acl.proto \ No newline at end of file diff --git a/src/ztp/src/main/proto/acl.proto b/src/ztp/src/main/proto/acl.proto new file mode 100644 index 000000000..3dba735dc --- /dev/null +++ b/src/ztp/src/main/proto/acl.proto @@ -0,0 +1,69 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package acl; + +enum AclRuleTypeEnum { + ACLRULETYPE_UNDEFINED = 0; + ACLRULETYPE_IPV4 = 1; + ACLRULETYPE_IPV6 = 2; + ACLRULETYPE_L2 = 3; + ACLRULETYPE_MPLS = 4; + ACLRULETYPE_MIXED = 5; +} + +enum AclForwardActionEnum { + ACLFORWARDINGACTION_UNDEFINED = 0; + ACLFORWARDINGACTION_DROP = 1; + ACLFORWARDINGACTION_ACCEPT = 2; + ACLFORWARDINGACTION_REJECT = 3; +} + +enum AclLogActionEnum { + ACLLOGACTION_UNDEFINED = 0; + ACLLOGACTION_NOLOG = 1; + ACLLOGACTION_SYSLOG = 2; +} + +message AclMatch { + uint32 dscp = 1; + uint32 protocol = 2; + string src_address = 3; + string dst_address = 4; + uint32 src_port = 5; + uint32 dst_port = 6; + uint32 start_mpls_label = 7; + uint32 end_mpls_label = 8; +} + +message AclAction { + AclForwardActionEnum forward_action = 1; + AclLogActionEnum log_action = 2; +} + +message AclEntry { + uint32 sequence_id = 1; + string description = 2; + AclMatch match = 3; + AclAction action = 4; +} + +message AclRuleSet { + string name = 1; + AclRuleTypeEnum type = 2; + string description = 3; + string user_id = 4; + repeated AclEntry entries = 5; +} diff --git a/src/ztp/src/main/proto/context.proto b/src/ztp/src/main/proto/context.proto deleted file mode 120000 index 7f33c4bc7..000000000 --- a/src/ztp/src/main/proto/context.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/context.proto \ No newline at end of file diff --git a/src/ztp/src/main/proto/context.proto b/src/ztp/src/main/proto/context.proto new file mode 100644 index 000000000..fce1e71ad --- /dev/null +++ b/src/ztp/src/main/proto/context.proto @@ -0,0 +1,615 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package context; + +import "acl.proto"; +import "kpi_sample_types.proto"; + +service ContextService { + rpc ListContextIds (Empty ) returns ( ContextIdList ) {} + rpc ListContexts (Empty ) returns ( ContextList ) {} + rpc GetContext (ContextId ) returns ( Context ) {} + rpc SetContext (Context ) returns ( ContextId ) {} + rpc RemoveContext (ContextId ) returns ( Empty ) {} + rpc GetContextEvents (Empty ) returns (stream ContextEvent ) {} + + rpc ListTopologyIds (ContextId ) returns ( TopologyIdList ) {} + rpc ListTopologies (ContextId ) returns ( TopologyList ) {} + rpc GetTopology (TopologyId ) returns ( Topology ) {} + rpc GetTopologyDetails (TopologyId ) returns ( TopologyDetails ) {} + rpc SetTopology (Topology ) returns ( TopologyId ) {} + rpc RemoveTopology (TopologyId ) returns ( Empty ) {} + rpc GetTopologyEvents (Empty ) returns (stream TopologyEvent ) {} + + rpc ListDeviceIds (Empty ) returns ( DeviceIdList ) {} + rpc ListDevices (Empty ) returns ( DeviceList ) {} + rpc GetDevice (DeviceId ) returns ( Device ) {} + rpc SetDevice (Device ) returns ( DeviceId ) {} + rpc RemoveDevice (DeviceId ) returns ( Empty ) {} + rpc GetDeviceEvents (Empty ) returns (stream DeviceEvent ) {} + rpc SelectDevice (DeviceFilter ) returns ( DeviceList ) {} + rpc ListEndPointNames (EndPointIdList) returns ( EndPointNameList) {} + + rpc ListLinkIds (Empty ) returns ( LinkIdList ) {} + rpc ListLinks (Empty ) returns ( LinkList ) {} + rpc GetLink (LinkId ) returns ( Link ) {} + rpc SetLink (Link ) returns ( LinkId ) {} + rpc RemoveLink (LinkId ) returns ( Empty ) {} + rpc GetLinkEvents (Empty ) returns (stream LinkEvent ) {} + + rpc ListServiceIds (ContextId ) returns ( ServiceIdList ) {} + rpc ListServices (ContextId ) returns ( ServiceList ) {} + rpc GetService (ServiceId ) returns ( Service ) {} + rpc SetService (Service ) returns ( ServiceId ) {} + rpc UnsetService (Service ) returns ( ServiceId ) {} + rpc RemoveService (ServiceId ) returns ( Empty ) {} + rpc GetServiceEvents (Empty ) returns (stream ServiceEvent ) {} + rpc SelectService (ServiceFilter ) returns ( ServiceList ) {} + + rpc ListSliceIds (ContextId ) returns ( SliceIdList ) {} + rpc ListSlices (ContextId ) returns ( SliceList ) {} + rpc GetSlice (SliceId ) returns ( Slice ) {} + rpc SetSlice (Slice ) returns ( SliceId ) {} + rpc UnsetSlice (Slice ) returns ( SliceId ) {} + rpc RemoveSlice (SliceId ) returns ( Empty ) {} + rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {} + rpc SelectSlice (SliceFilter ) returns ( SliceList ) {} + + rpc ListConnectionIds (ServiceId ) returns ( ConnectionIdList) {} + rpc ListConnections (ServiceId ) returns ( ConnectionList ) {} + rpc GetConnection (ConnectionId ) returns ( Connection ) {} + rpc SetConnection (Connection ) returns ( ConnectionId ) {} + rpc RemoveConnection (ConnectionId ) returns ( Empty ) {} + rpc GetConnectionEvents(Empty ) returns (stream ConnectionEvent ) {} +} + +// ----- Generic ------------------------------------------------------------------------------------------------------- +message Empty {} + +message Uuid { + string uuid = 1; +} + +enum EventTypeEnum { + EVENTTYPE_UNDEFINED = 0; + EVENTTYPE_CREATE = 1; + EVENTTYPE_UPDATE = 2; + EVENTTYPE_REMOVE = 3; +} + +message Timestamp { + double timestamp = 1; +} + +message Event { + Timestamp timestamp = 1; + EventTypeEnum event_type = 2; +} + +// ----- Context ------------------------------------------------------------------------------------------------------- +message ContextId { + Uuid context_uuid = 1; +} + +message Context { + ContextId context_id = 1; + string name = 2; + repeated TopologyId topology_ids = 3; + repeated ServiceId service_ids = 4; + repeated SliceId slice_ids = 5; + TeraFlowController controller = 6; +} + +message ContextIdList { + repeated ContextId context_ids = 1; +} + +message ContextList { + repeated Context contexts = 1; +} + +message ContextEvent { + Event event = 1; + ContextId context_id = 2; +} + + +// ----- Topology ------------------------------------------------------------------------------------------------------ +message TopologyId { + ContextId context_id = 1; + Uuid topology_uuid = 2; +} + +message Topology { + TopologyId topology_id = 1; + string name = 2; + repeated DeviceId device_ids = 3; + repeated LinkId link_ids = 4; +} + +message TopologyDetails { + TopologyId topology_id = 1; + string name = 2; + repeated Device devices = 3; + repeated Link links = 4; +} + +message TopologyIdList { + repeated TopologyId topology_ids = 1; +} + +message TopologyList { + repeated Topology topologies = 1; +} + +message TopologyEvent { + Event event = 1; + TopologyId topology_id = 2; +} + + +// ----- Device -------------------------------------------------------------------------------------------------------- +message DeviceId { + Uuid device_uuid = 1; +} + +message Device { + DeviceId device_id = 1; + string name = 2; + string device_type = 3; + DeviceConfig device_config = 4; + DeviceOperationalStatusEnum device_operational_status = 5; + repeated DeviceDriverEnum device_drivers = 6; + repeated EndPoint device_endpoints = 7; + repeated Component components = 8; // Used for inventory + DeviceId controller_id = 9; // Identifier of node controlling the actual device +} + +message Component { //Defined previously to this section - Tested OK + Uuid component_uuid = 1; + string name = 2; + string type = 3; + + map attributes = 4; // dict[attr.name => json.dumps(attr.value)] + string parent = 5; +} + +message DeviceConfig { + repeated ConfigRule config_rules = 1; +} + +enum DeviceDriverEnum { + DEVICEDRIVER_UNDEFINED = 0; // also used for emulated + DEVICEDRIVER_OPENCONFIG = 1; + DEVICEDRIVER_TRANSPORT_API = 2; + DEVICEDRIVER_P4 = 3; + DEVICEDRIVER_IETF_NETWORK_TOPOLOGY = 4; + DEVICEDRIVER_ONF_TR_532 = 5; + DEVICEDRIVER_XR = 6; + DEVICEDRIVER_IETF_L2VPN = 7; + DEVICEDRIVER_GNMI_OPENCONFIG = 8; + DEVICEDRIVER_FLEXSCALE = 9; + DEVICEDRIVER_IETF_ACTN = 10; + DEVICEDRIVER_SMARTNIC = 11; +} + +enum DeviceOperationalStatusEnum { + DEVICEOPERATIONALSTATUS_UNDEFINED = 0; + DEVICEOPERATIONALSTATUS_DISABLED = 1; + DEVICEOPERATIONALSTATUS_ENABLED = 2; +} + +message DeviceIdList { + repeated DeviceId device_ids = 1; +} + +message DeviceList { + repeated Device devices = 1; +} + +message DeviceFilter { + DeviceIdList device_ids = 1; + bool include_endpoints = 2; + bool include_config_rules = 3; + bool include_components = 4; +} + +message DeviceEvent { + Event event = 1; + DeviceId device_id = 2; + DeviceConfig device_config = 3; +} + + +// ----- Link ---------------------------------------------------------------------------------------------------------- +message LinkId { + Uuid link_uuid = 1; +} + +message LinkAttributes { + float total_capacity_gbps = 1; + float used_capacity_gbps = 2; +} + +message Link { + LinkId link_id = 1; + string name = 2; + repeated EndPointId link_endpoint_ids = 3; + LinkAttributes attributes = 4; +} + +message LinkIdList { + repeated LinkId link_ids = 1; +} + +message LinkList { + repeated Link links = 1; +} + +message LinkEvent { + Event event = 1; + LinkId link_id = 2; +} + + +// ----- Service ------------------------------------------------------------------------------------------------------- +message ServiceId { + ContextId context_id = 1; + Uuid service_uuid = 2; +} + +message Service { + ServiceId service_id = 1; + string name = 2; + ServiceTypeEnum service_type = 3; + repeated EndPointId service_endpoint_ids = 4; + repeated Constraint service_constraints = 5; + ServiceStatus service_status = 6; + ServiceConfig service_config = 7; + Timestamp timestamp = 8; +} + +enum ServiceTypeEnum { + SERVICETYPE_UNKNOWN = 0; + SERVICETYPE_L3NM = 1; + SERVICETYPE_L2NM = 2; + SERVICETYPE_TAPI_CONNECTIVITY_SERVICE = 3; + SERVICETYPE_TE = 4; + SERVICETYPE_E2E = 5; +} + +enum ServiceStatusEnum { + SERVICESTATUS_UNDEFINED = 0; + SERVICESTATUS_PLANNED = 1; + SERVICESTATUS_ACTIVE = 2; + SERVICESTATUS_UPDATING = 3; + SERVICESTATUS_PENDING_REMOVAL = 4; + SERVICESTATUS_SLA_VIOLATED = 5; +} + +message ServiceStatus { + ServiceStatusEnum service_status = 1; +} + +message ServiceConfig { + repeated ConfigRule config_rules = 1; +} + +message ServiceIdList { + repeated ServiceId service_ids = 1; +} + +message ServiceList { + repeated Service services = 1; +} + +message ServiceFilter { + ServiceIdList service_ids = 1; + bool include_endpoint_ids = 2; + bool include_constraints = 3; + bool include_config_rules = 4; +} + +message ServiceEvent { + Event event = 1; + ServiceId service_id = 2; +} + +// ----- Slice --------------------------------------------------------------------------------------------------------- +message SliceId { + ContextId context_id = 1; + Uuid slice_uuid = 2; +} + +message Slice { + SliceId slice_id = 1; + string name = 2; + repeated EndPointId slice_endpoint_ids = 3; + repeated Constraint slice_constraints = 4; + repeated ServiceId slice_service_ids = 5; + repeated SliceId slice_subslice_ids = 6; + SliceStatus slice_status = 7; + SliceConfig slice_config = 8; + SliceOwner slice_owner = 9; + Timestamp timestamp = 10; +} + +message SliceOwner { + Uuid owner_uuid = 1; + string owner_string = 2; +} + +enum SliceStatusEnum { + SLICESTATUS_UNDEFINED = 0; + SLICESTATUS_PLANNED = 1; + SLICESTATUS_INIT = 2; + SLICESTATUS_ACTIVE = 3; + SLICESTATUS_DEINIT = 4; + SLICESTATUS_SLA_VIOLATED = 5; +} + +message SliceStatus { + SliceStatusEnum slice_status = 1; +} + +message SliceConfig { + repeated ConfigRule config_rules = 1; +} + +message SliceIdList { + repeated SliceId slice_ids = 1; +} + +message SliceList { + repeated Slice slices = 1; +} + +message SliceFilter { + SliceIdList slice_ids = 1; + bool include_endpoint_ids = 2; + bool include_constraints = 3; + bool include_service_ids = 4; + bool include_subslice_ids = 5; + bool include_config_rules = 6; +} + +message SliceEvent { + Event event = 1; + SliceId slice_id = 2; +} + +// ----- Connection ---------------------------------------------------------------------------------------------------- +message ConnectionId { + Uuid connection_uuid = 1; +} + +message ConnectionSettings_L0 { + string lsp_symbolic_name = 1; +} + +message ConnectionSettings_L2 { + string src_mac_address = 1; + string dst_mac_address = 2; + uint32 ether_type = 3; + uint32 vlan_id = 4; + uint32 mpls_label = 5; + uint32 mpls_traffic_class = 6; +} + +message ConnectionSettings_L3 { + string src_ip_address = 1; + string dst_ip_address = 2; + uint32 dscp = 3; + uint32 protocol = 4; + uint32 ttl = 5; +} + +message ConnectionSettings_L4 { + uint32 src_port = 1; + uint32 dst_port = 2; + uint32 tcp_flags = 3; + uint32 ttl = 4; +} + +message ConnectionSettings { + ConnectionSettings_L0 l0 = 1; + ConnectionSettings_L2 l2 = 2; + ConnectionSettings_L3 l3 = 3; + ConnectionSettings_L4 l4 = 4; +} + +message Connection { + ConnectionId connection_id = 1; + ServiceId service_id = 2; + repeated EndPointId path_hops_endpoint_ids = 3; + repeated ServiceId sub_service_ids = 4; + ConnectionSettings settings = 5; +} + +message ConnectionIdList { + repeated ConnectionId connection_ids = 1; +} + +message ConnectionList { + repeated Connection connections = 1; +} + +message ConnectionEvent { + Event event = 1; + ConnectionId connection_id = 2; +} + + +// ----- Endpoint ------------------------------------------------------------------------------------------------------ +message EndPointId { + TopologyId topology_id = 1; + DeviceId device_id = 2; + Uuid endpoint_uuid = 3; +} + +message EndPoint { + EndPointId endpoint_id = 1; + string name = 2; + string endpoint_type = 3; + repeated kpi_sample_types.KpiSampleType kpi_sample_types = 4; + Location endpoint_location = 5; +} + +message EndPointName { + EndPointId endpoint_id = 1; + string device_name = 2; + string endpoint_name = 3; + string endpoint_type = 4; +} + +message EndPointIdList { + repeated EndPointId endpoint_ids = 1; +} + +message EndPointNameList { + repeated EndPointName endpoint_names = 1; +} + + +// ----- Configuration ------------------------------------------------------------------------------------------------- +enum ConfigActionEnum { + CONFIGACTION_UNDEFINED = 0; + CONFIGACTION_SET = 1; + CONFIGACTION_DELETE = 2; +} + +message ConfigRule_Custom { + string resource_key = 1; + string resource_value = 2; +} + +message ConfigRule_ACL { + EndPointId endpoint_id = 1; + acl.AclRuleSet rule_set = 2; +} + +message ConfigRule { + ConfigActionEnum action = 1; + oneof config_rule { + ConfigRule_Custom custom = 2; + ConfigRule_ACL acl = 3; + } +} + + +// ----- Constraint ---------------------------------------------------------------------------------------------------- +enum ConstraintActionEnum { + CONSTRAINTACTION_UNDEFINED = 0; + CONSTRAINTACTION_SET = 1; + CONSTRAINTACTION_DELETE = 2; +} + +message Constraint_Custom { + string constraint_type = 1; + string constraint_value = 2; +} + +message Constraint_Schedule { + float start_timestamp = 1; + float duration_days = 2; +} + +message GPS_Position { + float latitude = 1; + float longitude = 2; +} + +message Location { + oneof location { + string region = 1; + GPS_Position gps_position = 2; + } +} + +message Constraint_EndPointLocation { + EndPointId endpoint_id = 1; + Location location = 2; +} + +message Constraint_EndPointPriority { + EndPointId endpoint_id = 1; + uint32 priority = 2; +} + +message Constraint_SLA_Latency { + float e2e_latency_ms = 1; +} + +message Constraint_SLA_Capacity { + float capacity_gbps = 1; +} + +message Constraint_SLA_Availability { + uint32 num_disjoint_paths = 1; + bool all_active = 2; + float availability = 3; // 0.0 .. 100.0 percentage of availability +} + +enum IsolationLevelEnum { + NO_ISOLATION = 0; + PHYSICAL_ISOLATION = 1; + LOGICAL_ISOLATION = 2; + PROCESS_ISOLATION = 3; + PHYSICAL_MEMORY_ISOLATION = 4; + PHYSICAL_NETWORK_ISOLATION = 5; + VIRTUAL_RESOURCE_ISOLATION = 6; + NETWORK_FUNCTIONS_ISOLATION = 7; + SERVICE_ISOLATION = 8; +} + +message Constraint_SLA_Isolation_level { + repeated IsolationLevelEnum isolation_level = 1; +} + +message Constraint_Exclusions { + bool is_permanent = 1; + repeated DeviceId device_ids = 2; + repeated EndPointId endpoint_ids = 3; + repeated LinkId link_ids = 4; +} + +message Constraint { + ConstraintActionEnum action = 1; + oneof constraint { + Constraint_Custom custom = 2; + Constraint_Schedule schedule = 3; + Constraint_EndPointLocation endpoint_location = 4; + Constraint_EndPointPriority endpoint_priority = 5; + Constraint_SLA_Capacity sla_capacity = 6; + Constraint_SLA_Latency sla_latency = 7; + Constraint_SLA_Availability sla_availability = 8; + Constraint_SLA_Isolation_level sla_isolation = 9; + Constraint_Exclusions exclusions = 10; + } +} + + +// ----- Miscellaneous ------------------------------------------------------------------------------------------------- +message TeraFlowController { + ContextId context_id = 1; + string ip_address = 2; + uint32 port = 3; +} + +message AuthenticationResult { + ContextId context_id = 1; + bool authenticated = 2; +} diff --git a/src/ztp/src/main/proto/device.proto b/src/ztp/src/main/proto/device.proto deleted file mode 120000 index ad6e7c47e..000000000 --- a/src/ztp/src/main/proto/device.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/device.proto \ No newline at end of file diff --git a/src/ztp/src/main/proto/device.proto b/src/ztp/src/main/proto/device.proto new file mode 100644 index 000000000..30e60079d --- /dev/null +++ b/src/ztp/src/main/proto/device.proto @@ -0,0 +1,34 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package device; + +import "context.proto"; +import "monitoring.proto"; + +service DeviceService { + rpc AddDevice (context.Device ) returns (context.DeviceId ) {} + rpc ConfigureDevice (context.Device ) returns (context.DeviceId ) {} + rpc DeleteDevice (context.DeviceId ) returns (context.Empty ) {} + rpc GetInitialConfig(context.DeviceId ) returns (context.DeviceConfig) {} + rpc MonitorDeviceKpi(MonitoringSettings) returns (context.Empty ) {} +} + +message MonitoringSettings { + monitoring.KpiId kpi_id = 1; + monitoring.KpiDescriptor kpi_descriptor = 2; + float sampling_duration_s = 3; + float sampling_interval_s = 4; +} diff --git a/src/ztp/src/main/proto/kpi_sample_types.proto b/src/ztp/src/main/proto/kpi_sample_types.proto deleted file mode 120000 index 98e748bbf..000000000 --- a/src/ztp/src/main/proto/kpi_sample_types.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/kpi_sample_types.proto \ No newline at end of file diff --git a/src/ztp/src/main/proto/kpi_sample_types.proto b/src/ztp/src/main/proto/kpi_sample_types.proto new file mode 100644 index 000000000..5b234a4e3 --- /dev/null +++ b/src/ztp/src/main/proto/kpi_sample_types.proto @@ -0,0 +1,42 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package kpi_sample_types; + +enum KpiSampleType { + KPISAMPLETYPE_UNKNOWN = 0; + + KPISAMPLETYPE_PACKETS_TRANSMITTED = 101; + KPISAMPLETYPE_PACKETS_RECEIVED = 102; + KPISAMPLETYPE_PACKETS_DROPPED = 103; + KPISAMPLETYPE_BYTES_TRANSMITTED = 201; + KPISAMPLETYPE_BYTES_RECEIVED = 202; + KPISAMPLETYPE_BYTES_DROPPED = 203; + + KPISAMPLETYPE_LINK_TOTAL_CAPACITY_GBPS = 301; + KPISAMPLETYPE_LINK_USED_CAPACITY_GBPS = 302; + + KPISAMPLETYPE_ML_CONFIDENCE = 401; //. can be used by both optical and L3 without any issue + + KPISAMPLETYPE_OPTICAL_SECURITY_STATUS = 501; //. can be used by both optical and L3 without any issue + + KPISAMPLETYPE_L3_UNIQUE_ATTACK_CONNS = 601; + KPISAMPLETYPE_L3_TOTAL_DROPPED_PACKTS = 602; + KPISAMPLETYPE_L3_UNIQUE_ATTACKERS = 603; + KPISAMPLETYPE_L3_UNIQUE_COMPROMISED_CLIENTS = 604; + KPISAMPLETYPE_L3_SECURITY_STATUS_CRYPTO = 605; + + KPISAMPLETYPE_SERVICE_LATENCY_MS = 701; +} diff --git a/src/ztp/src/main/proto/monitoring.proto b/src/ztp/src/main/proto/monitoring.proto deleted file mode 120000 index aceaa7328..000000000 --- a/src/ztp/src/main/proto/monitoring.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/monitoring.proto \ No newline at end of file diff --git a/src/ztp/src/main/proto/monitoring.proto b/src/ztp/src/main/proto/monitoring.proto new file mode 100644 index 000000000..45ba48b02 --- /dev/null +++ b/src/ztp/src/main/proto/monitoring.proto @@ -0,0 +1,174 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package monitoring; + +import "context.proto"; +import "kpi_sample_types.proto"; + +service MonitoringService { + rpc SetKpi (KpiDescriptor ) returns (KpiId ) {} // Stable not final + rpc DeleteKpi (KpiId ) returns (context.Empty ) {} // Stable and final + rpc GetKpiDescriptor (KpiId ) returns (KpiDescriptor ) {} // Stable and final + rpc GetKpiDescriptorList (context.Empty ) returns (KpiDescriptorList ) {} // Stable and final + rpc IncludeKpi (Kpi ) returns (context.Empty ) {} // Stable and final + rpc MonitorKpi (MonitorKpiRequest ) returns (context.Empty ) {} // Stable and final + rpc QueryKpiData (KpiQuery ) returns (RawKpiTable ) {} // Not implemented + rpc SetKpiSubscription (SubsDescriptor ) returns (stream SubsResponse ) {} // Stable not final + rpc GetSubsDescriptor (SubscriptionID ) returns (SubsDescriptor ) {} // Stable and final + rpc GetSubscriptions (context.Empty ) returns (SubsList ) {} // Stable and final + rpc DeleteSubscription (SubscriptionID ) returns (context.Empty ) {} // Stable and final + rpc SetKpiAlarm (AlarmDescriptor ) returns (AlarmID ) {} // Stable not final + rpc GetAlarms (context.Empty ) returns (AlarmList ) {} // Stable and final + rpc GetAlarmDescriptor (AlarmID ) returns (AlarmDescriptor ) {} // Stable and final + rpc GetAlarmResponseStream(AlarmSubscription ) returns (stream AlarmResponse) {} // Not Stable not final + rpc DeleteAlarm (AlarmID ) returns (context.Empty ) {} // Stable and final + rpc GetStreamKpi (KpiId ) returns (stream Kpi ) {} // Stable not final + rpc GetInstantKpi (KpiId ) returns (Kpi ) {} // Stable not final +} + +message KpiDescriptor { + KpiId kpi_id = 1; + string kpi_description = 2; + repeated KpiId kpi_id_list = 3; + kpi_sample_types.KpiSampleType kpi_sample_type = 4; + context.DeviceId device_id = 5; + context.EndPointId endpoint_id = 6; + context.ServiceId service_id = 7; + context.SliceId slice_id = 8; + context.ConnectionId connection_id = 9; + context.LinkId link_id = 10; +} + +message MonitorKpiRequest { + KpiId kpi_id = 1; + float monitoring_window_s = 2; + float sampling_rate_s = 3; + // Pending add field to reflect Available Device Protocols +} + +message KpiQuery { + repeated KpiId kpi_ids = 1; + float monitoring_window_s = 2; + uint32 last_n_samples = 3; // used when you want something like "get the last N many samples + context.Timestamp start_timestamp = 4; // used when you want something like "get the samples since X date/time" + context.Timestamp end_timestamp = 5; // used when you want something like "get the samples until X date/time" +} + + +message RawKpi { // cell + context.Timestamp timestamp = 1; + KpiValue kpi_value = 2; +} + +message RawKpiList { // column + KpiId kpi_id = 1; + repeated RawKpi raw_kpis = 2; +} + +message RawKpiTable { // table + repeated RawKpiList raw_kpi_lists = 1; +} + +message KpiId { + context.Uuid kpi_id = 1; +} + +message Kpi { + KpiId kpi_id = 1; + context.Timestamp timestamp = 2; + KpiValue kpi_value = 3; +} + +message KpiValueRange { + KpiValue kpiMinValue = 1; + KpiValue kpiMaxValue = 2; + bool inRange = 3; // by default True + bool includeMinValue = 4; // False is outside the interval + bool includeMaxValue = 5; // False is outside the interval +} + +message KpiValue { + oneof value { + int32 int32Val = 1; + uint32 uint32Val = 2; + int64 int64Val = 3; + uint64 uint64Val = 4; + float floatVal = 5; + string stringVal = 6; + bool boolVal = 7; + } +} + + +message KpiList { + repeated Kpi kpi = 1; +} + +message KpiDescriptorList { + repeated KpiDescriptor kpi_descriptor_list = 1; +} + +message SubsDescriptor{ + SubscriptionID subs_id = 1; + KpiId kpi_id = 2; + float sampling_duration_s = 3; + float sampling_interval_s = 4; + context.Timestamp start_timestamp = 5; // used when you want something like "get the samples since X date/time" + context.Timestamp end_timestamp = 6; // used when you want something like "get the samples until X date/time" + // Pending add field to reflect Available Device Protocols +} + +message SubscriptionID { + context.Uuid subs_id = 1; +} + +message SubsResponse { + SubscriptionID subs_id = 1; + KpiList kpi_list = 2; +} + +message SubsList { + repeated SubsDescriptor subs_descriptor = 1; +} + +message AlarmDescriptor { + AlarmID alarm_id = 1; + string alarm_description = 2; + string name = 3; + KpiId kpi_id = 4; + KpiValueRange kpi_value_range = 5; + context.Timestamp timestamp = 6; +} + +message AlarmID{ + context.Uuid alarm_id = 1; +} + +message AlarmSubscription{ + AlarmID alarm_id = 1; + float subscription_timeout_s = 2; + float subscription_frequency_ms = 3; +} + +message AlarmResponse { + AlarmID alarm_id = 1; + string text = 2; + KpiList kpi_list = 3; +} + +message AlarmList { + repeated AlarmDescriptor alarm_descriptor = 1; +} diff --git a/src/ztp/src/main/proto/ztp.proto b/src/ztp/src/main/proto/ztp.proto deleted file mode 120000 index 9183ce531..000000000 --- a/src/ztp/src/main/proto/ztp.proto +++ /dev/null @@ -1 +0,0 @@ -../../../../../proto/ztp.proto \ No newline at end of file diff --git a/src/ztp/src/main/proto/ztp.proto b/src/ztp/src/main/proto/ztp.proto new file mode 100644 index 000000000..5c895900d --- /dev/null +++ b/src/ztp/src/main/proto/ztp.proto @@ -0,0 +1,69 @@ +// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package ztp; + +import "context.proto"; + +service ZtpService { + rpc ZtpGetDeviceRole(DeviceRoleId) returns (DeviceRole) {} + rpc ZtpGetDeviceRolesByDeviceId(context.DeviceId) returns (DeviceRoleList) {} + rpc ZtpAdd(DeviceRole) returns (DeviceRoleState) {} + rpc ZtpUpdate(DeviceRoleConfig) returns (DeviceRoleState) {} + rpc ZtpDelete(DeviceRole) returns (DeviceRoleState) {} + rpc ZtpDeleteAll(context.Empty) returns (DeviceDeletionResult) {} +} + +enum DeviceRoleType { + NONE = 0; + DEV_OPS = 1; + DEV_CONF = 2; + PIPELINE_CONF = 3; +} + +message DeviceRoleId { + context.Uuid devRoleId = 1; + context.DeviceId devId = 2; +} + +message DeviceRole { + DeviceRoleId devRoleId = 1; + DeviceRoleType devRoleType = 2; +} + +message DeviceRoleConfig { + DeviceRole devRole = 1; + context.DeviceConfig devConfig = 2; +} + +message DeviceRoleList { + repeated DeviceRole devRole = 1; +} + +message DeviceRoleState { + DeviceRoleId devRoleId = 1; + ZtpDeviceState devRoleState = 2; +} + +message DeviceDeletionResult { + repeated string deleted = 1; +} + +enum ZtpDeviceState { + ZTP_DEV_STATE_UNDEFINED = 0; + ZTP_DEV_STATE_CREATED = 1; + ZTP_DEV_STATE_UPDATED = 2; + ZTP_DEV_STATE_DELETED = 3; +} -- GitLab From 87b5aa6f30df31a03fbb7991a2bb51b027179e80 Mon Sep 17 00:00:00 2001 From: carcel Date: Fri, 22 Mar 2024 08:40:38 +0000 Subject: [PATCH 032/941] Update - Morpheus Client Extension --- src/common/tools/object_factory/Device.py | 5 +++-- .../drivers/smartnic/SmartnicDriver.py | 12 +++++++---- src/device/service/drivers/smartnic/Tools.py | 12 +++++++---- .../service/drivers/smartnic/__init__.py | 2 -- .../ietf-yang-types.yang | 0 .../openconfig-extensions.yang | 0 .../openconfig-inet-types.yang | 0 .../openconfig-probes-types.yang | 0 .../openconfig-probes.yang | 0 .../openconfig-types.yang | 0 .../probes-agent.yang | 0 .../references_probes_libraries.txt | 0 .../service/drivers/smartnic_probes/.gitkeep | 0 .../smartnics_probes_agent/.gitkeep | 0 .../nbi_plugins/agent_probes/Resources.py | 21 ++++++++++++------- .../nbi_plugins/agent_probes/Tools.py | 12 +++++++---- 16 files changed, 40 insertions(+), 24 deletions(-) rename src/device/service/drivers/{smartnic_probes => smartnic}/ietf-yang-types.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/openconfig-extensions.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/openconfig-inet-types.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/openconfig-probes-types.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/openconfig-probes.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/openconfig-types.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/probes-agent.yang (100%) rename src/device/service/drivers/{smartnic_probes => smartnic}/references_probes_libraries.txt (100%) delete mode 100644 src/device/service/drivers/smartnic_probes/.gitkeep delete mode 100644 src/device/service/drivers/smartnic_probes/smartnics_probes_agent/.gitkeep diff --git a/src/common/tools/object_factory/Device.py b/src/common/tools/object_factory/Device.py index b3182e302..5a7ff398a 100644 --- a/src/common/tools/object_factory/Device.py +++ b/src/common/tools/object_factory/Device.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy +import copy, logging from typing import Dict, List, Optional, Tuple from common.DeviceTypes import DeviceTypeEnum from common.proto.context_pb2 import DeviceDriverEnum, DeviceOperationalStatusEnum from common.tools.object_factory.ConfigRule import json_config_rule_set +LOGGER = logging.getLogger(__name__) DEVICE_DISABLED = DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_DISABLED DEVICE_EMUDC_TYPE = DeviceTypeEnum.EMULATED_DATACENTER.value @@ -62,13 +63,13 @@ def json_device( ): result = { 'device_id' : json_device_id(device_uuid), + 'name' : name, 'device_type' : device_type, 'device_config' : {'config_rules': copy.deepcopy(config_rules)}, 'device_operational_status': status, 'device_drivers' : copy.deepcopy(drivers), 'device_endpoints' : copy.deepcopy(endpoints), } - if name is not None: result['name'] = name return result def json_device_emulated_packet_router_disabled( diff --git a/src/device/service/drivers/smartnic/SmartnicDriver.py b/src/device/service/drivers/smartnic/SmartnicDriver.py index 4bad52db4..f827bbbff 100644 --- a/src/device/service/drivers/smartnic/SmartnicDriver.py +++ b/src/device/service/drivers/smartnic/SmartnicDriver.py @@ -20,8 +20,11 @@ from common.type_checkers.Checkers import chk_string, chk_type from device.service.driver_api._Driver import _Driver from . import ALL_RESOURCE_KEYS from .Tools import create_connectivity_service, find_key, config_getter, delete_connectivity_service +from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string + LOGGER = logging.getLogger(__name__) +from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_SERVICES, RESOURCE_INTERFACES DRIVER_NAME = 'smartnic' METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': DRIVER_NAME}) @@ -75,8 +78,9 @@ class SmartnicDriver(_Driver): for i, resource_key in enumerate(resource_keys): str_resource_name = 'resource_key[#{:d}]'.format(i) chk_string(str_resource_name, resource_key, allow_empty=False) - results.extend(config_getter( - self.__tapi_root, resource_key, timeout=self.__timeout)) + if resource_key == RESOURCE_ENDPOINTS: + results.extend(config_getter( + self.__tapi_root, resource_key, timeout=self.__timeout)) return results @metered_subclass_method(METRICS_POOL) @@ -87,9 +91,9 @@ class SmartnicDriver(_Driver): with self.__lock: for resource in resources: LOGGER.info('resource = {:s}'.format(str(resource))) - config_rules = find_key(resource, 'config_rules') + #config_rules = find_key(resource, 'config_rules') data = create_connectivity_service( - self.__tapi_root, config_rules, timeout=self.__timeout) + self.__tapi_root, resource[1], timeout=self.__timeout) results.extend(data) return results diff --git a/src/device/service/drivers/smartnic/Tools.py b/src/device/service/drivers/smartnic/Tools.py index 54961bbdd..bd155441c 100644 --- a/src/device/service/drivers/smartnic/Tools.py +++ b/src/device/service/drivers/smartnic/Tools.py @@ -50,6 +50,11 @@ def config_getter( result = [] try: response = requests.get(url, timeout=timeout, verify=False) + data = response.json() + for item in data: + tupla = ('/endpoints/endpoint', item) + result.append(tupla) + return result except requests.exceptions.Timeout: LOGGER.exception('Timeout connecting {:s}'.format(url)) return result @@ -57,7 +62,6 @@ def config_getter( LOGGER.exception('Exception retrieving {:s}'.format(resource_key)) result.append((resource_key, e)) return result - return response # try: # context = json.loads(response.content) @@ -72,13 +76,13 @@ def create_connectivity_service( root_url, config_rules, timeout : Optional[int] = None, auth : Optional[HTTPBasicAuth] = None ): - url = '{:s}/configure'.format(root_url) + url = '{:s}/manage-probe/configure'.format(root_url) headers = {'content-type': 'application/json'} results = [] try: LOGGER.info('Configuring Smartnic rules') response = requests.post( - url=url, data=json.dumps(config_rules), timeout=timeout, headers=headers, verify=False) + url=url, data=config_rules, timeout=timeout, headers=headers, verify=False) LOGGER.info('SmartNIC Probes response: {:s}'.format(str(response))) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception creating ConfigRule') @@ -92,7 +96,7 @@ def create_connectivity_service( def delete_connectivity_service(root_url, config_rules, timeout : Optional[int] = None, auth : Optional[HTTPBasicAuth] = None ): - url = '{:s}/configure'.format(root_url) + url = '{:s}/manage-probe/configure'.format(root_url) results = [] try: response = requests.delete(url=url, timeout=timeout, verify=False) diff --git a/src/device/service/drivers/smartnic/__init__.py b/src/device/service/drivers/smartnic/__init__.py index bc88d00fa..b3626c633 100644 --- a/src/device/service/drivers/smartnic/__init__.py +++ b/src/device/service/drivers/smartnic/__init__.py @@ -14,7 +14,5 @@ from device.service.driver_api._Driver import RESOURCE_ENDPOINTS, RESOURCE_SERVICES, RESOURCE_INTERFACES ALL_RESOURCE_KEYS = [ - RESOURCE_ENDPOINTS, - RESOURCE_SERVICES, RESOURCE_INTERFACES ] diff --git a/src/device/service/drivers/smartnic_probes/ietf-yang-types.yang b/src/device/service/drivers/smartnic/ietf-yang-types.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/ietf-yang-types.yang rename to src/device/service/drivers/smartnic/ietf-yang-types.yang diff --git a/src/device/service/drivers/smartnic_probes/openconfig-extensions.yang b/src/device/service/drivers/smartnic/openconfig-extensions.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/openconfig-extensions.yang rename to src/device/service/drivers/smartnic/openconfig-extensions.yang diff --git a/src/device/service/drivers/smartnic_probes/openconfig-inet-types.yang b/src/device/service/drivers/smartnic/openconfig-inet-types.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/openconfig-inet-types.yang rename to src/device/service/drivers/smartnic/openconfig-inet-types.yang diff --git a/src/device/service/drivers/smartnic_probes/openconfig-probes-types.yang b/src/device/service/drivers/smartnic/openconfig-probes-types.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/openconfig-probes-types.yang rename to src/device/service/drivers/smartnic/openconfig-probes-types.yang diff --git a/src/device/service/drivers/smartnic_probes/openconfig-probes.yang b/src/device/service/drivers/smartnic/openconfig-probes.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/openconfig-probes.yang rename to src/device/service/drivers/smartnic/openconfig-probes.yang diff --git a/src/device/service/drivers/smartnic_probes/openconfig-types.yang b/src/device/service/drivers/smartnic/openconfig-types.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/openconfig-types.yang rename to src/device/service/drivers/smartnic/openconfig-types.yang diff --git a/src/device/service/drivers/smartnic_probes/probes-agent.yang b/src/device/service/drivers/smartnic/probes-agent.yang similarity index 100% rename from src/device/service/drivers/smartnic_probes/probes-agent.yang rename to src/device/service/drivers/smartnic/probes-agent.yang diff --git a/src/device/service/drivers/smartnic_probes/references_probes_libraries.txt b/src/device/service/drivers/smartnic/references_probes_libraries.txt similarity index 100% rename from src/device/service/drivers/smartnic_probes/references_probes_libraries.txt rename to src/device/service/drivers/smartnic/references_probes_libraries.txt diff --git a/src/device/service/drivers/smartnic_probes/.gitkeep b/src/device/service/drivers/smartnic_probes/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/device/service/drivers/smartnic_probes/smartnics_probes_agent/.gitkeep b/src/device/service/drivers/smartnic_probes/smartnics_probes_agent/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py index 2b0a537cc..daafd79ab 100644 --- a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py @@ -181,10 +181,10 @@ class Device(_Resource): return format_grpc_to_json(self.device_client.AddDevice(grpc_device( device_uuid = device['device_id']['device_uuid']['uuid'], device_type = device['device_type'], - config_rules = device['device_config']['config_rules'], status = device['device_operational_status'], - drivers = device['device_drivers'], - endpoints = device['device_endpoints'] + endpoints = device['device_endpoints'], + config_rules = device['device_config']['config_rules'], + drivers = device['device_drivers'] ))) def put(self, device_uuid : str): # pylint: disable=unused-argument @@ -192,12 +192,17 @@ class Device(_Resource): return format_grpc_to_json(self.device_client.ConfigureDevice(grpc_device( device_uuid = device['device_id']['device_uuid']['uuid'], device_type = device['device_type'], - device_config = device['device_config']['config_rules'], - device_operational_status = device['device_operational_status'], - device_drivers = device['device_drivers'], - device_endpoints = device['device_endpoints'] + status = device['device_operational_status'], + endpoints = device['device_endpoints'], + config_rules = device['device_config']['config_rules'], + drivers = device['device_drivers'] + ))) + + def delete(self, device_uuid : str): + device = request.get_json()['devices'][0] + return format_grpc_to_json(self.device_client.DeleteDevice(grpc_device( + device_uuid = device['device_id']['device_uuid']['uuid'] ))) - class LinkIds(_Resource): def get(self): diff --git a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py index 17b6dcdfd..6cffbb5cc 100644 --- a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py +++ b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Tools.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from flask.json import jsonify from common.proto.context_pb2 import ( ConnectionId, ContextId, DeviceDriverEnum, Device, DeviceId, DeviceOperationalStatusEnum, LinkId, ServiceId, SliceId, TopologyId, Service, ServiceStatusEnum ) from common.proto.policy_pb2 import PolicyRuleId -from common.tools.grpc.Tools import grpc_message_to_json +from common.proto.context_pb2 import ConfigActionEnum +from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string from common.tools.object_factory.Connection import json_connection_id from common.tools.object_factory.Context import json_context_id from common.tools.object_factory.ConfigRule import json_config_rule @@ -30,6 +32,8 @@ from common.tools.object_factory.Service import json_service_id, json_service from common.tools.object_factory.Slice import json_slice_id from common.tools.object_factory.Topology import json_topology_id +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) def format_grpc_to_json(grpc_reply): return jsonify(grpc_message_to_json(grpc_reply)) @@ -44,8 +48,9 @@ def grpc_device_id(device_uuid): return DeviceId(**json_device_id(device_uuid)) def grpc_device( - device_uuid, device_type, config_rules=None, status=None, drivers=None, endpoints=None + device_uuid, device_type, status, endpoints=None, config_rules=None, drivers=None ): + json_config_rules = [ json_config_rule( config_rule['action'], @@ -68,8 +73,7 @@ def grpc_device( for endpoint in endpoints ] if endpoints else [] return Device(**json_device( - device_uuid, device_type, json_config_rules, json_status, - json_drivers, json_endpoints)) + device_uuid, device_type, json_status, None, json_endpoints, json_config_rules, json_drivers)) def grpc_link_id(link_uuid): return LinkId(**json_link_id(link_uuid)) -- GitLab From 9bdfc3c75bdb2dc7ad493272565436150cfb4f34 Mon Sep 17 00:00:00 2001 From: carcel Date: Fri, 22 Mar 2024 09:40:51 +0000 Subject: [PATCH 033/941] Update - Morpheus Client Extension --- src/device/service/drivers/smartnic/SmartnicDriver.py | 4 ++-- src/device/service/drivers/smartnic/Tools.py | 2 +- .../rest_server/nbi_plugins/agent_probes/Resources.py | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/device/service/drivers/smartnic/SmartnicDriver.py b/src/device/service/drivers/smartnic/SmartnicDriver.py index f827bbbff..35d05edde 100644 --- a/src/device/service/drivers/smartnic/SmartnicDriver.py +++ b/src/device/service/drivers/smartnic/SmartnicDriver.py @@ -104,9 +104,9 @@ class SmartnicDriver(_Driver): with self.__lock: for resource in resources: LOGGER.info('resource = {:s}'.format(str(resource))) - config_rules = find_key(resource, 'config_rules') + #config_rules = find_key(resource, 'config_rules') results.extend(delete_connectivity_service( - self.__tapi_root, config_rules, timeout=self.__timeout)) + self.__tapi_root, resource[1], timeout=self.__timeout)) return results @metered_subclass_method(METRICS_POOL) diff --git a/src/device/service/drivers/smartnic/Tools.py b/src/device/service/drivers/smartnic/Tools.py index bd155441c..13345b618 100644 --- a/src/device/service/drivers/smartnic/Tools.py +++ b/src/device/service/drivers/smartnic/Tools.py @@ -99,7 +99,7 @@ def delete_connectivity_service(root_url, config_rules, timeout : Optional[int] url = '{:s}/manage-probe/configure'.format(root_url) results = [] try: - response = requests.delete(url=url, timeout=timeout, verify=False) + response = requests.delete(url=url, data=config_rules, timeout=timeout, verify=False) except Exception as e: # pylint: disable=broad-except LOGGER.exception('Exception deleting ConfigRule') results.append(e) diff --git a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py index daafd79ab..50e8e77af 100644 --- a/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py +++ b/src/nbi/service/rest_server/nbi_plugins/agent_probes/Resources.py @@ -201,7 +201,12 @@ class Device(_Resource): def delete(self, device_uuid : str): device = request.get_json()['devices'][0] return format_grpc_to_json(self.device_client.DeleteDevice(grpc_device( - device_uuid = device['device_id']['device_uuid']['uuid'] + device_uuid = device['device_id']['device_uuid']['uuid'], + device_type = device['device_type'], + status = device['device_operational_status'], + endpoints = device['device_endpoints'], + config_rules = device['device_config']['config_rules'], + drivers = device['device_drivers'] ))) class LinkIds(_Resource): -- GitLab From b4b217029b981702d6cccddf9fe1a22f7e4909ed Mon Sep 17 00:00:00 2001 From: mansoca Date: Wed, 27 Mar 2024 11:45:39 +0000 Subject: [PATCH 034/941] Creation VTNManager service --- my_deploy.sh | 3 + src/vnt_manager/.gitlab-ci.yml | 38 ++++++++ src/vnt_manager/Config.py | 13 +++ src/vnt_manager/Dockerfile | 84 ++++++++++++++++ src/vnt_manager/__init__.py | 13 +++ src/vnt_manager/client/VNTManagerClient.py | 71 ++++++++++++++ src/vnt_manager/client/__init__.py | 13 +++ src/vnt_manager/requirements.in | 15 +++ src/vnt_manager/service/VNTManagerService.py | 35 +++++++ .../service/VNTManagerServiceServicerImpl.py | 95 +++++++++++++++++++ src/vnt_manager/service/__init__.py | 13 +++ src/vnt_manager/service/__main__.py | 80 ++++++++++++++++ 12 files changed, 473 insertions(+) create mode 100644 src/vnt_manager/.gitlab-ci.yml create mode 100644 src/vnt_manager/Config.py create mode 100644 src/vnt_manager/Dockerfile create mode 100644 src/vnt_manager/__init__.py create mode 100644 src/vnt_manager/client/VNTManagerClient.py create mode 100644 src/vnt_manager/client/__init__.py create mode 100644 src/vnt_manager/requirements.in create mode 100644 src/vnt_manager/service/VNTManagerService.py create mode 100644 src/vnt_manager/service/VNTManagerServiceServicerImpl.py create mode 100644 src/vnt_manager/service/__init__.py create mode 100644 src/vnt_manager/service/__main__.py diff --git a/my_deploy.sh b/my_deploy.sh index 7dd5e5c3e..212dd7bd6 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -52,6 +52,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene # Uncomment to activate E2E Orchestrator #export TFS_COMPONENTS="${TFS_COMPONENTS} e2e_orchestrator" +# Uncomment to activate VNT Manager +#export TFS_COMPONENTS="${TFS_COMPONENTS} vnt_manager" + # Set the tag you want to use for your images. export TFS_IMAGE_TAG="dev" diff --git a/src/vnt_manager/.gitlab-ci.yml b/src/vnt_manager/.gitlab-ci.yml new file mode 100644 index 000000000..d1b9da495 --- /dev/null +++ b/src/vnt_manager/.gitlab-ci.yml @@ -0,0 +1,38 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# build, tag and push the Docker image to the gitlab registry +build vntmanager: + variables: + IMAGE_NAME: 'vntmanager' # name of the microservice + IMAGE_TAG: 'latest' # tag of the container image (production, development, etc) + stage: build + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + script: + - docker buildx build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile . + - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" + after_script: + - docker images --filter="dangling=true" --quiet | xargs -r docker rmi + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)' + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"' + - changes: + - src/$IMAGE_NAME/**/*.{py,in,yml} + - src/$IMAGE_NAME/Dockerfile + - src/$IMAGE_NAME/tests/*.py + - src/$IMAGE_NAME/tests/Dockerfile + - manifests/${IMAGE_NAME}service.yaml + - .gitlab-ci.yml diff --git a/src/vnt_manager/Config.py b/src/vnt_manager/Config.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/Config.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/Dockerfile b/src/vnt_manager/Dockerfile new file mode 100644 index 000000000..8f40741ee --- /dev/null +++ b/src/vnt_manager/Dockerfile @@ -0,0 +1,84 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-slim + +# Install dependencies +RUN apt-get --yes --quiet --quiet update && \ + apt-get --yes --quiet --quiet install wget g++ && \ + rm -rf /var/lib/apt/lists/* + +# Set Python to show logs as they occur +ENV PYTHONUNBUFFERED=0 +ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + +# Download the gRPC health probe +RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +# Creating a user for security reasons +RUN groupadd -r teraflow && useradd -u 1001 --no-log-init -r -m -g teraflow teraflow +USER teraflow + +# set working directory +RUN mkdir -p /home/teraflow/controller/common/ +WORKDIR /home/teraflow/controller + +# Get Python packages per module +ENV VIRTUAL_ENV=/home/teraflow/venv +RUN python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +# Get generic Python packages +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --upgrade setuptools wheel +RUN python3 -m pip install --upgrade pip-tools + +# Get common Python packages +# Note: this step enables sharing the previous Docker build steps among all the Python components +COPY --chown=teraflow:teraflow common_requirements.in common_requirements.in +RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in +RUN python3 -m pip install -r common_requirements.txt + +# Add common files into working directory +WORKDIR /home/teraflow/controller/common +COPY --chown=teraflow:teraflow src/common/. ./ +RUN rm -rf proto + +# Create proto sub-folder, copy .proto files, and generate Python code +RUN mkdir -p /home/teraflow/controller/common/proto +WORKDIR /home/teraflow/controller/common/proto +RUN touch __init__.py +COPY --chown=teraflow:teraflow proto/*.proto ./ +RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto +RUN rm *.proto +RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; + +# Create module sub-folders +RUN mkdir -p /home/teraflow/controller/vnt_manager +WORKDIR /home/teraflow/controller + +# Get Python packages per module +COPY --chown=teraflow:teraflow ./src/vnt_manager/requirements.in vnt_manager/requirements.in +# consider common and specific requirements to avoid inconsistencies with dependencies +RUN pip-compile --quiet --output-file=vnt_manager/requirements.txt vnt_manager/requirements.in common_requirements.in +RUN python3 -m pip install -r vnt_manager/requirements.txt + +# Add component files into working directory +COPY --chown=teraflow:teraflow ./src/context/. context +COPY --chown=teraflow:teraflow ./src/vnt_manager/. vnt_manager + +# Start the service +ENTRYPOINT ["python", "-m", "vnt_manager.service"] diff --git a/src/vnt_manager/__init__.py b/src/vnt_manager/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/client/VNTManagerClient.py b/src/vnt_manager/client/VNTManagerClient.py new file mode 100644 index 000000000..95db3b6da --- /dev/null +++ b/src/vnt_manager/client/VNTManagerClient.py @@ -0,0 +1,71 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import grpc + +from common.Constants import ServiceNameEnum +from common.proto.context_pb2 import Empty +from common.proto.vntmanager_pb2_grpc import VNTManagerServiceStub +from common.Settings import get_service_host, get_service_port_grpc +from common.tools.client.RetryDecorator import delay_exponential, retry +from common.tools.grpc.Tools import grpc_message_to_json +# from common.proto.e2eorchestrator_pb2 import E2EOrchestratorRequest, E2EOrchestratorReply + +LOGGER = logging.getLogger(__name__) +MAX_RETRIES = 15 +DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0) +RETRY_DECORATOR = retry( + max_retries=MAX_RETRIES, + delay_function=DELAY_FUNCTION, + prepare_method_name="connect", +) + + +class VNTManagerClient: + def __init__(self, host=None, port=None): + if not host: + host = get_service_host(ServiceNameEnum.VNTMANAGER) + if not port: + port = get_service_port_grpc(ServiceNameEnum.VNTMANAGER) + self.endpoint = "{:s}:{:s}".format(str(host), str(port)) + LOGGER.debug("Creating channel to {:s}...".format(str(self.endpoint))) + self.channel = None + self.stub = None + self.connect() + LOGGER.debug("Channel created") + + def connect(self): + self.channel = grpc.insecure_channel(self.endpoint) + self.stub = VNTManagerServiceStub(self.channel) + + def close(self): + if self.channel is not None: + self.channel.close() + self.channel = None + self.stub = None + + """ + @RETRY_DECORATOR + def Compute(self, request: E2EOrchestratorRequest) -> E2EOrchestratorReply: + LOGGER.info( + "Compute request: {:s}".format(str(grpc_message_to_json(request))) + ) + response = self.stub.Compute(request) + LOGGER.info( + "Compute result: {:s}".format(str(grpc_message_to_json(response))) + ) + return response + """ \ No newline at end of file diff --git a/src/vnt_manager/client/__init__.py b/src/vnt_manager/client/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/requirements.in b/src/vnt_manager/requirements.in new file mode 100644 index 000000000..4c4720a2d --- /dev/null +++ b/src/vnt_manager/requirements.in @@ -0,0 +1,15 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +networkx \ No newline at end of file diff --git a/src/vnt_manager/service/VNTManagerService.py b/src/vnt_manager/service/VNTManagerService.py new file mode 100644 index 000000000..b61b213a6 --- /dev/null +++ b/src/vnt_manager/service/VNTManagerService.py @@ -0,0 +1,35 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from common.Constants import ServiceNameEnum +from common.proto.vntmanager_pb2_grpc import add_VNTManagerServiceServicer_to_server +from common.Settings import get_service_port_grpc +from common.tools.service.GenericGrpcService import GenericGrpcService +from .VNTManagerServiceServicerImpl import VNTManagerServiceServicerImpl + +LOGGER = logging.getLogger(__name__) + + +class VNTManagerService(GenericGrpcService): + def __init__(self, cls_name: str = __name__): + port = get_service_port_grpc(ServiceNameEnum.VNTMANAGER) + super().__init__(port, cls_name=cls_name) + self.vntmanager_servicer = VNTManagerServiceServicerImpl() + + def install_servicers(self): + add_VNTManagerServiceServicer_to_server( + self.vntmanager_servicer, self.server + ) diff --git a/src/vnt_manager/service/VNTManagerServiceServicerImpl.py b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py new file mode 100644 index 000000000..4869218a7 --- /dev/null +++ b/src/vnt_manager/service/VNTManagerServiceServicerImpl.py @@ -0,0 +1,95 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import networkx as nx +import grpc +import copy + +from common.Constants import ServiceNameEnum +from common.method_wrappers.Decorator import (MetricsPool, MetricTypeEnum, safe_and_metered_rpc_method) +from common.proto.vntmanager_pb2 import VNTManagerRequest, VNTManagerReply +from common.proto.context_pb2 import Empty, Connection, EndPointId +from common.proto.vntmanager_pb2_grpc import VNTManagerServiceServicer +from context.client.ContextClient import ContextClient +from context.service.database.uuids.EndPoint import endpoint_get_uuid + + +LOGGER = logging.getLogger(__name__) + +METRICS_POOL = MetricsPool("VNTManager", "RPC") + +context_client: ContextClient = ContextClient() + + +class E2EOrchestratorServiceServicerImpl(VNTManagerServiceServicer): + def __init__(self): + LOGGER.debug("Creating Servicer...") + LOGGER.debug("Servicer Created") + + """ + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def Compute(self, request: E2EOrchestratorRequest, context: grpc.ServicerContext) -> E2EOrchestratorReply: + endpoints_ids = [] + for endpoint_id in request.service.service_endpoint_ids: + endpoints_ids.append(endpoint_get_uuid(endpoint_id)[2]) + + graph = nx.Graph() + + devices = context_client.ListDevices(Empty()).devices + + for device in devices: + endpoints_uuids = [endpoint.endpoint_id.endpoint_uuid.uuid + for endpoint in device.device_endpoints] + for ep in endpoints_uuids: + graph.add_node(ep) + + for ep in endpoints_uuids: + for ep_i in endpoints_uuids: + if ep == ep_i: + continue + graph.add_edge(ep, ep_i) + + links = context_client.ListLinks(Empty()).links + for link in links: + eps = [] + for endpoint_id in link.link_endpoint_ids: + eps.append(endpoint_id.endpoint_uuid.uuid) + graph.add_edge(eps[0], eps[1]) + + + shortest = nx.shortest_path(graph, endpoints_ids[0], endpoints_ids[1]) + + path = E2EOrchestratorReply() + path.services.append(copy.deepcopy(request.service)) + for i in range(0, int(len(shortest)/2)): + conn = Connection() + ep_a_uuid = str(shortest[i*2]) + ep_z_uuid = str(shortest[i*2+1]) + + conn.connection_id.connection_uuid.uuid = str(ep_a_uuid) + '_->_' + str(ep_z_uuid) + + ep_a_id = EndPointId() + ep_a_id.endpoint_uuid.uuid = ep_a_uuid + conn.path_hops_endpoint_ids.append(ep_a_id) + + ep_z_id = EndPointId() + ep_z_id.endpoint_uuid.uuid = ep_z_uuid + conn.path_hops_endpoint_ids.append(ep_z_id) + + path.connections.append(conn) + + return path + """ \ No newline at end of file diff --git a/src/vnt_manager/service/__init__.py b/src/vnt_manager/service/__init__.py new file mode 100644 index 000000000..38d04994f --- /dev/null +++ b/src/vnt_manager/service/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/vnt_manager/service/__main__.py b/src/vnt_manager/service/__main__.py new file mode 100644 index 000000000..03fb4dd5d --- /dev/null +++ b/src/vnt_manager/service/__main__.py @@ -0,0 +1,80 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import signal +import sys +import threading + +from prometheus_client import start_http_server + +from common.Constants import ServiceNameEnum +from common.Settings import (ENVVAR_SUFIX_SERVICE_HOST, + ENVVAR_SUFIX_SERVICE_PORT_GRPC, get_env_var_name, + get_log_level, get_metrics_port, + wait_for_environment_variables) + +from .VNTManagerService import VNTManagerService + +terminate = threading.Event() +LOGGER = None + + +def signal_handler(signal, frame): # pylint: disable=redefined-outer-name + LOGGER.warning("Terminate signal received") + terminate.set() + + +def main(): + global LOGGER # pylint: disable=global-statement + + log_level = get_log_level() + logging.basicConfig(level=log_level) + LOGGER = logging.getLogger(__name__) + + wait_for_environment_variables( + [ + get_env_var_name(ServiceNameEnum.VNTMANAGER, ENVVAR_SUFIX_SERVICE_HOST), + get_env_var_name(ServiceNameEnum.VNTMANAGER, ENVVAR_SUFIX_SERVICE_PORT_GRPC), + ] + ) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + LOGGER.info("Starting...") + + # Start metrics server + metrics_port = get_metrics_port() + start_http_server(metrics_port) + + # Starting CentralizedCybersecurity service + grpc_service = VNTManagerService() + grpc_service.start() + LOGGER.info("Started...") + # Wait for Ctrl+C or termination signal + + while not terminate.wait(timeout=1): + pass + + + LOGGER.info("Terminating...") + grpc_service.stop() + + LOGGER.info("Bye") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) -- GitLab From 5ea83f3b9ddaa412a86830a93fc1fb62583bd29c Mon Sep 17 00:00:00 2001 From: sgambelluri Date: Wed, 27 Mar 2024 13:05:27 +0000 Subject: [PATCH 035/941] Endpoints Auto Descivory --- my_deploy.sh | 2 +- .../service/ContextServiceServicerImpl.py | 6 +- src/context/service/database/OpticalConfig.py | 74 ++++--- .../database/models/OpticalConfigModel.py | 42 +++- .../database/models/OpticalLinkModel.py | 78 +++++++ .../service/database/uuids/OpticalConfig.py | 17 ++ .../service/drivers/oc_driver/OCDriver.py | 5 +- .../drivers/oc_driver/templates/Tools.py | 11 +- src/tests/ofc24/run_test.sh | 17 ++ src/webui/service/__init__.py | 3 + src/webui/service/opticalconfig/__init__.py | 14 ++ src/webui/service/opticalconfig/forms.py | 22 ++ src/webui/service/opticalconfig/routes.py | 193 ++++++++++++++++++ src/webui/service/templates/base.html | 7 + .../opticalconfig/add_transceiver.html | 40 ++++ .../templates/opticalconfig/details.html | 109 ++++++++++ .../service/templates/opticalconfig/home.html | 68 ++++++ .../opticalconfig/update_interface.html | 73 +++++++ test.py | 41 ++++ 19 files changed, 783 insertions(+), 39 deletions(-) create mode 100644 src/context/service/database/models/OpticalLinkModel.py create mode 100644 src/context/service/database/uuids/OpticalConfig.py create mode 100644 src/tests/ofc24/run_test.sh create mode 100644 src/webui/service/opticalconfig/__init__.py create mode 100644 src/webui/service/opticalconfig/forms.py create mode 100644 src/webui/service/opticalconfig/routes.py create mode 100644 src/webui/service/templates/opticalconfig/add_transceiver.html create mode 100644 src/webui/service/templates/opticalconfig/details.html create mode 100644 src/webui/service/templates/opticalconfig/home.html create mode 100644 src/webui/service/templates/opticalconfig/update_interface.html create mode 100644 test.py diff --git a/my_deploy.sh b/my_deploy.sh index 7dd5e5c3e..9e4447349 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -20,7 +20,7 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/" # Set the list of components, separated by spaces, you want to build images for, and deploy. -export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_generator" +export TFS_COMPONENTS="context device pathcomp opticalcontroller service slice nbi webui " # Uncomment to activate Monitoring #export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring" diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index a102fa176..1be4cdac8 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -305,7 +305,7 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def GetOpticalConfig(self, request : Empty, context : grpc.ServicerContext) -> OpticalConfigList: result = get_opticalconfig(self.db_engine) - return OpticalConfigList(OpticalConfigs=result) + return OpticalConfigList(opticalconfigs=result) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def SetOpticalConfig(self, request : OpticalConfig, context : grpc.ServicerContext) -> OpticalConfigId: @@ -316,5 +316,5 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer def SelectOpticalConfig(self, request : OpticalConfigId, context : grpc.ServicerContext) -> OpticalConfig: result = select_opticalconfig(self.db_engine, request) optical_config_id = OpticalConfigId() - optical_config_id.CopyFrom(result.OpticalConfig_id) - return OpticalConfig(config=result.config, OpticalConfig_id=optical_config_id) + optical_config_id.CopyFrom(result.opticalconfig_id) + return OpticalConfig(config=result.config, opticalconfig_id=optical_config_id) diff --git a/src/context/service/database/OpticalConfig.py b/src/context/service/database/OpticalConfig.py index 9e7552bc1..973e93c33 100644 --- a/src/context/service/database/OpticalConfig.py +++ b/src/context/service/database/OpticalConfig.py @@ -18,7 +18,8 @@ from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker from sqlalchemy_cockroachdb import run_transaction from common.proto.context_pb2 import OpticalConfig, OpticalConfigId -from .models.OpticalConfigModel import OpticalConfigModel +from .models.OpticalConfigModel import OpticalConfigModel , OpticalChannelModel +from context.service.database.uuids.OpticalConfig import channel_get_uuid LOGGER = logging.getLogger(__name__) @@ -26,44 +27,61 @@ def get_opticalconfig(db_engine : Engine): def callback(session:Session): optical_configs = list() results = session.query(OpticalConfigModel).all() + for obj in results: + LOGGER.info(f"opticaln config obj from context {obj.dump()}") + optical_config = OpticalConfig() - optical_config.config = json.dump(obj.config) - optical_config.opticalconfig_id.opticalconfig_uuid = obj.opticalconfig_uuid + optical_config.config = json.dumps(obj.dump()) + optical_config.opticalconfig_id.opticalconfig_uuid = obj.dump_id()["opticalconfig_uuid"] optical_configs.append(optical_config) return optical_configs obj = run_transaction(sessionmaker(bind=db_engine), callback) return obj def set_opticalconfig(db_engine : Engine, request : OpticalConfig): + LOGGER.info(f"request {request} ") opticalconfig_id = OpticalConfigId() opticalconfig_id.opticalconfig_uuid = request.opticalconfig_id.opticalconfig_uuid - my_config_data = [] + OpticalConfig_data = [] if request.config: channels = [] transceivers = [] config = json.loads(request.config) - if 'channels' in config and len(config['channels']) > 0: - channels = [channel['name']['index'] for channel in config['channels']] + if 'transceivers' in config and len(config['transceivers']['transceiver']) > 0: transceivers = [transceiver for transceiver in config['transceivers']['transceiver']] - - my_config_data = [ - { - "opticalconfig_uuid": request.opticalconfig_id.opticalconfig_uuid, - "channels" : channels, - "transcievers" : transceivers, - "interfaces" : json.dumps(config["interfaces"]["interface"]), - "channel_namespace" : config["channel_namespace"], - "endpoints" : [json.dumps(endpoint) for endpoint in config["endpoints"]], - "frequency" : config["frequency"] if "frequency" in config else 0, - "operational_mode" : config["operational_mode"] if "operational_mode" in config else 0, - "output_power" : config["output_power"] if "output_power" in config else '', - } - ] - + + if 'channels' in config and len(config['channels']) > 0: + #channels = [channel['name']['index'] for channel in config['channels']] + for channel_params in config['channels']: + channels.append( + { + "channel_uuid":channel_get_uuid(channel_params['name']['index']), + "opticalconfig_uuid": request.opticalconfig_id.opticalconfig_uuid, + "channel_name" : channel_params['name']['index'], + "frequency" : int(channel_params["frequency"]) if "frequency" in channel_params else 0, + "operational_mode" : int(channel_params["operational-mode"]) if "operational-mode" in channel_params else 0, + "target_output_power" : channel_params["target-output-power"] if "target-output-power" in channel_params else '', + } + ) + + OpticalConfig_data.append( + { + "opticalconfig_uuid": request.opticalconfig_id.opticalconfig_uuid, + "transcievers" : transceivers, + "interfaces" : json.dumps(config["interfaces"]["interface"]), + "channel_namespace" : config["channel_namespace"], + "endpoints" : [json.dumps(endpoint) for endpoint in config["endpoints"]],} + + ) + + + LOGGER.info(f"optical config to set {OpticalConfig_data} ") + LOGGER.info(f"channels {channels}") def callback(session:Session)->bool: - stmt = insert(OpticalConfigModel).values(my_config_data) + stmt = insert(OpticalConfigModel).values(OpticalConfig_data) + stmt = stmt.on_conflict_do_update( index_elements=[OpticalConfigModel.opticalconfig_uuid], set_=dict( @@ -71,7 +89,17 @@ def set_opticalconfig(db_engine : Engine, request : OpticalConfig): ) ) stmt = stmt.returning(OpticalConfigModel.opticalconfig_uuid) - id = session.execute(stmt).fetchone() + opticalconfig_id = session.execute(stmt).fetchone() + if (len(channels)>0) : + + stmt = insert(OpticalChannelModel).values(channels) + + stmt = stmt.on_conflict_do_nothing( + index_elements=[OpticalChannelModel.channel_uuid , OpticalConfigModel.opticalconfig_uuid], + + ) + stmt = stmt.returning(OpticalChannelModel.channel_uuid) + opticalChannel_id = session.execute(stmt).fetchone() opticalconfig_id = run_transaction(sessionmaker(bind=db_engine), callback) return {'opticalconfig_uuid': opticalconfig_id} diff --git a/src/context/service/database/models/OpticalConfigModel.py b/src/context/service/database/models/OpticalConfigModel.py index 10cf197f9..8531f37b2 100644 --- a/src/context/service/database/models/OpticalConfigModel.py +++ b/src/context/service/database/models/OpticalConfigModel.py @@ -13,30 +13,58 @@ # limitations under the License. import json -from sqlalchemy import Column, String, Integer +from sqlalchemy import Column, String, Integer , ForeignKey from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy.orm import relationship from ._Base import _Base class OpticalConfigModel(_Base): __tablename__ = 'optical_config' opticalconfig_uuid = Column(String, primary_key=True) - channels = Column(ARRAY(String), nullable=True) + transcievers = Column(ARRAY(String), nullable=True) interfaces = Column(String, nullable=True) channel_namespace = Column(String, nullable=True) endpoints = Column(ARRAY(String), nullable=True) - frequency = Column(Integer, nullable=True) - operational_mode = Column(Integer, nullable=True) - output_power = Column(String, nullable=True) + channels = relationship("OpticalChannelModel") + + + def dump_id (self ): + return { + "opticalconfig_uuid":self.opticalconfig_uuid + } def dump(self): return { - "channels" : [{'name': {'index': channel}} for channel in self.channels], + "channels" : [channel.dump() for channel in self.channels], "transceivers" : {"transceiver": [transciever for transciever in self.transcievers]}, "interfaces" : {"interface": json.loads(self.interfaces)}, "channel_namespace" : self.channel_namespace, "endpoints" : [json.loads(endpoint) for endpoint in self.endpoints], + + } + + + +class OpticalChannelModel(_Base): + __tablename__ = 'optical_channel' + channel_uuid = Column(String, primary_key=True) + channel_name = Column (String,nullable=True) + frequency = Column(Integer, nullable=True) + operational_mode = Column(Integer, nullable=True) + target_output_power = Column(String, nullable=True) + opticalconfig_uuid = Column(ForeignKey('optical_config.opticalconfig_uuid', ondelete='CASCADE' ), primary_key=True) + opticalconfig = relationship('OpticalConfigModel', back_populates='channels') + def dump_id (self ): + return { + "channel_uuid":self.channel_uuid + } + + def dump(self): + return { + "name" :{'index':self.channel_name}, "frequency" : self.frequency, - "output_power" : self.output_power, + "target_output_power" : self.target_output_power, "operational_mode" : self.operational_mode, } + diff --git a/src/context/service/database/models/OpticalLinkModel.py b/src/context/service/database/models/OpticalLinkModel.py new file mode 100644 index 000000000..b94eeda93 --- /dev/null +++ b/src/context/service/database/models/OpticalLinkModel.py @@ -0,0 +1,78 @@ + +import operator +from sqlalchemy import CheckConstraint, Column, DateTime, Float, ForeignKey, Integer, String ,Boolean +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.types import ARRAY +from sqlalchemy.orm import relationship +from typing import Dict +from ._Base import _Base + +class OpticalLinkModel(_Base): + __tablename__ = 'opticallink' + + optical_link_uuid = Column(UUID(as_uuid=False), primary_key=True) + optical_link_name = Column(String, nullable=False) + length = Column(Integer, nullable=True) + source = Column(String, nullable=True) + target = Column(String, nullable=True) + optical_link_fiber= relationship("FiberModel") + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + + + + def dump_id(self) -> Dict: + return {'optical_link_uuid': {'uuid': self.link_uuid}} + + def dump(self) -> Dict: + result = { + 'optical_link_id' : self.dump_id(), + 'name' : self.optical_link_name, + 'details': { + "length" : self.length, + 'source' : self.source, + "target" : self.target, + 'fibers' : [ fiber.dump() for fiber in self.optical_link_fiber ] + } + + } + + return result + +class FiberModel(_Base): + __tablename__ = 'fiber' + fiber_uuid = Column(UUID(as_uuid=False), primary_key=True) + fiber_length = Column(Integer, nullable=True) + source_port = Column(String, nullable=True) + destination_port = Column(String, nullable=True) + local_peer_port = Column(String, nullable=True) + remote_peer_port = Column(String, nullable=True) + used = Column(Boolean ,nullable=true) + c_slots = Column (ARRAY(Integer),nullable=True) + l_slots = Column (ARRAY(Integer),nullable=True) + s_slots = Column (ARRAY(Integer),nullable=True) + optical_link_uuid = Column(ForeignKey('opticallink.optical_link_uuid', ondelete='CASCADE' ), primary_key=True) + optical_link = relationship('OpticalLinkModel', back_populates='optical_link_fibers') + + + def dump_id(self) -> Dict: + return {'fiber_uuid': {'uuid': self.fiber_uuid}} + + + def dump(self) -> Dict: + result = { + 'ID' : self.dump_id(), + 'length' : self.fiber_length, + "src_port" : self.source_port, + "dst_port" : self.destination_port, + "local_peer_port" : self.local_peer_port, + "remote_peer_port" : self.remote_peer_port, + "used" : self.used, + "c_slots" : self.c_slots , + "l_slots" : self.l_slots, + "s_slots" : self.s_slots + + } + + return result \ No newline at end of file diff --git a/src/context/service/database/uuids/OpticalConfig.py b/src/context/service/database/uuids/OpticalConfig.py new file mode 100644 index 000000000..0003b5712 --- /dev/null +++ b/src/context/service/database/uuids/OpticalConfig.py @@ -0,0 +1,17 @@ + +from common.method_wrappers.ServiceExceptions import InvalidArgumentsException +from ._Builder import get_uuid_from_string, get_uuid_random + +def channel_get_uuid( + channel_name :str , allow_random : bool = False +) -> str: + + + if len(channel_name) > 0: + return get_uuid_from_string(channel_name) + if allow_random: return get_uuid_random() + + raise InvalidArgumentsException([ + ('channel uuid', channel_name), + + ], extra_details=['Channel name is required to produce a channel UUID']) diff --git a/src/device/service/drivers/oc_driver/OCDriver.py b/src/device/service/drivers/oc_driver/OCDriver.py index 16f00cfb4..4b49c7e41 100644 --- a/src/device/service/drivers/oc_driver/OCDriver.py +++ b/src/device/service/drivers/oc_driver/OCDriver.py @@ -257,7 +257,8 @@ class OCDriver(_Driver): xml_data = self.__netconf_handler.get().data_xml transceivers,interfaces,channels_lst,channel_namespace,endpoints=extractor(data_xml=xml_data,resource_keys=filter_fields,dic=config) - + logging.info(f"xml response {xml_data}") + except Exception as e: # pylint: disable=broad-except MSG = 'Exception retrieving {:s}' self.__logger.info("error from getConfig %s",e) @@ -273,7 +274,7 @@ class OCDriver(_Driver): value_dic["interfaces"]=interfaces value_dic["channel_namespace"]=channel_namespace value_dic["endpoints"]=endpoints - + logging.info(f"parameters {value_dic}") opticalConfig.config=json.dumps(value_dic) opticalConfig.opticalconfig_id.opticalconfig_uuid=self.__device_uuid if self.__device_uuid is not None else "" config_id=context_client.SetOpticalConfig(opticalConfig) diff --git a/src/device/service/drivers/oc_driver/templates/Tools.py b/src/device/service/drivers/oc_driver/templates/Tools.py index 909bdd83b..bc69db6a6 100644 --- a/src/device/service/drivers/oc_driver/templates/Tools.py +++ b/src/device/service/drivers/oc_driver/templates/Tools.py @@ -140,9 +140,10 @@ def extract_channels_based_on_type (xml_data:str): return channel_names def extract_value(resource_key:str,xml_data:str,dic:dict,channel_name:str,channel_namespace:str): + logging.info(f"resource_key {resource_key} and channgel_name {channel_name} and channel_namespace {channel_namespace}") xml_bytes = xml_data.encode("utf-8") root = ET.fromstring(xml_bytes) - + channel_name=channel_name if 'index' not in channel_name else channel_name['index'] namespace = {'oc': 'http://openconfig.net/yang/platform', 'td': channel_namespace} @@ -153,10 +154,13 @@ def extract_value(resource_key:str,xml_data:str,dic:dict,channel_name:str,channe if (parameter is not None): value = parameter.text dic[resource_key]=value + else : + logging.info("parameter is None") else: + logging.info("element is None") print(" element not found.") - + logging.info(f"dic {dic}") return dic @@ -244,10 +248,11 @@ def extractor(data_xml:str,resource_keys:list,dic:dict): for channel_name in channel_names: dic={} - for resource_key in resource_keys : + for resource_key in resource_keys : if (resource_key != 'interface'): dic=extract_value(dic=dic,resource_key=resource_key,xml_data=data_xml,channel_name=channel_name,channel_namespace=channel_namespace) + dic["name"]=channel_name endpoints.append({"endpoint_uuid":{"uuid":channel_name}}) lst_dic.append(dic) diff --git a/src/tests/ofc24/run_test.sh b/src/tests/ofc24/run_test.sh new file mode 100644 index 000000000..660436552 --- /dev/null +++ b/src/tests/ofc24/run_test.sh @@ -0,0 +1,17 @@ +#!/bin/bash +docker stop na1 +docker rm na1 + screen -dmS t1 -T xterm sh -c "docker run -p 10.0.2.10:2023:2022 -v ~/tfs-ctrl/tempOC/files:/files --name na1 -it asgamb1/oc23bgp.img:latest +" + sleep 2 + if [ "$( docker container inspect -f '{{.State.Running}}' na1)" = "true" ]; then + + + + docker exec na1 sh -c " cp /files/platform_t1.xml demoECOC21.xml ; + + /confd/examples.confd/OC23/startNetconfAgent.sh; " + else + echo "your container is not running yet" + fi + diff --git a/src/webui/service/__init__.py b/src/webui/service/__init__.py index 63192016c..196652d3f 100644 --- a/src/webui/service/__init__.py +++ b/src/webui/service/__init__.py @@ -83,6 +83,9 @@ def create_app(use_config=None, web_app_root=None): from webui.service.load_gen.routes import load_gen # pylint: disable=import-outside-toplevel app.register_blueprint(load_gen) + + from webui.service.opticalconfig.routes import opticalconfig # pylint: disable=import-outside-toplevel + app.register_blueprint(opticalconfig) from webui.service.service.routes import service # pylint: disable=import-outside-toplevel app.register_blueprint(service) diff --git a/src/webui/service/opticalconfig/__init__.py b/src/webui/service/opticalconfig/__init__.py new file mode 100644 index 000000000..1549d9811 --- /dev/null +++ b/src/webui/service/opticalconfig/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/webui/service/opticalconfig/forms.py b/src/webui/service/opticalconfig/forms.py new file mode 100644 index 000000000..30c133279 --- /dev/null +++ b/src/webui/service/opticalconfig/forms.py @@ -0,0 +1,22 @@ + + +from flask_wtf import FlaskForm +from wtforms import StringField, SelectField, TextAreaField, SubmitField +from wtforms.validators import DataRequired, Length, NumberRange, ValidationError +from common.proto.context_pb2 import DeviceOperationalStatusEnum + +class UpdateDeviceForm(FlaskForm): + power = StringField('Power') + frequency= StringField("Frequency") + operational_mode=StringField("Operational Mode") + line_port=SelectField("Line Port") + + + submit = SubmitField('Update') + +class AddTrancseiver (FlaskForm): + transceiver = StringField("Transceiver") + submit = SubmitField('Add') +class UpdateInterfaceForm (FlaskForm): + ip=StringField("IP Address") + prefix_length=StringField("Prefix Length") \ No newline at end of file diff --git a/src/webui/service/opticalconfig/routes.py b/src/webui/service/opticalconfig/routes.py new file mode 100644 index 000000000..5445d140e --- /dev/null +++ b/src/webui/service/opticalconfig/routes.py @@ -0,0 +1,193 @@ +import base64, json, logging #, re +from flask import jsonify, redirect, render_template, Blueprint, flash, session, url_for, request +from common.proto.context_pb2 import (ContextList, Empty, TopologyId, TopologyList +,DeviceId,DeviceList ,OpticalConfig, OpticalConfigId ,OpticalConfigList) +from common.tools.descriptor.Loader import DescriptorLoader, compose_notifications +from common.tools.grpc.Tools import grpc_message_to_json_string +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.Topology import json_topology_id +from context.client.ContextClient import ContextClient +from device.client.DeviceClient import DeviceClient +from service.client.ServiceClient import ServiceClient +from slice.client.SliceClient import SliceClient +from webui.service.main.forms import ContextTopologyForm, DescriptorForm +from .forms import UpdateDeviceForm ,AddTrancseiver ,UpdateInterfaceForm + + + +opticalconfig = Blueprint('opticalconfig', __name__,url_prefix="/opticalconfig") + +context_client = ContextClient() +device_client = DeviceClient() +service_client = ServiceClient() +slice_client = SliceClient() + +LOGGER = logging.getLogger(__name__) + +DESCRIPTOR_LOADER_NUM_WORKERS = 10 + +@opticalconfig.get("/") +def home() : + config=[] + deviceId= DeviceId() + + if 'context_uuid' not in session or 'topology_uuid' not in session: + flash("Please select a context!", "warning") + return redirect(url_for("main.home")) + context_uuid = session['context_uuid'] + topology_uuid = session['topology_uuid'] + + context_client.connect() + opticalConfig_list:OpticalConfigList = context_client.GetOpticalConfig(Empty()) + logging.info("myconfigList %s",opticalConfig_list) + + for configs in opticalConfig_list.opticalconfigs: + + value=json.loads(configs.config) if type(configs.config)==str else configs.config + value["channels_number"]=len(value['channels']) + + # value['operationalMode']=value['operational-mode'] + # value['targetOutputPower']=value['target-output-power'] + value['opticalconfig_id']=configs.opticalconfig_id + # value['line_port']=value["line-port"] + + config.append(value) + + logging.info("opticalConfig %s",config) + + context_client.close() + + return render_template( + 'opticalconfig/home.html', config=config) +@opticalconfig.route('/detail',methods=['GET']) +def show_details(config_uuid): + opticalconfigId=OpticalConfigId() + opticalconfigId.opticalconfig_uuid=config_uuid + device_details=[] + context_client.connect() + response = context_client.SelectOpticalConfig(opticalconfigId) + context_client.close() + LOGGER.info("response %s",response) + opticalConfig = OpticalConfig() + opticalConfig.CopyFrom(response) + + config =json.loads(opticalConfig.config) + LOGGER.info("config details %s",config) + interfaces=config["interfaces"] + new_config={} + + for channel in config['channels'] : + + new_config["name"]=channel['name'] + new_config['operationalMode']=channel['operational_mode'] if 'operational_mode' in channel else '' + new_config['targetOutputPower']=channel['target_output_power'] if 'target_output_power' in channel else '' + new_config["frequency"]=channel['frequency'] if 'frequency' in channel else '' + new_config['line_port']=channel["line-port"] if 'line-port' in channel else '' + + device_details.append(new_config) + LOGGER.info("config details %s",device_details) + return render_template('opticalconfig/details.html', device=device_details,config_id=config_uuid,interfaces=interfaces) +@opticalconfig.route('//update', methods=['GET', 'POST']) +def update(config_uuid,channel_name): + form = UpdateDeviceForm() + + opticalconfigId=OpticalConfigId() + opticalconfigId.opticalconfig_uuid=config_uuid + context_client.connect() + response = context_client.SelectOpticalConfig(opticalconfigId) + context_client.close() + LOGGER.info("response %s",response) + opticalconfig = OpticalConfig() + opticalconfig.CopyFrom(response) + config =json.loads(opticalconfig.config) + new_config={} + for channel in config['channels']: + if (channel["name"] == channel_name): + new_config=channel + form.frequency.default = channel["frequency"] + form.operational_mode.default=channel["operational-mode"] + form.power.default=channel["target-output-power"] + form.line_port.choices = [("","")] + + for transceiver in config["transceivers"]['transceiver']: + + form.line_port.choices.append((transceiver,transceiver)) + # listing enum values + + if form.validate_on_submit(): + + new_config["target-output-power"] =form.power.data if form.power.data != '' else new_config['target-output-power'] + new_config["frequency"]=form.frequency.data if form.frequency.data != '' else new_config['frequency'] + new_config["operational-mode"]=form.operational_mode.data if form.operational_mode.data != '' else new_config['operational-mode'] + new_config["line-port"]=form.line_port.data if form.line_port.data != '' else new_config['line-port'] + + opticalconfig.config =json.dumps(new_config) + LOGGER.info("myconfig copied %s",opticalconfig) + try: + device_client.connect() + device_client.ConfigureOpticalDevice(opticalconfig) + device_client.close() + flash(f' device was updated.', 'success') + return redirect(url_for('opticalconfig.show_details',config_uuid=config_uuid)) + except Exception as e: # pylint: disable=broad-except + flash(f'Problem updating the device. {e}', 'danger') + return render_template('myabout/update.html', device=response, form=form, submit_text='Update Device',channel_name=channel_name) +@opticalconfig.route('//update_interface', methods=['GET', 'POST']) +def update_interface (config_uuid,interface_name): + form = UpdateInterfaceForm() + opticalconfigId=OpticalConfigId() + opticalconfigId.opticalconfig_uuid=config_uuid + context_client.connect() + response = context_client.SelectOpticalConfig(myid) + context_client.close() + LOGGER.info("response %s",response) + opticalconfig = OpticalConfig() + opticalconfig.CopyFrom(response) + config =json.loads(opticalconfig.config) + new_config={} + if form.validate_on_submit(): + new_config["ip"]=form.ip.data if form.ip.data != "" else config["interfaces"]["interface"]["ip"] + new_config["prefix-length"]=form.prefix_length.data if form.prefix_length.data != "" else config["interfaces"]["interface"]["prefix-length"] + new_config["name"]=config["interfaces"]["interface"]["name"] + new_config["enabled"]=config["interfaces"]["interface"]["enabled"] + + opticalconfig.config=json.dumps({"update_interface":new_config}) + try: + device_client.connect() + device_client.ConfigureOpticalDevice(opticalconfig) + device_client.close() + flash(f' device was updated.', 'success') + return redirect(url_for('opticalconfig.show_details',config_uuid=config_uuid)) + except Exception as e: # pylint: disable=broad-except + flash(f'Problem updating the device. {e}', 'danger') + return render_template('opticalconfig/update_interface.html', + device=response, form=form, submit_text='Update interface',interface_name=interface_name) + +@opticalconfig.route('/add_transceiver', methods=['GET','POST']) +def add_transceiver (config_uuid): + config={} + addtrancseiver=AddTrancseiver() + opticalconfigId=OpticalConfig() + opticalconfigId.opticalconfig_uuid=config_uuid + context_client.connect() + response = context_client.SelectOpticalConfig(opticalconfigId) + context_client.close() + opticlConfig=OpticalConfig() + opticlConfig.CopyFrom(response) + if addtrancseiver.validate_on_submit(): + config["add_transceiver"]=addtrancseiver.transceiver.data + opticlConfig.config=json.dumps(config) + + try: + device_client.connect() + device_client.ConfigureOpticalDevice(opticlConfig) + device_client.close() + flash(f' device was updated.', 'success') + return redirect(url_for('opticalconfig.update',config_uuid=config_uuid)) + except Exception as e: # pylint: disable=broad-except + flash(f'Problem updating the device. {e}', 'danger') + return render_template('opticalconfig/add_transceiver.html',form=addtrancseiver, submit_text='Add Trancseiver') + + + + \ No newline at end of file diff --git a/src/webui/service/templates/base.html b/src/webui/service/templates/base.html index 60cd5aebd..e081b2710 100644 --- a/src/webui/service/templates/base.html +++ b/src/webui/service/templates/base.html @@ -83,6 +83,13 @@ Slice {% endif %} +