diff --git a/deploy/tfs.sh b/deploy/tfs.sh index be83d7f5b2669abe8330adefa8a8feac27a1dab8..facba7cfd85abfc6a615528d837c37e09406f9d5 100755 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -422,7 +422,7 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]]; then }, "secureJsonData": {"password": "'${QDB_PASSWORD}'"} }' ${GRAFANA_URL_UPDATED}/api/datasources - printf "\n\n" + echo # adding the datasource of the metrics collection framework curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{ diff --git a/manifests/cockroachdb/cluster.yaml b/manifests/cockroachdb/cluster.yaml index f7444c0067cc9c2c07b53c85d765bb81d1c20c05..b618c6f17c2e00b9ff74509955b060c67ea3d2cc 100644 --- a/manifests/cockroachdb/cluster.yaml +++ b/manifests/cockroachdb/cluster.yaml @@ -42,7 +42,8 @@ spec: # You can set either a version of the db or a specific image name # cockroachDBVersion: v22.2.0 image: - name: cockroachdb/cockroach:v22.2.0 + #name: cockroachdb/cockroach:v22.2.0 + name: cockroachdb/cockroach:latest-v22.2 # nodes refers to the number of crdb pods that are created # via the statefulset nodes: 3 diff --git a/manifests/cockroachdb/single-node.yaml b/manifests/cockroachdb/single-node.yaml index 72454a0904fa70b6b4062dae8ef7e2e5d8625648..20d7e18c69a9eb75bcb70c9123455926d41eebc6 100644 --- a/manifests/cockroachdb/single-node.yaml +++ b/manifests/cockroachdb/single-node.yaml @@ -61,6 +61,7 @@ spec: containers: - name: cockroachdb image: cockroachdb/cockroach:latest-v22.2 + imagePullPolicy: Always args: - start-single-node ports: diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index ad54f4b6c2682c381c1c5238a013e1d12e177764..f9a6d987d18bb3d994538c85b2ec14024553b45b 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -45,11 +45,11 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:2020"] resources: requests: - cpu: 128m - memory: 64Mi - limits: - cpu: 256m + cpu: 250m memory: 128Mi + limits: + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service diff --git a/manifests/serviceservice.yaml b/manifests/serviceservice.yaml index ce90aa18854522f1c08e213cb554c70af70bac36..7d7bdaa4ef9ad4972da6236071810c63a9faa4f8 100644 --- a/manifests/serviceservice.yaml +++ b/manifests/serviceservice.yaml @@ -45,11 +45,11 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:3030"] resources: requests: - cpu: 32m - memory: 32Mi + cpu: 250m + memory: 128Mi limits: - cpu: 128m - memory: 64Mi + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service diff --git a/manifests/sliceservice.yaml b/manifests/sliceservice.yaml index 8f312e8e0c89c5b8ed1923622078ea16b6bd876e..e7e5c1604a8b971424ff5f7e5bf292c4b263cbfe 100644 --- a/manifests/sliceservice.yaml +++ b/manifests/sliceservice.yaml @@ -50,11 +50,11 @@ spec: command: ["/bin/grpc_health_probe", "-addr=:4040"] resources: requests: - cpu: 32m + cpu: 250m memory: 128Mi limits: - cpu: 128m - memory: 256Mi + cpu: 1000m + memory: 1024Mi --- apiVersion: v1 kind: Service diff --git a/src/common/method_wrappers/Decorator.py b/src/common/method_wrappers/Decorator.py index c5cbfb5659df7697de93a99da39a228d5df04001..a5db54125518933199c27662bc0c044988d1daac 100644 --- a/src/common/method_wrappers/Decorator.py +++ b/src/common/method_wrappers/Decorator.py @@ -34,11 +34,17 @@ METRIC_TO_CLASS_PARAMS = { MetricTypeEnum.HISTOGRAM_DURATION: (Histogram, { 'buckets': ( # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - INF) + #0.0010, 0.0025, 0.0050, 0.0075, + #0.0100, 0.0250, 0.0500, 0.0750, + #0.1000, 0.2500, 0.5000, 0.7500, + #1.0000, 2.5000, 5.0000, 7.5000, + 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.007, 0.008, 0.009, # 1~9 ms + 0.010, 0.020, 0.030, 0.040, 0.050, 0.060, 0.070, 0.080, 0.090, # 10~90 ms + 0.100, 0.200, 0.300, 0.400, 0.500, 0.600, 0.700, 0.800, 0.900, # 100~900 ms + 1.000, 2.000, 3.000, 4.000, 5.000, 6.000, 7.000, 8.000, 9.000, # 1~9 sec + 10.00, 20.00, 30.00, 40.00, 50.00, 60.00, 70.00, 80.00, 90.00, # 10~90 sec + 100.0, 110.0, 120.0, INF # 100sec~2min & Infinity + ) }) } diff --git a/src/common/method_wrappers/tests/grafana_prometheus_device_config_exec_details.json b/src/common/method_wrappers/tests/grafana_prometheus_device_config_exec_details.json new file mode 100644 index 0000000000000000000000000000000000000000..980b9583a898f8fb2d84c22329ce8053e120b7f8 --- /dev/null +++ b/src/common/method_wrappers/tests/grafana_prometheus_device_config_exec_details.json @@ -0,0 +1,185 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 30, + "iteration": 1682003744753, + "links": [], + "panels": [ + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "linear", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "max": null, + "min": 0, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "prometheus", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 22, + "w": 24, + "x": 0, + "y": 0 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "60s", + "legend": { + "show": true + }, + "pluginVersion": "7.5.4", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(\r\n max_over_time(tfs_device_exec_details_configuredevice_histogram_duration_bucket{pod=~\"[[pod]]\", step_name=~\"[[step_name]]\"}[1m]) -\r\n min_over_time(tfs_device_exec_details_configuredevice_histogram_duration_bucket{pod=~\"[[pod]]\", step_name=~\"[[step_name]]\"}[1m])\r\n) by (le)", + "format": "heatmap", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Histogram", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(tfs_device_exec_details_configuredevice_histogram_duration_bucket, pod)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "pod", + "options": [], + "query": { + "query": "label_values(tfs_device_exec_details_configuredevice_histogram_duration_bucket, pod)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "get_device", + "set_device" + ], + "value": [ + "get_device", + "set_device" + ] + }, + "datasource": "prometheus", + "definition": "label_values(tfs_device_exec_details_configuredevice_histogram_duration_bucket, step_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Step Name", + "multi": true, + "name": "step_name", + "options": [], + "query": { + "query": "label_values(tfs_device_exec_details_configuredevice_histogram_duration_bucket, step_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "2023-04-20T15:11:48.272Z", + "to": "2023-04-20T15:18:10.464Z" + }, + "timepicker": {}, + "timezone": "", + "title": "TFS / ConfigureDevice Details", + "uid": "GfzKHbPVk", + "version": 4 +} \ No newline at end of file diff --git a/src/common/tools/context_queries/Connection.py b/src/common/tools/context_queries/Connection.py new file mode 100644 index 0000000000000000000000000000000000000000..3021335131332dab73d6d645f4c7937f499732ef --- /dev/null +++ b/src/common/tools/context_queries/Connection.py @@ -0,0 +1,43 @@ +# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc, logging +from typing import Optional +from common.proto.context_pb2 import Connection, ConnectionId +from context.client.ContextClient import ContextClient + +LOGGER = logging.getLogger(__name__) + +def get_connection_by_id( + context_client : ContextClient, connection_id : ConnectionId, rw_copy : bool = False +) -> Optional[Connection]: + try: + ro_connection : Connection = context_client.GetConnection(connection_id) + if not rw_copy: return ro_connection + rw_connection = Connection() + rw_connection.CopyFrom(ro_connection) + return rw_connection + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member + #connection_uuid = connection_id.connection_uuid.uuid + #LOGGER.exception('Unable to get connection({:s})'.format(str(connection_uuid))) + return None + +def get_connection_by_uuid( + context_client : ContextClient, connection_uuid : str, rw_copy : bool = False +) -> Optional[Connection]: + # pylint: disable=no-member + connection_id = ConnectionId() + connection_id.connection_uuid.uuid = connection_uuid + return get_connection_by_id(context_client, connection_id, rw_copy=rw_copy) diff --git a/src/common/tools/context_queries/Device.py b/src/common/tools/context_queries/Device.py index 166882f2fd90475d6bc64b4c4a7c44535fc6aa64..95f0e90b740d3b167f09db394e259599db20a59b 100644 --- a/src/common/tools/context_queries/Device.py +++ b/src/common/tools/context_queries/Device.py @@ -14,23 +14,34 @@ import grpc, logging from typing import List, Optional, Set -from common.proto.context_pb2 import ContextId, Device, DeviceId, Empty, Topology, TopologyId +from common.proto.context_pb2 import ContextId, Device, DeviceFilter, Empty, Topology, TopologyId from common.tools.object_factory.Topology import json_topology_id from context.client.ContextClient import ContextClient LOGGER = logging.getLogger(__name__) -def get_device(context_client : ContextClient, device_uuid : str, rw_copy : bool = False) -> Optional[Device]: +def get_device( + context_client : ContextClient, device_uuid : str, rw_copy : bool = False, + include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True +) -> Optional[Device]: + device_filter = DeviceFilter() + device_id = device_filter.device_ids.device_ids.add() # pylint: disable=no-member + device_id.device_uuid.uuid = device_uuid + device_filter.include_endpoints = include_endpoints + device_filter.include_config_rules = include_config_rules + device_filter.include_components = include_components + try: - # pylint: disable=no-member - device_id = DeviceId() - device_id.device_uuid.uuid = device_uuid - ro_device = context_client.GetDevice(device_id) + ro_devices = context_client.SelectDevice(device_filter) + if len(ro_devices.devices) == 0: return None + assert len(ro_devices.devices) == 1 + ro_device = ro_devices.devices[0] if not rw_copy: return ro_device rw_device = Device() rw_device.CopyFrom(ro_device) return rw_device - except grpc.RpcError: + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member #LOGGER.exception('Unable to get Device({:s})'.format(str(device_uuid))) return None diff --git a/src/common/tools/context_queries/Service.py b/src/common/tools/context_queries/Service.py index 25716152c3f37fec93df340073bae8871e16c3a7..b3b74827a5c86838cb4330ae89c1297652dc59b0 100644 --- a/src/common/tools/context_queries/Service.py +++ b/src/common/tools/context_queries/Service.py @@ -15,25 +15,43 @@ import grpc, logging from typing import Optional from common.Constants import DEFAULT_CONTEXT_NAME -from common.proto.context_pb2 import Service, ServiceId +from common.proto.context_pb2 import Service, ServiceFilter, ServiceId from context.client.ContextClient import ContextClient LOGGER = logging.getLogger(__name__) -def get_service( - context_client : ContextClient, service_uuid : str, context_uuid : str = DEFAULT_CONTEXT_NAME, - rw_copy : bool = False - ) -> Optional[Service]: +def get_service_by_id( + context_client : ContextClient, service_id : ServiceId, rw_copy : bool = False, + include_endpoint_ids : bool = True, include_constraints : bool = True, include_config_rules : bool = True +) -> Optional[Service]: + service_filter = ServiceFilter() + service_filter.service_ids.service_ids.append(service_id) # pylint: disable=no-member + service_filter.include_endpoint_ids = include_endpoint_ids + service_filter.include_constraints = include_constraints + service_filter.include_config_rules = include_config_rules + try: - # pylint: disable=no-member - service_id = ServiceId() - service_id.context_id.context_uuid.uuid = context_uuid - service_id.service_uuid.uuid = service_uuid - ro_service = context_client.GetService(service_id) + ro_services = context_client.SelectService(service_filter) + if len(ro_services.services) == 0: return None + assert len(ro_services.services) == 1 + ro_service = ro_services.services[0] if not rw_copy: return ro_service rw_service = Service() rw_service.CopyFrom(ro_service) return rw_service - except grpc.RpcError: + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member #LOGGER.exception('Unable to get service({:s} / {:s})'.format(str(context_uuid), str(service_uuid))) return None + +def get_service_by_uuid( + context_client : ContextClient, service_uuid : str, context_uuid : str = DEFAULT_CONTEXT_NAME, + rw_copy : bool = False, include_endpoint_ids : bool = True, include_constraints : bool = True, + include_config_rules : bool = True +) -> Optional[Service]: + service_id = ServiceId() + service_id.context_id.context_uuid.uuid = context_uuid # pylint: disable=no-member + service_id.service_uuid.uuid = service_uuid # pylint: disable=no-member + return get_service_by_id( + context_client, service_id, rw_copy=rw_copy, include_endpoint_ids=include_endpoint_ids, + include_constraints=include_constraints, include_config_rules=include_config_rules) diff --git a/src/common/tools/context_queries/Slice.py b/src/common/tools/context_queries/Slice.py index e5fb86d7a5aa8c08bf323f641737efcf8eec14ef..c3ce572fce8b3fb209b46b561a4004979dce4913 100644 --- a/src/common/tools/context_queries/Slice.py +++ b/src/common/tools/context_queries/Slice.py @@ -15,25 +15,47 @@ import grpc, logging from typing import Optional from common.Constants import DEFAULT_CONTEXT_NAME -from common.proto.context_pb2 import Slice, SliceId +from common.proto.context_pb2 import Slice, SliceFilter, SliceId from context.client.ContextClient import ContextClient LOGGER = logging.getLogger(__name__) -def get_slice( - context_client : ContextClient, slice_uuid : str, context_uuid : str = DEFAULT_CONTEXT_NAME, - rw_copy : bool = False - ) -> Optional[Slice]: +def get_slice_by_id( + context_client : ContextClient, slice_id : SliceId, rw_copy : bool = False, include_endpoint_ids : bool = True, + include_constraints : bool = True, include_service_ids : bool = True, include_subslice_ids : bool = True, + include_config_rules : bool = True +) -> Optional[Slice]: + slice_filter = SliceFilter() + slice_id = slice_filter.slice_ids.slice_ids.append(slice_id) # pylint: disable=no-member + slice_filter.include_endpoint_ids = include_endpoint_ids + slice_filter.include_constraints = include_constraints + slice_filter.include_service_ids = include_service_ids + slice_filter.include_subslice_ids = include_subslice_ids + slice_filter.include_config_rules = include_config_rules + try: - # pylint: disable=no-member - slice_id = SliceId() - slice_id.context_id.context_uuid.uuid = context_uuid - slice_id.slice_uuid.uuid = slice_uuid - ro_slice = context_client.GetSlice(slice_id) + ro_slices = context_client.SelectSlice(slice_filter) + if len(ro_slices.slices) == 0: return None + assert len(ro_slices.slices) == 1 + ro_slice = ro_slices.slices[0] if not rw_copy: return ro_slice rw_slice = Slice() rw_slice.CopyFrom(ro_slice) return rw_slice - except grpc.RpcError: + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member #LOGGER.exception('Unable to get slice({:s} / {:s})'.format(str(context_uuid), str(slice_uuid))) return None + +def get_slice_by_uuid( + context_client : ContextClient, slice_uuid : str, context_uuid : str = DEFAULT_CONTEXT_NAME, + rw_copy : bool = False, include_endpoint_ids : bool = True, include_constraints : bool = True, + include_service_ids : bool = True, include_subslice_ids : bool = True, include_config_rules : bool = True +) -> Optional[Slice]: + slice_id = SliceId() + slice_id.context_id.context_uuid.uuid = context_uuid # pylint: disable=no-member + slice_id.slice_uuid.uuid = slice_uuid # pylint: disable=no-member + return get_slice_by_id( + context_client, slice_id, rw_copy=rw_copy, include_endpoint_ids=include_endpoint_ids, + include_constraints=include_constraints, include_service_ids=include_service_ids, + include_subslice_ids=include_subslice_ids, include_config_rules=include_config_rules) diff --git a/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_Service.py b/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_Service.py index f12c4526aec27a60e579540ecae90720c707a117..9a33cd2281d4dfd7a0a8dac964b9b35d7975cf28 100644 --- a/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_Service.py +++ b/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_Service.py @@ -17,7 +17,7 @@ from flask import request from flask.json import jsonify from flask_restful import Resource from common.proto.context_pb2 import SliceStatusEnum -from common.tools.context_queries.Slice import get_slice +from common.tools.context_queries.Slice import get_slice_by_uuid from context.client.ContextClient import ContextClient from slice.client.SliceClient import SliceClient from ..tools.Authentication import HTTP_AUTH @@ -34,7 +34,7 @@ class L2VPN_Service(Resource): try: context_client = ContextClient() - target = get_slice(context_client, vpn_id, rw_copy=True) + target = get_slice_by_uuid(context_client, vpn_id, rw_copy=True) if target is None: raise Exception('VPN({:s}) not found in database'.format(str(vpn_id))) @@ -59,7 +59,7 @@ class L2VPN_Service(Resource): try: context_client = ContextClient() - target = get_slice(context_client, vpn_id) + target = get_slice_by_uuid(context_client, vpn_id) if target is None: LOGGER.warning('VPN({:s}) not found in database. Nothing done.'.format(str(vpn_id))) else: diff --git a/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_SiteNetworkAccesses.py b/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_SiteNetworkAccesses.py index ff7ad3c1481d3c0f3cdf7a6b6004f62677948ecc..7e829479a0a3dbd4968d488a22dc62219fa5376c 100644 --- a/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_SiteNetworkAccesses.py +++ b/src/compute/service/rest_server/nbi_plugins/ietf_l2vpn/L2VPN_SiteNetworkAccesses.py @@ -20,7 +20,7 @@ from flask.wrappers import Response from flask_restful import Resource from werkzeug.exceptions import UnsupportedMediaType from common.proto.context_pb2 import Slice -from common.tools.context_queries.Slice import get_slice +from common.tools.context_queries.Slice import get_slice_by_uuid from common.tools.grpc.ConfigRules import update_config_rule_custom from common.tools.grpc.Constraints import ( update_constraint_custom_dict, update_constraint_endpoint_location, update_constraint_endpoint_priority, @@ -68,7 +68,7 @@ def process_site_network_access(context_client : ContextClient, site_id : str, s address_ip, address_prefix, remote_router, circuit_id ) = mapping - target = get_slice(context_client, vpn_id, rw_copy=True) + target = get_slice_by_uuid(context_client, vpn_id, rw_copy=True) if target is None: raise Exception('VPN({:s}) not found in database'.format(str(vpn_id))) endpoint_ids = target.slice_endpoint_ids # pylint: disable=no-member diff --git a/src/compute/service/rest_server/nbi_plugins/ietf_network_slice/NSS_Service.py b/src/compute/service/rest_server/nbi_plugins/ietf_network_slice/NSS_Service.py index f9b17c8b144ad6cf477b978ceb50b3497c9c074b..32ee81e801e1d93a23a8a752aa6c63cfffe43a82 100644 --- a/src/compute/service/rest_server/nbi_plugins/ietf_network_slice/NSS_Service.py +++ b/src/compute/service/rest_server/nbi_plugins/ietf_network_slice/NSS_Service.py @@ -16,7 +16,7 @@ import logging from flask.json import jsonify from flask_restful import Resource from common.proto.context_pb2 import SliceStatusEnum -from common.tools.context_queries.Slice import get_slice +from common.tools.context_queries.Slice import get_slice_by_uuid from common.tools.grpc.Tools import grpc_message_to_json from context.client.ContextClient import ContextClient from slice.client.SliceClient import SliceClient @@ -32,7 +32,7 @@ class NSS_Service(Resource): try: context_client = ContextClient() - target = get_slice(context_client, slice_id, rw_copy=True) + target = get_slice_by_uuid(context_client, slice_id, rw_copy=True) if target is None: raise Exception('Slice({:s}) not found in database'.format(str(slice_id))) @@ -56,7 +56,7 @@ class NSS_Service(Resource): LOGGER.debug('DELETE Slice ID: {:s}'.format(str(slice_id))) try: context_client = ContextClient() - target = get_slice(context_client, slice_id) + target = get_slice_by_uuid(context_client, slice_id) response = jsonify({}) response.status_code = HTTP_OK diff --git a/src/context/data/sql_hash_join_full_scan_tests.sql b/src/context/data/sql_hash_join_full_scan_tests.sql new file mode 100644 index 0000000000000000000000000000000000000000..ebead1be6cae62bce06ab9324c0381b9266eee9c --- /dev/null +++ b/src/context/data/sql_hash_join_full_scan_tests.sql @@ -0,0 +1,119 @@ +-- When inserting config rules, for instance related to device +-- If we insert few rules (3~4 rows), does a lookup join with more rows does hash join which is less performant... +-- To be investigated... + +----------------------------------------------------------------------------------------------------- +-- Scenario: tests database with device and device_configrule tables + +CREATE DATABASE tests; +USE tests; + +CREATE TYPE public.orm_deviceoperationalstatusenum AS ENUM ('UNDEFINED', 'DISABLED', 'ENABLED'); +CREATE TYPE public.orm_devicedriverenum AS ENUM ('UNDEFINED', 'OPENCONFIG', 'TRANSPORT_API', 'P4', 'IETF_NETWORK_TOPOLOGY', 'ONF_TR_352', 'XR', 'IETF_L2VPN'); +CREATE TYPE public.configrulekindenum AS ENUM ('CUSTOM', 'ACL'); +CREATE TYPE public.orm_configactionenum AS ENUM ('UNDEFINED', 'SET', 'DELETE'); + +CREATE TABLE public.device ( + device_uuid UUID NOT NULL, + device_name VARCHAR NOT NULL, + device_type VARCHAR NOT NULL, + device_operational_status public.orm_deviceoperationalstatusenum NOT NULL, + device_drivers public.orm_devicedriverenum[] NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + CONSTRAINT device_pkey PRIMARY KEY (device_uuid ASC) +); + +CREATE TABLE public.device_configrule ( + configrule_uuid UUID NOT NULL, + device_uuid UUID NOT NULL, + "position" INT8 NOT NULL, + kind public.configrulekindenum NOT NULL, + action public.orm_configactionenum NOT NULL, + data VARCHAR NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + CONSTRAINT device_configrule_pkey PRIMARY KEY (configrule_uuid ASC), + CONSTRAINT device_configrule_device_uuid_fkey FOREIGN KEY (device_uuid) REFERENCES public.device(device_uuid) ON DELETE CASCADE, + INDEX device_configrule_device_uuid_rec_idx (device_uuid ASC) STORING ("position", kind, action, data, created_at, updated_at), + CONSTRAINT check_position_value CHECK ("position" >= 0:::INT8) +); + +----------------------------------------------------------------------------------------------------- +-- Populate devices + +INSERT INTO device (device_uuid, device_name, device_type, device_operational_status, device_drivers, created_at, updated_at) VALUES +('a3645f8a-5f1f-4d91-8b11-af4104e57f52'::UUID, 'R1', 'router', 'ENABLED', ARRAY['UNDEFINED'], '2023-04-21 07:51:00.0', '2023-04-21 07:51:00.0'), +('7c1e923c-145c-48c5-8016-0d1f596cb4c1'::UUID, 'R2', 'router', 'ENABLED', ARRAY['UNDEFINED'], '2023-04-21 07:51:00.0', '2023-04-21 07:51:00.0') +ON CONFLICT (device_uuid) DO UPDATE SET +device_name=excluded.device_name, +device_operational_status=excluded.device_operational_status, +updated_at=excluded.updated_at +RETURNING device.created_at, device.updated_at; + +----------------------------------------------------------------------------------------------------- +-- Examine insertion of config rules... + +-- Helpful commands: +-- ANALYZE (VERBOSE, TYPES) +-- EXPLAIN (VERBOSE, TYPES) + +-- Rows with realistic data +EXPLAIN (TYPES, VERBOSE) INSERT INTO device_configrule (configrule_uuid, device_uuid, position, kind, action, data, created_at, updated_at) VALUES +('5491b521-76a2-57c4-b622-829131374b4b', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 0, 'CUSTOM', 'SET', '{"resource_key": "_connect/address", "resource_value": "127.0.0.1"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('1f39fb84-2337-5735-a873-2bc7cd50bdd2', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 1, 'CUSTOM', 'SET', '{"resource_key": "_connect/port", "resource_value": "0"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('5d4c72f9-7acc-5ab2-a41e-0d4bc625943e', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 2, 'CUSTOM', 'SET', '{"resource_key": "_connect/settings", "resource_value": "{\\n\\"endpoints\\": [\\n{\\n\\"sample_types\\": [],\\n\\"type\\": \\"copper\\",\\n\\"uuid\\ ... (573 characters truncated) ... "type\\": \\"copper\\",\\n\\"uuid\\": \\"2/5\\"\\n},\\n{\\n\\"sample_types\\": [],\\n\\"type\\": \\"copper\\",\\n\\"uuid\\": \\"2/6\\"\\n}\\n]\\n}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('d14c3fc7-4998-5707-b1c4-073d553a86ef', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 3, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[1/1]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"1/1\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('e3268fba-e695-59d0-b26e-014930f416fd', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 4, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[1/2]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"1/2\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('043eb444-81c8-5ac9-83df-0c6a74bad534', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 5, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[1/3]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"1/3\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('df79d0ea-bcda-57fc-8926-e9a9257628dd', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 6, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[2/1]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"2/1\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('94f41adc-1041-5a8a-81f5-1224d884ae57', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 7, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[2/2]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"2/2\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('b27c0207-dc43-59db-a856-74aaab4f1a19', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 8, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[2/3]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"2/3\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('5902903f-0be4-5ec6-a133-c3e2f9ae05a6', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 9, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[2/4]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"2/4\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('eb20a698-99f2-5369-a228-610cd289297a', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 10, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[2/5]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"2/5\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('3a73b766-195c-59ec-a66e-d32391bc35a3', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 11, 'CUSTOM', 'SET', '{"resource_key": "/endpoints/endpoint[2/6]", "resource_value": "{\\"sample_types\\": {}, \\"type\\": \\"copper\\", \\"uuid\\": \\"2/6\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('353ba35c-5ec6-5f38-8ca9-e6c024772282', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 12, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:126]", "resource_value": "{\\"name\\": \\"ELAN-AC:126\\", \\"type\\": \\"L2VSI\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('45582643-09a1-5ade-beac-2bf057549d38', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 13, 'CUSTOM', 'SET', '{"resource_key": "/interface[2/4.126]/subinterface[0]", "resource_value": "{\\"index\\": 0, \\"name\\": \\"2/4.126\\", \\"type\\": \\"l2vlan\\", \\"vlan_id\\": 26}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('9484afcd-b010-561e-ba83-6b0654d8816c', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 14, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:126]/interface[2/4.126]", "resource_value": "{\\"id\\": \\"2/4.126\\", \\"interface\\": \\"2/4.126\\", \\"name\\": \\"ELAN-AC:126\\", \\"subinterface\\": 0}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('f1cc8161-eaee-5139-a3e5-207fbe11800d', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 15, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:126]/connection_point[VC-1]", "resource_value": "{\\"VC_ID\\": \\"126\\", \\"connection_point\\": \\"VC-1\\", \\"name\\": \\"ELAN-AC:126\\", \\"remote_system\\": \\"10.0.0.6\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('f1836e1d-74a1-51be-944f-a2cedc297812', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 16, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:128]", "resource_value": "{\\"name\\": \\"ELAN-AC:128\\", \\"type\\": \\"L2VSI\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('86f152ea-6abf-5bd4-b2c8-eddfe2826847', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 17, 'CUSTOM', 'SET', '{"resource_key": "/interface[1/2.128]/subinterface[0]", "resource_value": "{\\"index\\": 0, \\"name\\": \\"1/2.128\\", \\"type\\": \\"l2vlan\\", \\"vlan_id\\": 28}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('c36e9d88-0ee3-5826-9a27-3b25a7520121', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 18, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:128]/interface[1/2.128]", "resource_value": "{\\"id\\": \\"1/2.128\\", \\"interface\\": \\"1/2.128\\", \\"name\\": \\"ELAN-AC:128\\", \\"subinterface\\": 0}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('a9d4c170-ca55-5969-8329-5bbbceec5bd6', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 19, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:128]/connection_point[VC-1]", "resource_value": "{\\"VC_ID\\": \\"128\\", \\"connection_point\\": \\"VC-1\\", \\"name\\": \\"ELAN-AC:128\\", \\"remote_system\\": \\"10.0.0.3\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('d86ceb87-e87a-5b1d-b3d9-15af96233500', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 20, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:136]", "resource_value": "{\\"name\\": \\"ELAN-AC:136\\", \\"type\\": \\"L2VSI\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('cef724a0-3a51-5dd7-b9e5-bde174f4a8d2', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 21, 'CUSTOM', 'SET', '{"resource_key": "/interface[2/6.136]/subinterface[0]", "resource_value": "{\\"index\\": 0, \\"name\\": \\"2/6.136\\", \\"type\\": \\"l2vlan\\", \\"vlan_id\\": 36}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('9a01ca29-4ef6-50f3-84f0-a219e7c1689e', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 22, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:136]/interface[2/6.136]", "resource_value": "{\\"id\\": \\"2/6.136\\", \\"interface\\": \\"2/6.136\\", \\"name\\": \\"ELAN-AC:136\\", \\"subinterface\\": 0}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('9212773d-6a4f-5cce-ae5b-85adfdf6674e', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 23, 'CUSTOM', 'SET', '{"resource_key": "/network_instance[ELAN-AC:136]/connection_point[VC-1]", "resource_value": "{\\"VC_ID\\": \\"136\\", \\"connection_point\\": \\"VC-1\\", \\"name\\": \\"ELAN-AC:136\\", \\"remote_system\\": \\"10.0.0.2\\"}"}', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892') +ON CONFLICT (configrule_uuid) DO UPDATE SET position = excluded.position, action = excluded.action, data = excluded.data, updated_at = excluded.updated_at +RETURNING device_configrule.created_at, device_configrule.updated_at; + + +-- Rows with empty data (still does full scan and hash join) +-- If only 3~4 rows are inserted it does lookup join... +EXPLAIN INSERT INTO device_configrule (configrule_uuid, device_uuid, position, kind, action, data, created_at, updated_at) VALUES +('5491b521-76a2-57c4-b622-829131374b4b', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 0, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('1f39fb84-2337-5735-a873-2bc7cd50bdd2', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 1, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('5d4c72f9-7acc-5ab2-a41e-0d4bc625943e', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 2, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('d14c3fc7-4998-5707-b1c4-073d553a86ef', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 3, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('e3268fba-e695-59d0-b26e-014930f416fd', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 4, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('043eb444-81c8-5ac9-83df-0c6a74bad534', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 5, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('df79d0ea-bcda-57fc-8926-e9a9257628dd', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 6, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('94f41adc-1041-5a8a-81f5-1224d884ae57', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 7, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('b27c0207-dc43-59db-a856-74aaab4f1a19', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 8, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('5902903f-0be4-5ec6-a133-c3e2f9ae05a6', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 9, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('eb20a698-99f2-5369-a228-610cd289297a', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 10, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('3a73b766-195c-59ec-a66e-d32391bc35a3', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 11, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('353ba35c-5ec6-5f38-8ca9-e6c024772282', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 12, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('45582643-09a1-5ade-beac-2bf057549d38', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 13, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('9484afcd-b010-561e-ba83-6b0654d8816c', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 14, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('f1cc8161-eaee-5139-a3e5-207fbe11800d', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 15, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('f1836e1d-74a1-51be-944f-a2cedc297812', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 16, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('86f152ea-6abf-5bd4-b2c8-eddfe2826847', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 17, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('c36e9d88-0ee3-5826-9a27-3b25a7520121', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 18, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('a9d4c170-ca55-5969-8329-5bbbceec5bd6', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 19, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('d86ceb87-e87a-5b1d-b3d9-15af96233500', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 20, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('cef724a0-3a51-5dd7-b9e5-bde174f4a8d2', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 21, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('9a01ca29-4ef6-50f3-84f0-a219e7c1689e', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 22, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892'), +('9212773d-6a4f-5cce-ae5b-85adfdf6674e', 'a3645f8a-5f1f-4d91-8b11-af4104e57f52', 23, 'CUSTOM', 'SET', '', '2023-04-20 17:33:54.044892', '2023-04-20 17:33:54.044892') +ON CONFLICT (configrule_uuid) DO UPDATE SET position = excluded.position, action = excluded.action, data = excluded.data, updated_at = excluded.updated_at +RETURNING device_configrule.created_at, device_configrule.updated_at; diff --git a/src/context/service/ContextServiceServicerImpl.py b/src/context/service/ContextServiceServicerImpl.py index 789ee7a78c6bcff3e62a6dd373bd58dbb2e7a960..6d540b4945df8516697c957316294a452186ddb1 100644 --- a/src/context/service/ContextServiceServicerImpl.py +++ b/src/context/service/ContextServiceServicerImpl.py @@ -36,7 +36,7 @@ from .database.EndPoint import endpoint_list_names from .database.Link import link_delete, link_get, link_list_ids, link_list_objs, link_set from .database.PolicyRule import ( policyrule_delete, policyrule_get, policyrule_list_ids, policyrule_list_objs, policyrule_set) -from .database.Service import service_delete, service_get, service_list_ids, service_list_objs, service_select, service_set +from .database.Service import service_delete, service_get, service_list_ids, service_list_objs, service_select, service_set, service_unset from .database.Slice import slice_delete, slice_get, slice_list_ids, slice_list_objs, slice_select, slice_set, slice_unset from .database.Topology import ( topology_delete, topology_get, topology_get_details, topology_list_ids, topology_list_objs, topology_set) @@ -162,7 +162,7 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer return Empty() @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) - def SelectDevices(self, request : DeviceFilter, context : grpc.ServicerContext) -> DeviceList: + def SelectDevice(self, request : DeviceFilter, context : grpc.ServicerContext) -> DeviceList: return DeviceList(devices=device_select(self.db_engine, request)) @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) @@ -231,6 +231,14 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer notify_event(self.messagebroker, TOPIC_SERVICE, event_type, {'service_id': service_id}) return ServiceId(**service_id) + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) + def UnsetService(self, request : Service, context : grpc.ServicerContext) -> ServiceId: + service_id,updated = service_unset(self.db_engine, request) + if updated: + event_type = EventTypeEnum.EVENTTYPE_UPDATE + notify_event(self.messagebroker, TOPIC_SERVICE, event_type, {'service_id': service_id}) + return ServiceId(**service_id) + @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def RemoveService(self, request : ServiceId, context : grpc.ServicerContext) -> Empty: service_id,deleted = service_delete(self.db_engine, request) diff --git a/src/context/service/database/ConfigRule.py b/src/context/service/database/ConfigRule.py index 09723cc6f6b31e2496bf5ab475f50d0aa58f95c2..c5b259a2dfc2ba684f6881dfb2a9a79b3a36032a 100644 --- a/src/context/service/database/ConfigRule.py +++ b/src/context/service/database/ConfigRule.py @@ -21,7 +21,8 @@ from typing import Dict, List, Optional, Set from common.proto.context_pb2 import ConfigRule from common.tools.grpc.Tools import grpc_message_to_json_string from .models.enums.ConfigAction import ORM_ConfigActionEnum, grpc_to_enum__config_action -from .models.ConfigRuleModel import ConfigRuleKindEnum, ConfigRuleModel +from .models.ConfigRuleModel import ( + ConfigRuleKindEnum, DeviceConfigRuleModel, ServiceConfigRuleModel, SliceConfigRuleModel) from .uuids._Builder import get_uuid_from_string from .uuids.EndPoint import endpoint_get_uuid @@ -83,6 +84,16 @@ def upsert_config_rules( session : Session, config_rules : List[Dict], is_delete : bool = False, device_uuid : Optional[str] = None, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None, ) -> bool: + if device_uuid is not None and service_uuid is None and slice_uuid is None: + klass = DeviceConfigRuleModel + elif device_uuid is None and service_uuid is not None and slice_uuid is None: + klass = ServiceConfigRuleModel + elif device_uuid is None and service_uuid is None and slice_uuid is not None: + klass = SliceConfigRuleModel + else: + MSG = 'DataModel cannot be identified (device_uuid={:s}, service_uuid={:s}, slice_uuid={:s})' + raise Exception(MSG.format(str(device_uuid), str(service_uuid), str(slice_uuid))) + uuids_to_delete : Set[str] = set() uuids_to_upsert : Dict[str, int] = dict() rules_to_upsert : List[Dict] = list() @@ -108,11 +119,11 @@ def upsert_config_rules( delete_affected = False if len(uuids_to_delete) > 0: - stmt = delete(ConfigRuleModel) - if device_uuid is not None: stmt = stmt.where(ConfigRuleModel.device_uuid == device_uuid ) - if service_uuid is not None: stmt = stmt.where(ConfigRuleModel.service_uuid == service_uuid) - if slice_uuid is not None: stmt = stmt.where(ConfigRuleModel.slice_uuid == slice_uuid ) - stmt = stmt.where(ConfigRuleModel.configrule_uuid.in_(uuids_to_delete)) + stmt = delete(klass) + if device_uuid is not None: stmt = stmt.where(klass.device_uuid == device_uuid ) + if service_uuid is not None: stmt = stmt.where(klass.service_uuid == service_uuid) + if slice_uuid is not None: stmt = stmt.where(klass.slice_uuid == slice_uuid ) + stmt = stmt.where(klass.configrule_uuid.in_(uuids_to_delete)) #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt))) configrule_deletes = session.execute(stmt) @@ -121,9 +132,9 @@ def upsert_config_rules( upsert_affected = False if len(rules_to_upsert) > 0: - stmt = insert(ConfigRuleModel).values(rules_to_upsert) + stmt = insert(klass).values(rules_to_upsert) stmt = stmt.on_conflict_do_update( - index_elements=[ConfigRuleModel.configrule_uuid], + index_elements=[klass.configrule_uuid], set_=dict( position = stmt.excluded.position, action = stmt.excluded.action, @@ -131,7 +142,7 @@ def upsert_config_rules( updated_at = stmt.excluded.updated_at, ) ) - stmt = stmt.returning(ConfigRuleModel.created_at, ConfigRuleModel.updated_at) + stmt = stmt.returning(klass.created_at, klass.updated_at) #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt))) configrule_updates = session.execute(stmt).fetchall() diff --git a/src/context/service/database/Constraint.py b/src/context/service/database/Constraint.py index 3a73f6589f9332aa4c84f8f296f2cb56db3048bf..592d7f4c545a222092ca95924afafa69d2798d7c 100644 --- a/src/context/service/database/Constraint.py +++ b/src/context/service/database/Constraint.py @@ -20,7 +20,7 @@ from sqlalchemy.orm import Session from typing import Dict, List, Optional from common.proto.context_pb2 import Constraint from common.tools.grpc.Tools import grpc_message_to_json_string -from .models.ConstraintModel import ConstraintKindEnum, ConstraintModel +from .models.ConstraintModel import ConstraintKindEnum, ServiceConstraintModel, SliceConstraintModel from .uuids._Builder import get_uuid_from_string from .uuids.EndPoint import endpoint_get_uuid @@ -84,6 +84,14 @@ def upsert_constraints( session : Session, constraints : List[Dict], is_delete : bool = False, service_uuid : Optional[str] = None, slice_uuid : Optional[str] = None ) -> bool: + if service_uuid is not None and slice_uuid is None: + klass = ServiceConstraintModel + elif service_uuid is None and slice_uuid is not None: + klass = SliceConstraintModel + else: + MSG = 'DataModel cannot be identified (service_uuid={:s}, slice_uuid={:s})' + raise Exception(MSG.format(str(service_uuid), str(slice_uuid))) + uuids_to_upsert : Dict[str, int] = dict() rules_to_upsert : List[Dict] = list() for constraint in constraints: @@ -100,10 +108,10 @@ def upsert_constraints( # Delete all constraints not in uuids_to_upsert delete_affected = False if len(uuids_to_upsert) > 0: - stmt = delete(ConstraintModel) - if service_uuid is not None: stmt = stmt.where(ConstraintModel.service_uuid == service_uuid) - if slice_uuid is not None: stmt = stmt.where(ConstraintModel.slice_uuid == slice_uuid ) - stmt = stmt.where(ConstraintModel.constraint_uuid.not_in(set(uuids_to_upsert.keys()))) + stmt = delete(klass) + if service_uuid is not None: stmt = stmt.where(klass.service_uuid == service_uuid) + if slice_uuid is not None: stmt = stmt.where(klass.slice_uuid == slice_uuid ) + stmt = stmt.where(klass.constraint_uuid.not_in(set(uuids_to_upsert.keys()))) #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) #LOGGER.warning('delete stmt={:s}'.format(str(str_stmt))) constraint_deletes = session.execute(stmt) @@ -112,16 +120,16 @@ def upsert_constraints( upsert_affected = False if not is_delete and len(constraints) > 0: - stmt = insert(ConstraintModel).values(constraints) + stmt = insert(klass).values(constraints) stmt = stmt.on_conflict_do_update( - index_elements=[ConstraintModel.constraint_uuid], + index_elements=[klass.constraint_uuid], set_=dict( position = stmt.excluded.position, data = stmt.excluded.data, updated_at = stmt.excluded.updated_at, ) ) - stmt = stmt.returning(ConstraintModel.created_at, ConstraintModel.updated_at) + stmt = stmt.returning(klass.created_at, klass.updated_at) #str_stmt = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}) #LOGGER.warning('upsert stmt={:s}'.format(str(str_stmt))) constraint_updates = session.execute(stmt).fetchall() diff --git a/src/context/service/database/Service.py b/src/context/service/database/Service.py index 32484a3095c3d937392f580597339fe047d36e3f..b6916dc3a19fef4bde3aff93300e63f360b362c0 100644 --- a/src/context/service/database/Service.py +++ b/src/context/service/database/Service.py @@ -13,11 +13,12 @@ # limitations under the License. import datetime, logging +from sqlalchemy import and_ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, selectinload, sessionmaker from sqlalchemy_cockroachdb import run_transaction -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Set, Tuple from common.proto.context_pb2 import ContextId, Service, ServiceFilter, ServiceId from common.method_wrappers.ServiceExceptions import InvalidArgumentException, NotFoundException from common.tools.object_factory.Context import json_context_id @@ -146,6 +147,45 @@ def service_set(db_engine : Engine, request : Service) -> Tuple[Dict, bool]: updated = run_transaction(sessionmaker(bind=db_engine), callback) return json_service_id(service_uuid, json_context_id(context_uuid)),updated +def service_unset(db_engine : Engine, request : Service) -> Tuple[Dict, bool]: + raw_context_uuid = request.service_id.context_id.context_uuid.uuid + raw_service_uuid = request.service_id.service_uuid.uuid + raw_service_name = request.name + service_name = raw_service_uuid if len(raw_service_name) == 0 else raw_service_name + context_uuid,service_uuid = service_get_uuid(request.service_id, service_name=service_name, allow_random=False) + + service_endpoint_uuids : Set[str] = set() + for i,endpoint_id in enumerate(request.service_endpoint_ids): + endpoint_context_uuid = endpoint_id.topology_id.context_id.context_uuid.uuid + if len(endpoint_context_uuid) == 0: endpoint_context_uuid = context_uuid + if endpoint_context_uuid not in {raw_context_uuid, context_uuid}: + raise InvalidArgumentException( + 'request.service_endpoint_ids[{:d}].topology_id.context_id.context_uuid.uuid'.format(i), + endpoint_context_uuid, + ['should be == request.service_id.context_id.context_uuid.uuid({:s})'.format(raw_context_uuid)]) + service_endpoint_uuids.add(endpoint_get_uuid(endpoint_id, allow_random=False)[2]) + + now = datetime.datetime.utcnow() + constraints = compose_constraints_data(request.service_constraints, now, service_uuid=service_uuid) + config_rules = compose_config_rules_data(request.service_config.config_rules, now, service_uuid=service_uuid) + + def callback(session : Session) -> bool: + num_deletes = 0 + if len(service_endpoint_uuids) > 0: + num_deletes += session.query(ServiceEndPointModel)\ + .filter(and_( + ServiceEndPointModel.service_uuid == service_uuid, + ServiceEndPointModel.endpoint_uuid.in_(service_endpoint_uuids) + )).delete() + + changed_constraints = upsert_constraints(session, constraints, is_delete=True, service_uuid=service_uuid) + changed_config_rules = upsert_config_rules(session, config_rules, is_delete=True, service_uuid=service_uuid) + + return num_deletes > 0 or changed_constraints or changed_config_rules + + updated = run_transaction(sessionmaker(bind=db_engine), callback) + return json_service_id(service_uuid, json_context_id(context_uuid)),updated + def service_delete(db_engine : Engine, request : ServiceId) -> Tuple[Dict, bool]: context_uuid,service_uuid = service_get_uuid(request, allow_random=False) def callback(session : Session) -> bool: diff --git a/src/context/service/database/models/ConfigRuleModel.py b/src/context/service/database/models/ConfigRuleModel.py index d7bb97cd0fec1037e98c8713b885b2d5141cae63..5d14b62a83b71d7a146f74e649435ed941dad6d3 100644 --- a/src/context/service/database/models/ConfigRuleModel.py +++ b/src/context/service/database/models/ConfigRuleModel.py @@ -24,13 +24,11 @@ class ConfigRuleKindEnum(enum.Enum): CUSTOM = 'custom' ACL = 'acl' -class ConfigRuleModel(_Base): - __tablename__ = 'configrule' +class DeviceConfigRuleModel(_Base): + __tablename__ = 'device_configrule' configrule_uuid = Column(UUID(as_uuid=False), primary_key=True) - device_uuid = Column(ForeignKey('device.device_uuid', ondelete='CASCADE'), nullable=True, index=True) - service_uuid = Column(ForeignKey('service.service_uuid', ondelete='CASCADE'), nullable=True, index=True) - slice_uuid = Column(ForeignKey('slice.slice_uuid', ondelete='CASCADE'), nullable=True, index=True) + device_uuid = Column(ForeignKey('device.device_uuid', ondelete='CASCADE'), nullable=False) #, index=True position = Column(Integer, nullable=False) kind = Column(Enum(ConfigRuleKindEnum), nullable=False) action = Column(Enum(ORM_ConfigActionEnum), nullable=False) @@ -41,7 +39,51 @@ class ConfigRuleModel(_Base): __table_args__ = ( CheckConstraint(position >= 0, name='check_position_value'), #UniqueConstraint('device_uuid', 'position', name='unique_per_device' ), + ) + + def dump(self) -> Dict: + return { + 'action': self.action.value, + self.kind.value: json.loads(self.data), + } + +class ServiceConfigRuleModel(_Base): + __tablename__ = 'service_configrule' + + configrule_uuid = Column(UUID(as_uuid=False), primary_key=True) + service_uuid = Column(ForeignKey('service.service_uuid', ondelete='CASCADE'), nullable=False) #, index=True + position = Column(Integer, nullable=False) + kind = Column(Enum(ConfigRuleKindEnum), nullable=False) + action = Column(Enum(ORM_ConfigActionEnum), nullable=False) + data = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + __table_args__ = ( + CheckConstraint(position >= 0, name='check_position_value'), #UniqueConstraint('service_uuid', 'position', name='unique_per_service'), + ) + + def dump(self) -> Dict: + return { + 'action': self.action.value, + self.kind.value: json.loads(self.data), + } + +class SliceConfigRuleModel(_Base): + __tablename__ = 'slice_configrule' + + configrule_uuid = Column(UUID(as_uuid=False), primary_key=True) + slice_uuid = Column(ForeignKey('slice.slice_uuid', ondelete='CASCADE'), nullable=False) #, index=True + position = Column(Integer, nullable=False) + kind = Column(Enum(ConfigRuleKindEnum), nullable=False) + action = Column(Enum(ORM_ConfigActionEnum), nullable=False) + data = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + __table_args__ = ( + CheckConstraint(position >= 0, name='check_position_value'), #UniqueConstraint('slice_uuid', 'position', name='unique_per_slice' ), ) diff --git a/src/context/service/database/models/ConstraintModel.py b/src/context/service/database/models/ConstraintModel.py index 2412080c1a2883e7bed85e6e22f389270b3f73bc..cbbe0b5d7280a6f14d645b66abd4df444abb41aa 100644 --- a/src/context/service/database/models/ConstraintModel.py +++ b/src/context/service/database/models/ConstraintModel.py @@ -31,12 +31,11 @@ class ConstraintKindEnum(enum.Enum): SLA_AVAILABILITY = 'sla_availability' SLA_ISOLATION = 'sla_isolation' -class ConstraintModel(_Base): - __tablename__ = 'constraint' +class ServiceConstraintModel(_Base): + __tablename__ = 'service_constraint' constraint_uuid = Column(UUID(as_uuid=False), primary_key=True) - service_uuid = Column(ForeignKey('service.service_uuid', ondelete='CASCADE'), nullable=True, index=True) - slice_uuid = Column(ForeignKey('slice.slice_uuid', ondelete='CASCADE'), nullable=True, index=True) + service_uuid = Column(ForeignKey('service.service_uuid', ondelete='CASCADE'), nullable=False) #, index=True position = Column(Integer, nullable=False) kind = Column(Enum(ConstraintKindEnum), nullable=False) data = Column(String, nullable=False) @@ -46,6 +45,24 @@ class ConstraintModel(_Base): __table_args__ = ( CheckConstraint(position >= 0, name='check_position_value'), #UniqueConstraint('service_uuid', 'position', name='unique_per_service'), + ) + + def dump(self) -> Dict: + return {self.kind.value: json.loads(self.data)} + +class SliceConstraintModel(_Base): + __tablename__ = 'slice_constraint' + + constraint_uuid = Column(UUID(as_uuid=False), primary_key=True) + slice_uuid = Column(ForeignKey('slice.slice_uuid', ondelete='CASCADE'), nullable=False) #, index=True + position = Column(Integer, nullable=False) + kind = Column(Enum(ConstraintKindEnum), nullable=False) + data = Column(String, nullable=False) + created_at = Column(DateTime, nullable=False) + updated_at = Column(DateTime, nullable=False) + + __table_args__ = ( + CheckConstraint(position >= 0, name='check_position_value'), #UniqueConstraint('slice_uuid', 'position', name='unique_per_slice' ), ) diff --git a/src/context/service/database/models/DeviceModel.py b/src/context/service/database/models/DeviceModel.py index 24130841d2bafde3608f2fa1cbdd476d28acba46..beb500d601aa725c5c0d3c01633aebf31aa23e5b 100644 --- a/src/context/service/database/models/DeviceModel.py +++ b/src/context/service/database/models/DeviceModel.py @@ -33,7 +33,7 @@ class DeviceModel(_Base): updated_at = Column(DateTime, nullable=False) #topology_devices = relationship('TopologyDeviceModel', back_populates='device') - config_rules = relationship('ConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device' + config_rules = relationship('DeviceConfigRuleModel', passive_deletes=True) # lazy='joined', back_populates='device' endpoints = relationship('EndPointModel', passive_deletes=True) # lazy='joined', back_populates='device' def dump_id(self) -> Dict: @@ -48,8 +48,8 @@ class DeviceModel(_Base): for config_rule in sorted(self.config_rules, key=operator.attrgetter('position')) ]} - #def dump_components(self) -> List[Dict]: - # return [] + def dump_components(self) -> List[Dict]: + return [] def dump(self, include_endpoints : bool = True, include_config_rules : bool = True, include_components : bool = True, @@ -63,5 +63,5 @@ class DeviceModel(_Base): } if include_endpoints: result['device_endpoints'] = self.dump_endpoints() if include_config_rules: result['device_config'] = self.dump_config_rules() - #if include_components: result['components'] = self.dump_components() + if include_components: result['component'] = self.dump_components() return result diff --git a/src/context/service/database/models/ServiceModel.py b/src/context/service/database/models/ServiceModel.py index ef6e1b06aaaa616ede6f9633e4e0d7fc0aabf336..2895a7ce9ddfeefd025a9397040a01423c9682a4 100644 --- a/src/context/service/database/models/ServiceModel.py +++ b/src/context/service/database/models/ServiceModel.py @@ -34,8 +34,8 @@ class ServiceModel(_Base): context = relationship('ContextModel', back_populates='services', lazy='selectin') service_endpoints = relationship('ServiceEndPointModel') # lazy='selectin', back_populates='service' - constraints = relationship('ConstraintModel', passive_deletes=True) # lazy='selectin', back_populates='service' - config_rules = relationship('ConfigRuleModel', passive_deletes=True) # lazy='selectin', back_populates='service' + constraints = relationship('ServiceConstraintModel', passive_deletes=True) # lazy='selectin', back_populates='service' + config_rules = relationship('ServiceConfigRuleModel', passive_deletes=True) # lazy='selectin', back_populates='service' def dump_id(self) -> Dict: return { diff --git a/src/context/service/database/models/SliceModel.py b/src/context/service/database/models/SliceModel.py index 423af244e186301cf3132eea3fc7cbea16bf9fe9..d3befa66b4c7ecba4a28fefa745a7c2214f37caf 100644 --- a/src/context/service/database/models/SliceModel.py +++ b/src/context/service/database/models/SliceModel.py @@ -37,8 +37,8 @@ class SliceModel(_Base): slice_services = relationship('SliceServiceModel') # lazy='selectin', back_populates='slice' slice_subslices = relationship( 'SliceSubSliceModel', primaryjoin='slice.c.slice_uuid == slice_subslice.c.slice_uuid') - constraints = relationship('ConstraintModel', passive_deletes=True) # lazy='selectin', back_populates='slice' - config_rules = relationship('ConfigRuleModel', passive_deletes=True) # lazy='selectin', back_populates='slice' + constraints = relationship('SliceConstraintModel', passive_deletes=True) # lazy='selectin', back_populates='slice' + config_rules = relationship('SliceConfigRuleModel', passive_deletes=True) # lazy='selectin', back_populates='slice' def dump_id(self) -> Dict: return { @@ -46,7 +46,6 @@ class SliceModel(_Base): 'slice_uuid': {'uuid': self.slice_uuid}, } - def dump_endpoint_ids(self) -> List[Dict]: return [ slice_endpoint.endpoint.dump_id() diff --git a/src/context/service/database/models/_Base.py b/src/context/service/database/models/_Base.py index a10de60eb8731132ec815de1ff897c06ac12b665..b87b9b06d6adc5825ab5dd84cf64347eb9c26f66 100644 --- a/src/context/service/database/models/_Base.py +++ b/src/context/service/database/models/_Base.py @@ -30,23 +30,23 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: return text(INDEX_STORING.format(index_name, table_name, str_index_fields, str_storing_fields)) statements = [ - index_storing('configrule_device_uuid_rec_idx', 'configrule', ['device_uuid'], [ - 'service_uuid', 'slice_uuid', 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' + index_storing('device_configrule_device_uuid_rec_idx', 'device_configrule', ['device_uuid'], [ + 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' ]), - index_storing('configrule_service_uuid_rec_idx', 'configrule', ['service_uuid'], [ - 'device_uuid', 'slice_uuid', 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' + index_storing('service_configrule_service_uuid_rec_idx', 'service_configrule', ['service_uuid'], [ + 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' ]), - index_storing('configrule_slice_uuid_rec_idx', 'configrule', ['slice_uuid'], [ - 'device_uuid', 'service_uuid', 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' + index_storing('slice_configrule_slice_uuid_rec_idx', 'slice_configrule', ['slice_uuid'], [ + 'position', 'kind', 'action', 'data', 'created_at', 'updated_at' ]), index_storing('connection_service_uuid_rec_idx', 'connection', ['service_uuid'], [ 'settings', 'created_at', 'updated_at' ]), - index_storing('constraint_service_uuid_rec_idx', 'constraint', ['service_uuid'], [ - 'slice_uuid', 'position', 'kind', 'data', 'created_at', 'updated_at' + index_storing('service_constraint_service_uuid_rec_idx', 'service_constraint', ['service_uuid'], [ + 'position', 'kind', 'data', 'created_at', 'updated_at' ]), - index_storing('constraint_slice_uuid_rec_idx', 'constraint', ['slice_uuid'], [ - 'service_uuid', 'position', 'kind', 'data', 'created_at', 'updated_at' + index_storing('slice_constraint_slice_uuid_rec_idx', 'slice_constraint', ['slice_uuid'], [ + 'position', 'kind', 'data', 'created_at', 'updated_at' ]), index_storing('endpoint_device_uuid_rec_idx', 'endpoint', ['device_uuid'], [ 'topology_uuid', 'name', 'endpoint_type', 'kpi_sample_types', 'created_at', 'updated_at' @@ -57,7 +57,6 @@ def create_performance_enhancers(db_engine : sqlalchemy.engine.Engine) -> None: index_storing('slice_context_uuid_rec_idx', 'slice', ['context_uuid'], [ 'slice_name', 'slice_status', 'slice_owner_uuid', 'slice_owner_string', 'created_at', 'updated_at' ]), - index_storing('topology_context_uuid_rec_idx', 'topology', ['context_uuid'], [ 'topology_name', 'created_at', 'updated_at' ]), diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index 205d769acb76992aeba33fc54b7e7b8fbbdc8d06..e7fec041802cc661b14617a8ebfec0864c738b39 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import grpc, logging, time from typing import Dict -import grpc, logging -from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method +from prometheus_client import Histogram +from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, safe_and_metered_rpc_method from common.method_wrappers.ServiceExceptions import NotFoundException, OperationFailedException from common.proto.context_pb2 import ( Device, DeviceConfig, DeviceDriverEnum, DeviceId, DeviceOperationalStatusEnum, Empty, Link) @@ -36,6 +37,10 @@ LOGGER = logging.getLogger(__name__) METRICS_POOL = MetricsPool('Device', 'RPC') +METRICS_POOL_DETAILS = MetricsPool('Device', 'exec_details', labels={ + 'step_name': '', +}) + class DeviceServiceServicerImpl(DeviceServiceServicer): def __init__(self, driver_instance_cache : DriverInstanceCache, monitoring_loops : MonitoringLoops) -> None: LOGGER.debug('Creating Servicer...') @@ -105,7 +110,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): context_client.SetLink(sub_links) # Update endpoint monitoring resources with UUIDs - device_with_uuids = context_client.GetDevice(device_id) + device_with_uuids = get_device( + context_client, device_id.device_uuid.uuid, rw_copy=False, include_endpoints=True, + include_components=False, include_config_rules=False) populate_endpoint_monitoring_resources(device_with_uuids, self.monitoring_loops) context_client.close() @@ -115,29 +122,41 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): @safe_and_metered_rpc_method(METRICS_POOL, LOGGER) def ConfigureDevice(self, request : Device, context : grpc.ServicerContext) -> DeviceId: + t0 = time.time() device_id = request.device_id device_uuid = device_id.device_uuid.uuid self.mutex_queues.wait_my_turn(device_uuid) + t1 = time.time() try: context_client = ContextClient() - device = get_device(context_client, device_uuid, rw_copy=True) + t2 = time.time() + device = get_device( + context_client, device_uuid, rw_copy=True, include_endpoints=False, include_components=False, + include_config_rules=True) if device is None: raise NotFoundException('Device', device_uuid, extra_details='loading in ConfigureDevice') + t3 = time.time() device_controller_uuid = get_device_controller_uuid(device) if device_controller_uuid is not None: - device = get_device(context_client, device_controller_uuid, rw_copy=True) + device = get_device( + context_client, device_controller_uuid, rw_copy=True, include_endpoints=False, + include_components=False, include_config_rules=True) if device is None: raise NotFoundException( 'Device', device_controller_uuid, extra_details='loading in ConfigureDevice') + device_uuid = device.device_id.device_uuid.uuid driver : _Driver = get_driver(self.driver_instance_cache, device) if driver is None: msg = ERROR_MISSING_DRIVER.format(device_uuid=str(device_uuid)) raise OperationFailedException('ConfigureDevice', extra_details=msg) if DeviceDriverEnum.DEVICEDRIVER_P4 in device.device_drivers: + device = get_device( + context_client, device_uuid, rw_copy=False, include_endpoints=True, include_components=False, + include_config_rules=True) # P4 Driver, by now, has no means to retrieve endpoints # We allow defining the endpoints manually update_endpoints(request, device) @@ -149,13 +168,17 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): if request.device_operational_status != DeviceOperationalStatusEnum.DEVICEOPERATIONALSTATUS_UNDEFINED: device.device_operational_status = request.device_operational_status + t4 = time.time() # TODO: use of datastores (might be virtual ones) to enable rollbacks resources_to_set, resources_to_delete = compute_rules_to_add_delete(device, request) + t5 = time.time() errors = [] errors.extend(configure_rules(device, driver, resources_to_set)) + t6 = time.time() errors.extend(deconfigure_rules(device, driver, resources_to_delete)) + t7 = time.time() if len(errors) > 0: for error in errors: LOGGER.error(error) raise OperationFailedException('ConfigureDevice', extra_details=errors) @@ -166,8 +189,23 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): # TODO: Add logic to inspect endpoints and keep only those ones modified with respect to Context. del device.device_endpoints[:] + t8 = time.time() # Note: Rules are updated by configure_rules() and deconfigure_rules() methods. device_id = context_client.SetDevice(device) + + t9 = time.time() + + histogram_duration : Histogram = METRICS_POOL_DETAILS.get_or_create( + 'ConfigureDevice', MetricTypeEnum.HISTOGRAM_DURATION) + histogram_duration.labels(step_name='total' ).observe(t9-t0) + histogram_duration.labels(step_name='wait_queue' ).observe(t1-t0) + histogram_duration.labels(step_name='execution' ).observe(t9-t1) + histogram_duration.labels(step_name='get_device' ).observe(t3-t2) + histogram_duration.labels(step_name='split_rules' ).observe(t5-t4) + histogram_duration.labels(step_name='configure_rules' ).observe(t6-t5) + histogram_duration.labels(step_name='deconfigure_rules').observe(t7-t6) + histogram_duration.labels(step_name='set_device' ).observe(t9-t8) + return device_id finally: self.mutex_queues.signal_done(device_uuid) @@ -179,7 +217,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): self.mutex_queues.wait_my_turn(device_uuid) try: context_client = ContextClient() - device = get_device(context_client, device_uuid, rw_copy=False) + device = get_device( + context_client, device_uuid, rw_copy=False, include_endpoints=False, include_config_rules=False, + include_components=False) if device is None: raise NotFoundException('Device', device_uuid, extra_details='loading in DeleteDevice') device_uuid = device.device_id.device_uuid.uuid @@ -198,7 +238,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): self.mutex_queues.wait_my_turn(device_uuid) try: context_client = ContextClient() - device = get_device(context_client, device_uuid, rw_copy=False) + device = get_device( + context_client, device_uuid, rw_copy=False, include_endpoints=False, include_components=False, + include_config_rules=True) if device is None: raise NotFoundException('Device', device_uuid, extra_details='loading in DeleteDevice') @@ -237,7 +279,9 @@ class DeviceServiceServicerImpl(DeviceServiceServicer): self.mutex_queues.wait_my_turn(device_uuid) try: context_client = ContextClient() - device = get_device(context_client, device_uuid, rw_copy=False) + device = get_device( + context_client, device_uuid, rw_copy=False, include_endpoints=False, include_components=False, + include_config_rules=True) if device is None: raise NotFoundException('Device', device_uuid, extra_details='loading in DeleteDevice') diff --git a/src/device/service/drivers/emulated/EmulatedDriver.py b/src/device/service/drivers/emulated/EmulatedDriver.py index 14925f9f78d143cd998065a43afb624b20c04bfb..2acb288784d6da5b202f14c2534ee1a59486a20e 100644 --- a/src/device/service/drivers/emulated/EmulatedDriver.py +++ b/src/device/service/drivers/emulated/EmulatedDriver.py @@ -19,7 +19,7 @@ from apscheduler.executors.pool import ThreadPoolExecutor from apscheduler.job import Job from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.background import BackgroundScheduler -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.type_checkers.Checkers import chk_float, chk_length, chk_string, chk_type from device.service.driver_api._Driver import _Driver from device.service.driver_api.AnyTreeTools import TreeNode, dump_subtree, get_subnode, set_subnode_value @@ -31,23 +31,7 @@ LOGGER = logging.getLogger(__name__) RE_GET_ENDPOINT_FROM_INTERFACE = re.compile(r'^\/interface\[([^\]]+)\].*') -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0001, 0.00025, 0.00050, 0.00075, - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0, 25.0, 50.0, 75.0, - 100.0, INF -) METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'emulated'}) -METRICS_POOL.get_or_create('GetInitialConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('GetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class EmulatedDriver(_Driver): def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called diff --git a/src/device/service/drivers/openconfig/OpenConfigDriver.py b/src/device/service/drivers/openconfig/OpenConfigDriver.py index 569a0400ee709ebe3b0fa0694dd80801efc4f7fa..2399b9ac01258a21a4da6a9aa0e5bc09ea851951 100644 --- a/src/device/service/drivers/openconfig/OpenConfigDriver.py +++ b/src/device/service/drivers/openconfig/OpenConfigDriver.py @@ -21,7 +21,7 @@ from apscheduler.job import Job from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.background import BackgroundScheduler from ncclient.manager import Manager, connect_ssh -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.tools.client.RetryDecorator import delay_exponential from common.type_checkers.Checkers import chk_length, chk_string, chk_type, chk_float from device.service.driver_api.Exceptions import UnsupportedResourceKeyException @@ -235,23 +235,7 @@ def edit_config( results = [e for _ in resources] # if commit fails, set exception in each resource return results -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0001, 0.00025, 0.00050, 0.00075, - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0, 25.0, 50.0, 75.0, - 100.0, INF -) METRICS_POOL = MetricsPool('Device', 'Driver', labels={'driver': 'openconfig'}) -METRICS_POOL.get_or_create('GetInitialConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('GetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('UnsubscribeState', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class OpenConfigDriver(_Driver): def __init__(self, address : str, port : int, **settings) -> None: # pylint: disable=super-init-not-called diff --git a/src/service/service/ServiceServiceServicerImpl.py b/src/service/service/ServiceServiceServicerImpl.py index 0b2e0760161c109a2ba6a5feecc931e8bcf5c14f..6531376b84732b1ec80e335cfc6cd816be944b0a 100644 --- a/src/service/service/ServiceServiceServicerImpl.py +++ b/src/service/service/ServiceServiceServicerImpl.py @@ -19,12 +19,12 @@ from common.method_wrappers.ServiceExceptions import AlreadyExistsException, Inv from common.proto.context_pb2 import Empty, Service, ServiceId, ServiceStatusEnum, ServiceTypeEnum from common.proto.pathcomp_pb2 import PathCompRequest from common.proto.service_pb2_grpc import ServiceServiceServicer +from common.tools.context_queries.Service import get_service_by_id from common.tools.grpc.Tools import grpc_message_to_json, grpc_message_to_json_string from context.client.ContextClient import ContextClient from pathcomp.frontend.client.PathCompClient import PathCompClient from .service_handler_api.ServiceHandlerFactory import ServiceHandlerFactory from .task_scheduler.TaskScheduler import TasksScheduler -from .tools.ContextGetters import get_service LOGGER = logging.getLogger(__name__) @@ -69,7 +69,9 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): # check that service does not exist context_client = ContextClient() - current_service = get_service(context_client, request.service_id) + current_service = get_service_by_id( + context_client, request.service_id, rw_copy=False, + include_config_rules=False, include_constraints=False, include_endpoint_ids=False) if current_service is not None: context_uuid = request.service_id.context_id.context_uuid.uuid service_uuid = request.service_id.service_uuid.uuid @@ -86,7 +88,9 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): # Set service status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the service is # being modified. context_client = ContextClient() - _service : Optional[Service] = get_service(context_client, request.service_id) + _service : Optional[Service] = get_service_by_id( + context_client, request.service_id, rw_copy=False, + include_config_rules=False, include_constraints=False, include_endpoint_ids=False) service = Service() service.CopyFrom(request if _service is None else _service) if service.service_type == ServiceTypeEnum.SERVICETYPE_UNKNOWN: # pylint: disable=no-member @@ -106,7 +110,11 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): service.service_config.config_rules.add().CopyFrom(config_rule) # pylint: disable=no-member service_id_with_uuids = context_client.SetService(service) - service_with_uuids = context_client.GetService(service_id_with_uuids) + + # PathComp requires endpoints, constraints and config rules + service_with_uuids = get_service_by_id( + context_client, service_id_with_uuids, rw_copy=False, + include_config_rules=True, include_constraints=True, include_endpoint_ids=True) num_disjoint_paths = 0 for constraint in request.service_constraints: @@ -147,10 +155,8 @@ class ServiceServiceServicerImpl(ServiceServiceServicer): # Set service status to "SERVICESTATUS_PENDING_REMOVAL" to ensure rest of components are aware the service is # being modified. - _service : Optional[Service] = get_service(context_client, request) - if _service is None: raise Exception('Service({:s}) not found'.format(grpc_message_to_json_string(request))) - service = Service() - service.CopyFrom(_service) + service : Optional[Service] = get_service_by_id(context_client, request, rw_copy=True) + if service is None: raise Exception('Service({:s}) not found'.format(grpc_message_to_json_string(request))) # pylint: disable=no-member service.service_status.service_status = ServiceStatusEnum.SERVICESTATUS_PENDING_REMOVAL context_client.SetService(service) diff --git a/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py b/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py index 0a2261ceb4e1a63c984cf833121e3a87e13c0e9f..416c10f72fe2199ce241c4d527d9c58ce93d2b44 100644 --- a/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py +++ b/src/service/service/service_handlers/l2nm_emulated/L2NMEmulatedServiceHandler.py @@ -14,7 +14,7 @@ import json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type @@ -26,22 +26,7 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0000, 25.000, 50.0000, 75.000, - 100.0, INF -) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l2nm_emulated'}) -METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L2NMEmulatedServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l2nm_ietfl2vpn/L2NM_IETFL2VPN_ServiceHandler.py b/src/service/service/service_handlers/l2nm_ietfl2vpn/L2NM_IETFL2VPN_ServiceHandler.py index 880a6c5a20d618c9d9f21701b7ef3886dbb9fc21..2e832516b4e8c12028dcf82681406940b248e0f4 100644 --- a/src/service/service/service_handlers/l2nm_ietfl2vpn/L2NM_IETFL2VPN_ServiceHandler.py +++ b/src/service/service/service_handlers/l2nm_ietfl2vpn/L2NM_IETFL2VPN_ServiceHandler.py @@ -26,7 +26,7 @@ from service.service.task_scheduler.TaskExecutor import TaskExecutor LOGGER = logging.getLogger(__name__) -METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'tapi_tapi'}) +METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l2nm_ietf_l2vpn'}) class L2NM_IETFL2VPN_ServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l2nm_openconfig/ConfigRules.py b/src/service/service/service_handlers/l2nm_openconfig/ConfigRules.py index 63e818a8303f9939abe8f776cd34b3066cb84ec1..5afedb33dea6783af9cdb88b86bc186a279de9cc 100644 --- a/src/service/service/service_handlers/l2nm_openconfig/ConfigRules.py +++ b/src/service/service/service_handlers/l2nm_openconfig/ConfigRules.py @@ -98,22 +98,22 @@ def teardown_config_rules( json_config_rules = [ - json_config_rule_delete( - '/network_instance[{:s}]/connection_point[{:s}]'.format(network_instance_name, connection_point_id), - {'name': network_instance_name, 'connection_point': connection_point_id, 'VC_ID': circuit_id}), + #json_config_rule_delete( + # '/network_instance[{:s}]/connection_point[{:s}]'.format(network_instance_name, connection_point_id), + # {'name': network_instance_name, 'connection_point': connection_point_id, 'VC_ID': circuit_id}), + + #json_config_rule_delete( + # '/network_instance[{:s}]/interface[{:s}]'.format(network_instance_name, if_cirid_name), + # {'name': network_instance_name, 'id': if_cirid_name, 'interface': if_cirid_name, + # 'subinterface': sub_interface_index}), json_config_rule_delete( - '/network_instance[{:s}]/interface[{:s}]'.format(network_instance_name, if_cirid_name), - {'name': network_instance_name, 'id': if_cirid_name, 'interface': if_cirid_name, - 'subinterface': sub_interface_index}), + '/network_instance[{:s}]'.format(network_instance_name), + {'name': network_instance_name}), json_config_rule_delete( '/interface[{:s}]/subinterface[{:d}]'.format(if_cirid_name, sub_interface_index), {'name': if_cirid_name, 'index': sub_interface_index}), - json_config_rule_delete( - '/network_instance[{:s}]'.format(network_instance_name), - {'name': network_instance_name}), - ] return json_config_rules diff --git a/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py index d511c8947ecb43052fd154ab3ce3293a468b4263..aae9e968b44af52170fdf6f6ecfab76fe90e2b52 100644 --- a/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l2nm_openconfig/L2NMOpenConfigServiceHandler.py @@ -14,7 +14,7 @@ import json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type @@ -26,22 +26,7 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0000, 25.000, 50.0000, 75.000, - 100.0, INF -) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l2nm_openconfig'}) -METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L2NMOpenConfigServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py b/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py index 18da03b08c0ea490b60c41df3894392022ddf228..de02a43caffd91ae047bd73d319e969af6265c5c 100644 --- a/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py @@ -14,7 +14,7 @@ import json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type @@ -26,22 +26,7 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0000, 25.000, 50.0000, 75.000, - 100.0, INF -) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l3nm_emulated'}) -METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L3NMEmulatedServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py index b2639ddad58e4c453f1b1e2dc87fce8861ad79a2..b14a005e12947cc99b4d46ad0c58c9aae5778d05 100644 --- a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py +++ b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py @@ -14,7 +14,7 @@ import json, logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.Device import json_device_id from common.type_checkers.Checkers import chk_type @@ -26,22 +26,7 @@ from .ConfigRules import setup_config_rules, teardown_config_rules LOGGER = logging.getLogger(__name__) -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0000, 25.000, 50.0000, 75.000, - 100.0, INF -) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'l3nm_openconfig'}) -METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) class L3NMOpenConfigServiceHandler(_ServiceHandler): def __init__( # pylint: disable=super-init-not-called diff --git a/src/service/service/service_handlers/p4/p4_service_handler.py b/src/service/service/service_handlers/p4/p4_service_handler.py index 8d609c11c9c1c4f25c0d387290c11de36af69a9a..41cfcc5952601a16a13cd691f2e424017936aaa3 100644 --- a/src/service/service/service_handlers/p4/p4_service_handler.py +++ b/src/service/service/service_handlers/p4/p4_service_handler.py @@ -18,7 +18,7 @@ P4 service handler for the TeraFlowSDN controller. import logging from typing import Any, List, Optional, Tuple, Union -from common.method_wrappers.Decorator import MetricTypeEnum, MetricsPool, metered_subclass_method, INF +from common.method_wrappers.Decorator import MetricsPool, metered_subclass_method from common.proto.context_pb2 import ConfigRule, DeviceId, Service from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set from common.tools.object_factory.Device import json_device_id @@ -28,22 +28,7 @@ from service.service.task_scheduler.TaskExecutor import TaskExecutor LOGGER = logging.getLogger(__name__) -HISTOGRAM_BUCKETS = ( - # .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, INF - 0.0010, 0.0025, 0.0050, 0.0075, - 0.0100, 0.0250, 0.0500, 0.0750, - 0.1000, 0.2500, 0.5000, 0.7500, - 1.0000, 2.5000, 5.0000, 7.5000, - 10.0000, 25.000, 50.0000, 75.000, - 100.0, INF -) METRICS_POOL = MetricsPool('Service', 'Handler', labels={'handler': 'p4'}) -METRICS_POOL.get_or_create('SetEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteEndpoint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConstraint', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('SetConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) -METRICS_POOL.get_or_create('DeleteConfig', MetricTypeEnum.HISTOGRAM_DURATION, buckets=HISTOGRAM_BUCKETS) def create_rule_set(endpoint_a, endpoint_b): return json_config_rule_set( diff --git a/src/service/service/task_scheduler/TaskExecutor.py b/src/service/service/task_scheduler/TaskExecutor.py index 3d157e3145d4b195c0251a4ab79f710a38726569..96751e83770e1b98df4770cf74bb453f6a0519ef 100644 --- a/src/service/service/task_scheduler/TaskExecutor.py +++ b/src/service/service/task_scheduler/TaskExecutor.py @@ -17,11 +17,13 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Dict, Optional, Union from common.method_wrappers.ServiceExceptions import NotFoundException from common.proto.context_pb2 import Connection, ConnectionId, Device, DeviceId, Service, ServiceId +from common.tools.context_queries.Connection import get_connection_by_id +from common.tools.context_queries.Device import get_device +from common.tools.context_queries.Service import get_service_by_id from common.tools.object_factory.Device import json_device_id from context.client.ContextClient import ContextClient from device.client.DeviceClient import DeviceClient from service.service.service_handler_api.ServiceHandlerFactory import ServiceHandlerFactory, get_service_handler_class -from service.service.tools.ContextGetters import get_connection, get_device, get_service from service.service.tools.ObjectKeys import get_connection_key, get_device_key, get_service_key if TYPE_CHECKING: @@ -72,7 +74,7 @@ class TaskExecutor: connection_key = get_connection_key(connection_id) connection = self._load_grpc_object(CacheableObjectType.CONNECTION, connection_key) if connection is None: - connection = get_connection(self._context_client, connection_id) + connection = get_connection_by_id(self._context_client, connection_id) if connection is None: raise NotFoundException('Connection', connection_key) connection : Connection = self._store_editable_grpc_object( CacheableObjectType.CONNECTION, connection_key, Connection, connection) @@ -94,7 +96,7 @@ class TaskExecutor: device_key = get_device_key(device_id) device = self._load_grpc_object(CacheableObjectType.DEVICE, device_key) if device is None: - device = get_device(self._context_client, device_id) + device = get_device(self._context_client, device_id.device_uuid.uuid) if device is None: raise NotFoundException('Device', device_key) device : Device = self._store_editable_grpc_object( CacheableObjectType.DEVICE, device_key, Device, device) @@ -145,7 +147,7 @@ class TaskExecutor: service_key = get_service_key(service_id) service = self._load_grpc_object(CacheableObjectType.SERVICE, service_key) if service is None: - service = get_service(self._context_client, service_id) + service = get_service_by_id(self._context_client, service_id) if service is None: raise NotFoundException('Service', service_key) service : service = self._store_editable_grpc_object( CacheableObjectType.SERVICE, service_key, Service, service) diff --git a/src/service/service/tools/ContextGetters.py b/src/service/service/tools/ContextGetters.py deleted file mode 100644 index 9b1d6224d1e4201cbc0720e7ce818a86e5ae2042..0000000000000000000000000000000000000000 --- a/src/service/service/tools/ContextGetters.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import grpc -from typing import Optional -from common.proto.context_pb2 import Connection, ConnectionId, Device, DeviceId, Service, ServiceId -from context.client.ContextClient import ContextClient - -def get_connection(context_client : ContextClient, connection_id : ConnectionId) -> Optional[Connection]: - try: - connection : Connection = context_client.GetConnection(connection_id) - return connection - except grpc.RpcError as e: - if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member - return None - -def get_device(context_client : ContextClient, device_id : DeviceId) -> Optional[Device]: - try: - device : Device = context_client.GetDevice(device_id) - return device - except grpc.RpcError as e: - if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member - return None - -def get_service(context_client : ContextClient, service_id : ServiceId) -> Optional[Service]: - try: - service : Service = context_client.GetService(service_id) - return service - except grpc.RpcError as e: - if e.code() != grpc.StatusCode.NOT_FOUND: raise # pylint: disable=no-member - return None diff --git a/src/slice/service/SliceServiceServicerImpl.py b/src/slice/service/SliceServiceServicerImpl.py index acec3ae303266714ae7f50c5c0d78fc41d350ea1..f91c55e281e8ed5f994dea3dce43a63184669795 100644 --- a/src/slice/service/SliceServiceServicerImpl.py +++ b/src/slice/service/SliceServiceServicerImpl.py @@ -19,7 +19,7 @@ from common.proto.context_pb2 import ( from common.proto.slice_pb2_grpc import SliceServiceServicer from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method from common.tools.context_queries.InterDomain import is_multi_domain -from common.tools.context_queries.Slice import get_slice +from common.tools.context_queries.Slice import get_slice_by_id from common.tools.grpc.ConfigRules import copy_config_rules from common.tools.grpc.Constraints import copy_constraints from common.tools.grpc.EndPointIds import copy_endpoint_ids @@ -44,9 +44,7 @@ class SliceServiceServicerImpl(SliceServiceServicer): # Set slice status to "SERVICESTATUS_PLANNED" to ensure rest of components are aware the slice is # being modified. context_client = ContextClient() - slice_ro : Optional[Service] = get_slice( - context_client, request.slice_id.slice_uuid.uuid, request.slice_id.context_id.context_uuid.uuid, - rw_copy=False) + slice_ro : Optional[Slice] = get_slice_by_id(context_client, request.slice_id, rw_copy=False) slice_rw = Slice() slice_rw.CopyFrom(request if slice_ro is None else slice_ro) diff --git a/src/slice/service/slice_grouper/Tools.py b/src/slice/service/slice_grouper/Tools.py index ca957f3c7760eb65b649d22ecb5b57dee3e08dab..c815a19d5477ec82c2c2702ba58bb5b092144692 100644 --- a/src/slice/service/slice_grouper/Tools.py +++ b/src/slice/service/slice_grouper/Tools.py @@ -18,7 +18,7 @@ from common.Settings import get_setting from common.method_wrappers.ServiceExceptions import NotFoundException from common.proto.context_pb2 import IsolationLevelEnum, Slice, SliceId, SliceStatusEnum from common.tools.context_queries.Context import create_context -from common.tools.context_queries.Slice import get_slice +from common.tools.context_queries.Slice import get_slice_by_uuid from context.client.ContextClient import ContextClient from slice.service.slice_grouper.MetricsExporter import MetricsExporter @@ -70,7 +70,7 @@ def create_slice_groups( slice_group_ids : Dict[str, SliceId] = dict() for slice_group in slice_groups: slice_group_name = slice_group[0] - slice_group_obj = get_slice(context_client, slice_group_name, DEFAULT_CONTEXT_NAME) + slice_group_obj = get_slice_by_uuid(context_client, slice_group_name, DEFAULT_CONTEXT_NAME) if slice_group_obj is None: slice_group_obj = create_slice_group( DEFAULT_CONTEXT_NAME, slice_group_name, slice_group[2], slice_group[1]) @@ -111,7 +111,7 @@ def add_slice_to_group(slice_obj : Slice, selected_group : Tuple[str, float, flo slice_uuid = slice_obj.slice_id.slice_uuid.uuid context_client = ContextClient() - slice_group_obj = get_slice(context_client, group_name, DEFAULT_CONTEXT_NAME, rw_copy=True) + slice_group_obj = get_slice_by_uuid(context_client, group_name, DEFAULT_CONTEXT_NAME, rw_copy=True) if slice_group_obj is None: raise NotFoundException('Slice', group_name, extra_details='while adding to group') @@ -148,7 +148,7 @@ def remove_slice_from_group(slice_obj : Slice, selected_group : Tuple[str, float slice_uuid = slice_obj.slice_id.slice_uuid.uuid context_client = ContextClient() - slice_group_obj = get_slice(context_client, group_name, DEFAULT_CONTEXT_NAME, rw_copy=True) + slice_group_obj = get_slice_by_uuid(context_client, group_name, DEFAULT_CONTEXT_NAME, rw_copy=True) if slice_group_obj is None: raise NotFoundException('Slice', group_name, extra_details='while removing from group') diff --git a/src/webui/service/service/routes.py b/src/webui/service/service/routes.py index 70a5b5bad41df6520cb2facdad94cfee04f726cd..08312e5257d13c4b55b83733ded689c7565c4790 100644 --- a/src/webui/service/service/routes.py +++ b/src/webui/service/service/routes.py @@ -18,17 +18,18 @@ import grpc from collections import defaultdict from flask import current_app, redirect, render_template, Blueprint, flash, session, url_for, request from common.proto.context_pb2 import ( - IsolationLevelEnum, Service, ServiceId, ServiceTypeEnum, ServiceStatusEnum, Connection, Empty, DeviceDriverEnum, ConfigActionEnum, Device, DeviceList) + IsolationLevelEnum, Service, ServiceId, ServiceTypeEnum, ServiceStatusEnum, Connection, Empty, DeviceDriverEnum, + ConfigActionEnum, Device, DeviceList) from common.tools.context_queries.Context import get_context from common.tools.context_queries.Topology import get_topology from common.tools.context_queries.EndPoint import get_endpoint_names -from common.tools.context_queries.Service import get_service +from common.tools.context_queries.Service import get_service_by_uuid +from common.tools.object_factory.ConfigRule import json_config_rule_set +from common.tools.object_factory.Context import json_context_id +from common.tools.object_factory.Topology import json_topology_id from context.client.ContextClient import ContextClient from service.client.ServiceClient import ServiceClient from typing import Optional, Set -from common.tools.object_factory.Topology import json_topology_id -from common.tools.object_factory.ConfigRule import json_config_rule_set -from common.tools.object_factory.Context import json_context_id service = Blueprint('service', __name__, url_prefix='/service') @@ -254,7 +255,7 @@ def detail(service_uuid: str): context_client.connect() endpoint_ids = list() - service_obj = get_service(context_client, service_uuid, rw_copy=False) + service_obj = get_service_by_uuid(context_client, service_uuid, rw_copy=False) if service_obj is None: flash('Context({:s})/Service({:s}) not found'.format(str(context_uuid), str(service_uuid)), 'danger') service_obj = Service() diff --git a/src/webui/service/slice/routes.py b/src/webui/service/slice/routes.py index cd1b672d5c1014b0e8aa301ed7b5a1f6d910f6df..a66b316b8f1e2e3266ce37fa8f55f4f8a1042ca1 100644 --- a/src/webui/service/slice/routes.py +++ b/src/webui/service/slice/routes.py @@ -17,7 +17,7 @@ from flask import current_app, redirect, render_template, Blueprint, flash, sess from common.proto.context_pb2 import IsolationLevelEnum, Slice, SliceId, SliceStatusEnum from common.tools.context_queries.Context import get_context from common.tools.context_queries.EndPoint import get_endpoint_names -from common.tools.context_queries.Slice import get_slice +from common.tools.context_queries.Slice import get_slice_by_uuid from context.client.ContextClient import ContextClient from slice.client.SliceClient import SliceClient @@ -76,7 +76,7 @@ def detail(slice_uuid: str): try: context_client.connect() - slice_obj = get_slice(context_client, slice_uuid, rw_copy=False) + slice_obj = get_slice_by_uuid(context_client, slice_uuid, rw_copy=False) if slice_obj is None: flash('Context({:s})/Slice({:s}) not found'.format(str(context_uuid), str(slice_uuid)), 'danger') slice_obj = Slice()