From 2f9064818d99e3b473364351b809d1f7e73f5b38 Mon Sep 17 00:00:00 2001
From: Lluis Gifre <lluis.gifre@cttc.es>
Date: Tue, 15 Feb 2022 15:28:27 +0000
Subject: [PATCH] Fix multiple issues related to integration of components and
cleanup
---
manifests/computeservice.yaml | 16 -------
manifests/contextservice.yaml | 18 --------
manifests/dbscanservingservice.yaml | 17 --------
manifests/monitoringservice.yaml | 1 -
manifests/opticalattackmitigatorservice.yaml | 17 --------
...ticalcentralizedattackdetectorservice.yaml | 17 --------
manifests/webuiservice.yaml | 23 ----------
src/common/tests/EventTools.py | 42 +++++++++++++------
src/context/client/EventsCollector.py | 19 +++++----
.../Config.py | 2 +-
.../L3NMOpenConfigServiceHandler.py | 4 ++
src/tests/ofc22/expose_services.yaml | 41 ++++++++++++++++++
src/tests/ofc22/tests/Objects.py | 2 +-
.../ofc22/tests/test_functional_bootstrap.py | 12 +++---
14 files changed, 93 insertions(+), 138 deletions(-)
diff --git a/manifests/computeservice.yaml b/manifests/computeservice.yaml
index 16507dfcb..e9fbe72d8 100644
--- a/manifests/computeservice.yaml
+++ b/manifests/computeservice.yaml
@@ -53,19 +53,3 @@ spec:
protocol: TCP
port: 9090
targetPort: 9090
----
-apiVersion: v1
-kind: Service
-metadata:
- name: computeservice-public
-spec:
- type: NodePort
- selector:
- app: computeservice
- ports:
- - name: http
- protocol: TCP
- port: 8080
- targetPort: 8080
- nodePort: 30808
----
diff --git a/manifests/contextservice.yaml b/manifests/contextservice.yaml
index c8fcb38f7..e93fdbb12 100644
--- a/manifests/contextservice.yaml
+++ b/manifests/contextservice.yaml
@@ -73,21 +73,3 @@ spec:
protocol: TCP
port: 8080
targetPort: 8080
----
-apiVersion: v1
-kind: Service
-metadata:
- name: contextservice-public
- labels:
- app: contextservice
-spec:
- type: NodePort
- selector:
- app: contextservice
- ports:
- - name: http
- protocol: TCP
- port: 8080
- targetPort: 8080
- nodePort: 31808
----
diff --git a/manifests/dbscanservingservice.yaml b/manifests/dbscanservingservice.yaml
index ca8982db2..32b22d00f 100644
--- a/manifests/dbscanservingservice.yaml
+++ b/manifests/dbscanservingservice.yaml
@@ -47,20 +47,3 @@ spec:
- name: grpc
port: 10006
targetPort: 10006
----
-apiVersion: v1
-kind: Service
-metadata:
- name: dbscanservingservice-public
- labels:
- app: dbscanservingservice
-spec:
- type: NodePort
- selector:
- app: dbscanservingservice
- ports:
- - name: http
- protocol: TCP
- port: 10006
- targetPort: 10006
----
diff --git a/manifests/monitoringservice.yaml b/manifests/monitoringservice.yaml
index a908d7d64..047a73a6c 100644
--- a/manifests/monitoringservice.yaml
+++ b/manifests/monitoringservice.yaml
@@ -74,4 +74,3 @@ spec:
protocol: TCP
port: 8086
targetPort: 8086
----
diff --git a/manifests/opticalattackmitigatorservice.yaml b/manifests/opticalattackmitigatorservice.yaml
index 1cc03ba05..5f676d09f 100644
--- a/manifests/opticalattackmitigatorservice.yaml
+++ b/manifests/opticalattackmitigatorservice.yaml
@@ -47,20 +47,3 @@ spec:
- name: grpc
port: 10007
targetPort: 10007
----
-apiVersion: v1
-kind: Service
-metadata:
- name: opticalattackmitigatorservice-public
- labels:
- app: opticalattackmitigatorservice
-spec:
- type: NodePort
- selector:
- app: opticalattackmitigatorservice
- ports:
- - name: http
- protocol: TCP
- port: 10007
- targetPort: 10007
----
diff --git a/manifests/opticalcentralizedattackdetectorservice.yaml b/manifests/opticalcentralizedattackdetectorservice.yaml
index d3ceb3ea4..08e9b10c2 100644
--- a/manifests/opticalcentralizedattackdetectorservice.yaml
+++ b/manifests/opticalcentralizedattackdetectorservice.yaml
@@ -47,20 +47,3 @@ spec:
- name: grpc
port: 10005
targetPort: 10005
----
-apiVersion: v1
-kind: Service
-metadata:
- name: opticalcentralizedattackdetectorservice-public
- labels:
- app: opticalcentralizedattackdetectorservice
-spec:
- type: NodePort
- selector:
- app: opticalcentralizedattackdetectorservice
- ports:
- - name: grpc
- protocol: TCP
- port: 10005
- targetPort: 10005
----
diff --git a/manifests/webuiservice.yaml b/manifests/webuiservice.yaml
index 4e71d43ef..c467a5cf2 100644
--- a/manifests/webuiservice.yaml
+++ b/manifests/webuiservice.yaml
@@ -89,26 +89,3 @@ spec:
- name: http
port: 8004
targetPort: 8004
----
-apiVersion: v1
-kind: Service
-metadata:
- name: webuiservice-public
- labels:
- app: webuiservice
-spec:
- type: NodePort
- selector:
- app: webuiservice
- ports:
- - name: http
- protocol: TCP
- port: 8004
- targetPort: 8004
- nodePort: 30800
- - name: grafana
- protocol: TCP
- port: 3000
- targetPort: 3000
- nodePort: 30300
----
diff --git a/src/common/tests/EventTools.py b/src/common/tests/EventTools.py
index c263be309..bb300774b 100644
--- a/src/common/tests/EventTools.py
+++ b/src/common/tests/EventTools.py
@@ -36,34 +36,52 @@ def event_to_key(event):
entity_id = entity_id_selector_function(event)
return (event_class_name, event.event.event_type, grpc_message_to_json_string(entity_id))
-def check_events(events_collector : EventsCollector, expected_events : List[Tuple[str, int, Dict]]):
+def check_events(
+ events_collector : EventsCollector, expected_events : List[Tuple[str, int, Dict]],
+ fail_if_missing_events : bool = True, fail_if_unexpected_events : bool = False,
+ timeout_per_event = 1.0, max_wait_time = 30.0
+) -> None:
expected_events_map = {}
- num_events_expected = 0
+ num_expected_events = 0
for event_classname, event_type_id, event_ids in expected_events:
event_key = (event_classname, event_type_id, json.dumps(event_ids, sort_keys=True))
event_count = expected_events_map.get(event_key, 0)
expected_events_map[event_key] = event_count + 1
- num_events_expected += 1
+ num_expected_events += 1
- events_received = events_collector.get_events(block=True, count=num_events_expected)
- for i, event_received in enumerate(events_received):
+ i, wait_time = 0, 0
+ while num_expected_events > 0:
+ event_received = events_collector.get_event(block=True, timeout=timeout_per_event)
+ if event_received is None:
+ wait_time += timeout_per_event
+ if wait_time > max_wait_time: break
+ continue
LOGGER.info('event_received[{:d}] = {:s}'.format(i, str(event_received)))
event_key = event_to_key(event_received)
event_count = expected_events_map.pop(event_key, 0)
+ if event_count > 0: num_expected_events -= 1
event_count -= 1
if event_count != 0: expected_events_map[event_key] = event_count
- if len(expected_events_map) > 0:
+ if len(expected_events_map) == 0:
+ LOGGER.info('EventsCheck passed')
+ else:
missing_events = {}
unexpected_events = {}
for event_key,event_count in expected_events_map.items():
if event_count > 0:
- if (event_key[0] == 'DeviceEvent') and (event_key[1] == EVENT_UPDATE):
- continue # sometimes device update events, specially with massive port updates, might be lost
missing_events[event_key] = event_count
if event_count < 0:
unexpected_events[event_key] = -event_count
- msg = ['EventCheck failed:']
- if len(missing_events) > 0: msg.append('missing_events={:s}'.format(str(missing_events)))
- if len(unexpected_events) > 0: msg.append('unexpected_events={:s}'.format(str(unexpected_events)))
- if len(msg) > 1: raise Exception(' '.join(msg))
+ msg_except = ['EventCheck failed:']
+ msg_logger = ['EventCheck:']
+ if len(missing_events) > 0:
+ msg = 'missing_events={:s}'.format(str(missing_events))
+ if fail_if_missing_events: msg_except.append(msg)
+ msg_logger.append(msg)
+ if len(unexpected_events) > 0:
+ msg = 'unexpected_events={:s}'.format(str(unexpected_events))
+ if fail_if_unexpected_events: msg_except.append(msg)
+ msg_logger.append(msg)
+ if len(msg_logger) > 1: LOGGER.warning(' '.join(msg_logger))
+ if len(msg_except) > 1: raise Exception(' '.join(msg_except))
diff --git a/src/context/client/EventsCollector.py b/src/context/client/EventsCollector.py
index 3f0ce45fd..b58e8504b 100644
--- a/src/context/client/EventsCollector.py
+++ b/src/context/client/EventsCollector.py
@@ -46,22 +46,23 @@ class EventsCollector:
self._connection_thread.start()
def get_event(self, block : bool = True, timeout : float = 0.1):
- return self._events_queue.get(block=block, timeout=timeout)
+ try:
+ return self._events_queue.get(block=block, timeout=timeout)
+ except queue.Empty: # pylint: disable=catching-non-exception
+ return None
def get_events(self, block : bool = True, timeout : float = 0.1, count : int = None):
events = []
if count is None:
while True:
- try:
- events.append(self.get_event(block=block, timeout=timeout))
- except queue.Empty: # pylint: disable=catching-non-exception
- break
+ event = self.get_event(block=block, timeout=timeout)
+ if event is None: break
+ events.append(event)
else:
for _ in range(count):
- try:
- events.append(self.get_event(block=block, timeout=timeout))
- except queue.Empty: # pylint: disable=catching-non-exception
- pass
+ event = self.get_event(block=block, timeout=timeout)
+ if event is None: continue
+ events.append(event)
return sorted(events, key=lambda e: e.event.timestamp)
def stop(self):
diff --git a/src/opticalcentralizedattackdetector/Config.py b/src/opticalcentralizedattackdetector/Config.py
index 5a6b7aa56..41d802dd4 100644
--- a/src/opticalcentralizedattackdetector/Config.py
+++ b/src/opticalcentralizedattackdetector/Config.py
@@ -11,7 +11,7 @@ GRPC_GRACE_PERIOD = 60
# service settings
MONITORING_INTERVAL = 2 # monitoring interval in seconds
#TODO: adjust the addresses below for the specific case
-MONITORING_SERVICE_ADDRESS = 'monitoring' # address/name of the monitoring service
+MONITORING_SERVICE_ADDRESS = 'monitoringservice' # address/name of the monitoring service
# MONITORING_SERVICE_ADDRESS = '10.99.41.20' # address/name of the monitoring service
CONTEXT_SERVICE_ADDRESS = 'contextservice' # address/name of the monitoring service
# CONTEXT_SERVICE_ADDRESS = '10.107.199.65' # address/name of the monitoring service
diff --git a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py
index eb1914fc3..894f68b3e 100644
--- a/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py
+++ b/src/service/service/service_handlers/l3nm_openconfig/L3NMOpenConfigServiceHandler.py
@@ -147,6 +147,10 @@ class L3NMOpenConfigServiceHandler(_ServiceHandler):
'/interface[{:s}]/subinterface[{:d}]'.format(endpoint_uuid, sub_interface_index), {
'name': endpoint_uuid, 'index': sub_interface_index,
}),
+ config_rule_delete(
+ '/interface[{:s}]'.format(endpoint_uuid), {
+ 'name': endpoint_uuid,
+ }),
config_rule_delete(
'/network_instance[{:s}]'.format(network_instance_name), {
'name': network_instance_name
diff --git a/src/tests/ofc22/expose_services.yaml b/src/tests/ofc22/expose_services.yaml
index 4f553e2d6..0bd049ef8 100644
--- a/src/tests/ofc22/expose_services.yaml
+++ b/src/tests/ofc22/expose_services.yaml
@@ -20,6 +20,11 @@ spec:
port: 6379
targetPort: 6379
nodePort: 30637
+ - name: http
+ protocol: TCP
+ port: 8080
+ targetPort: 8080
+ nodePort: 31808
---
apiVersion: v1
kind: Service
@@ -55,3 +60,39 @@ spec:
targetPort: 8086
nodePort: 30886
---
+apiVersion: v1
+kind: Service
+metadata:
+ name: computeservice-public
+spec:
+ type: NodePort
+ selector:
+ app: computeservice
+ ports:
+ - name: http
+ protocol: TCP
+ port: 8080
+ targetPort: 8080
+ nodePort: 30808
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: webuiservice-public
+ labels:
+ app: webuiservice
+spec:
+ type: NodePort
+ selector:
+ app: webuiservice
+ ports:
+ - name: http
+ protocol: TCP
+ port: 8004
+ targetPort: 8004
+ nodePort: 30800
+ - name: grafana
+ protocol: TCP
+ port: 3000
+ targetPort: 3000
+ nodePort: 30300
diff --git a/src/tests/ofc22/tests/Objects.py b/src/tests/ofc22/tests/Objects.py
index 00b4f4f1f..fe20575a7 100644
--- a/src/tests/ofc22/tests/Objects.py
+++ b/src/tests/ofc22/tests/Objects.py
@@ -47,7 +47,7 @@ except ImportError:
DEVICE_O1_ADDRESS = '0.0.0.0'
DEVICE_O1_PORT = 4900
-#USE_REAL_DEVICES = False # Force to use emulated devices
+#USE_REAL_DEVICES = False # Uncomment to force to use emulated devices
def json_endpoint_ids(device_id : Dict, endpoint_descriptors : List[Tuple[str, str, List[int]]]):
return [
diff --git a/src/tests/ofc22/tests/test_functional_bootstrap.py b/src/tests/ofc22/tests/test_functional_bootstrap.py
index d1ef2f6d8..a65b60378 100644
--- a/src/tests/ofc22/tests/test_functional_bootstrap.py
+++ b/src/tests/ofc22/tests/test_functional_bootstrap.py
@@ -115,14 +115,14 @@ def test_devices_bootstraping(
expected_events.extend([
# Device creation, update for automation to start the device
('DeviceEvent', EVENT_CREATE, json_device_id(device_uuid)),
- ('DeviceEvent', EVENT_UPDATE, json_device_id(device_uuid)),
+ #('DeviceEvent', EVENT_UPDATE, json_device_id(device_uuid)),
])
- response = context_client.GetDevice(response)
- for endpoint in response.device_endpoints:
- for _ in endpoint.kpi_sample_types:
- # Monitoring configures monitoring for endpoint
- expected_events.append(('DeviceEvent', EVENT_UPDATE, json_device_id(device_uuid)))
+ #response = context_client.GetDevice(response)
+ #for endpoint in response.device_endpoints:
+ # for _ in endpoint.kpi_sample_types:
+ # # Monitoring configures monitoring for endpoint
+ # expected_events.append(('DeviceEvent', EVENT_UPDATE, json_device_id(device_uuid)))
# ----- Validate Collected Events ----------------------------------------------------------------------------------
check_events(events_collector, expected_events)
--
GitLab