diff --git a/clean_testing_environment.sh b/clean_testing_environment.sh new file mode 100755 index 0000000000000000000000000000000000000000..2cec5c5a4d290bf6fd5f314c4f37d2d7db0232ae --- /dev/null +++ b/clean_testing_environment.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +find . -iname __pycache__ | xargs -r rm -rf +find . -iname .benchmarks | xargs -r rm -rf +find . -iname .pytest_cache | xargs -r rm -rf +find . -iname .coverage | xargs -r rm -rf diff --git a/coverage/.coveragerc b/coverage/.coveragerc new file mode 100644 index 0000000000000000000000000000000000000000..e5e634c2c256103b1796d9309a3433ae9f248e70 --- /dev/null +++ b/coverage/.coveragerc @@ -0,0 +1,18 @@ +[run] +data_file = ~/teraflow/controller/coverage/.coverage +source = . +omit = + */proto/* + */__main__.py + +[report] +exclude_lines = + pragma: no cover + if\ TYPE\_CHECKING\: + raise\ NotImplementedError + +[html] +directory = ~/teraflow/controller/coverage/html_report + +[xml] +output = ~/teraflow/controller/coverage/report.xml diff --git a/data/topo_nsfnet.json b/data/topo_nsfnet.json new file mode 100644 index 0000000000000000000000000000000000000000..f2cbb53a8f3f9648c4a8b60f97c2212485ac5cc9 --- /dev/null +++ b/data/topo_nsfnet.json @@ -0,0 +1,136 @@ +{ + "topoId": { + "contextId": {"contextUuid": {"uuid": "default"}}, + "topoId": {"uuid": "topo0-nsfnet"} + }, + "device" : [ + {"device_id": {"device_id": {"uuid": "1" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "1" }}, "port_id": {"uuid" : "101"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "2" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "2" }}, "port_id": {"uuid" : "201"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "3" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "3" }}, "port_id": {"uuid" : "301"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "4" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "4" }}, "port_id": {"uuid" : "401"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "5" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "5" }}, "port_id": {"uuid" : "501"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "6" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "6" }}, "port_id": {"uuid" : "601"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "7" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "7" }}, "port_id": {"uuid" : "701"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "8" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "8" }}, "port_id": {"uuid" : "801"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "9" }}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "9" }}, "port_id": {"uuid" : "901"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "10"}}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "10"}}, "port_id": {"uuid" : "1001"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "11"}}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "11"}}, "port_id": {"uuid" : "1101"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "12"}}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "12"}}, "port_id": {"uuid" : "1201"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "13"}}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "13"}}, "port_id": {"uuid" : "1301"}}, "port_type": "LINE"} + ]}, + {"device_id": {"device_id": {"uuid": "14"}}, "device_type": "ROADM", "device_config": {"device_config": ""}, "devOperationalStatus": 1, "endpointList" : [ + {"port_id": {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "14"}}, "port_id": {"uuid" : "1401"}}, "port_type": "LINE"} + ]} + ], + "link" : [ + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "1" }}, "port_id": {"uuid": "101" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "2" }}, "port_id": {"uuid": "201" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "1" }}, "port_id": {"uuid": "101" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "3" }}, "port_id": {"uuid": "301" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "1" }}, "port_id": {"uuid": "101" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "8" }}, "port_id": {"uuid": "801" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "2" }}, "port_id": {"uuid": "201" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "3" }}, "port_id": {"uuid": "301" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "2" }}, "port_id": {"uuid": "201" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "4" }}, "port_id": {"uuid": "401" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "3" }}, "port_id": {"uuid": "301" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "6" }}, "port_id": {"uuid": "601" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "4" }}, "port_id": {"uuid": "401" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "5" }}, "port_id": {"uuid": "501" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "4" }}, "port_id": {"uuid": "401" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "11"}}, "port_id": {"uuid": "1101"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "5" }}, "port_id": {"uuid": "501" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "6" }}, "port_id": {"uuid": "601" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "5" }}, "port_id": {"uuid": "501" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "7" }}, "port_id": {"uuid": "701" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "6" }}, "port_id": {"uuid": "601" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "10"}}, "port_id": {"uuid": "1001"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "6" }}, "port_id": {"uuid": "601" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "13"}}, "port_id": {"uuid": "1301"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "7" }}, "port_id": {"uuid": "701" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "8" }}, "port_id": {"uuid": "801" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "8" }}, "port_id": {"uuid": "801" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "9" }}, "port_id": {"uuid": "901" }} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "9" }}, "port_id": {"uuid": "901" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "10"}}, "port_id": {"uuid": "1001"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "9" }}, "port_id": {"uuid": "901" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "12"}}, "port_id": {"uuid": "1201"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "9" }}, "port_id": {"uuid": "901" }}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "14"}}, "port_id": {"uuid": "1401"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "11"}}, "port_id": {"uuid": "1101"}}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "12"}}, "port_id": {"uuid": "1201"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "11"}}, "port_id": {"uuid": "1101"}}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "14"}}, "port_id": {"uuid": "1401"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "12"}}, "port_id": {"uuid": "1201"}}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "13"}}, "port_id": {"uuid": "1301"}} + ]}, + {"endpointList" : [ + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "13"}}, "port_id": {"uuid": "1301"}}, + {"topoId": {"topoId": {"uuid": "topo0-nsfnet"}}, "dev_id": {"device_id": {"uuid": "14"}}, "port_id": {"uuid": "1401"}} + ]} + ] +} diff --git a/install_development_dependencies.sh b/install_development_dependencies.sh new file mode 100755 index 0000000000000000000000000000000000000000..b4c1b4777505c6053234ec5d1343c5743cb9152a --- /dev/null +++ b/install_development_dependencies.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pip install --upgrade pip setuptools wheel pip-tools pylint pytest pytest-benchmark coverage diff --git a/proto/context.proto b/proto/context.proto index ca6b17a92e602a39fe3f1adc6793f71b3c587da0..e578f60fbaa78620abb09e81ad7fad2afaf39c07 100644 --- a/proto/context.proto +++ b/proto/context.proto @@ -1,11 +1,12 @@ -//Example of topology syntax = "proto3"; package context; service ContextService { rpc GetTopology (Empty) returns (Topology) {} - + + rpc AddLink(Link) returns (LinkId) {} + rpc DeleteLink(LinkId) returns (Empty) {} } message Empty { @@ -29,7 +30,8 @@ message Topology { } message Link { - repeated EndPointId endpointList = 1; + LinkId link_id = 1; + repeated EndPointId endpointList = 2; } message TopologyId { @@ -69,13 +71,18 @@ message DeviceId { Uuid device_id = 1; } +message LinkId { + Uuid link_id = 1; +} + message Uuid { string uuid = 1; } enum DeviceOperationalStatus { - DISABLED = 0; - ENABLED = 1; + KEEP_STATUS = 0; // Do not change operational status of device (used in configure) + DISABLED = -1; + ENABLED = 1; } message TeraFlowController { diff --git a/proto/device.proto b/proto/device.proto index a277e8fdee63b24e8d4ab155b0d0701662e9dff4..4fe74b78afd3790e392c1df4df66d409316dda05 100644 --- a/proto/device.proto +++ b/proto/device.proto @@ -1,4 +1,3 @@ -//Example of topology syntax = "proto3"; package device; @@ -6,8 +5,6 @@ import "context.proto"; service DeviceService { rpc AddDevice(context.Device) returns (context.DeviceId) {} - rpc ConfigureDevice(context.DeviceConfig) returns (context.DeviceId) {} + rpc ConfigureDevice(context.Device) returns (context.DeviceId) {} rpc DeleteDevice(context.DeviceId) returns (context.Empty) {} } - - diff --git a/report_coverage.sh b/report_coverage.sh new file mode 100755 index 0000000000000000000000000000000000000000..752f1383da444eca42b91e1301f4abf0402b7e70 --- /dev/null +++ b/report_coverage.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +cd $(dirname $0)/src +RCFILE=~/teraflow/controller/coverage/.coveragerc + +echo +echo "Coverage report:" +echo "----------------" +coverage report --rcfile=$RCFILE --skip-covered --sort cover --show-missing +#coverage html --rcfile=$RCFILE +#coverage xml --rcfile=$RCFILE diff --git a/report_coverage_device.sh b/report_coverage_device.sh new file mode 100755 index 0000000000000000000000000000000000000000..f884fb1c7806069412e29fcb11ba278974520c35 --- /dev/null +++ b/report_coverage_device.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +./report_coverage.sh | grep --color -E -i "^.*device.*$|$" diff --git a/run_unitary_tests.sh b/run_unitary_tests.sh new file mode 100755 index 0000000000000000000000000000000000000000..1305084800620e0e97f9f85849917540178b5368 --- /dev/null +++ b/run_unitary_tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +cd $(dirname $0)/src +RCFILE=~/teraflow/controller/coverage/.coveragerc + +# Run unitary tests and analyze coverage of code at same time +coverage run --rcfile=$RCFILE -m pytest --log-level=DEBUG --verbose \ + common/database/tests/test_unitary_inmemory.py \ + device/tests/test_unitary.py + +## Run integration tests and analyze coverage of code at same time +#coverage run --rcfile=$RCFILE --append -m pytest --log-level=WARN --verbose \ +# common/database/tests/test_integration_redis.py \ +# device/tests/test_integration.py diff --git a/src/common/Checkers.py b/src/common/Checkers.py new file mode 100644 index 0000000000000000000000000000000000000000..4ac047a1c6d9078b66d70e6a97755cd3ac6349cf --- /dev/null +++ b/src/common/Checkers.py @@ -0,0 +1,53 @@ +from typing import Any, Set, Union + +def chk_none(name : str, value : Any) -> Any: + if value is not None: + msg = '{}({}) is not None.' + raise AttributeError(msg.format(str(name), str(value))) + return value + +def chk_not_none(name : str, value : Any) -> Any: + if value is None: + msg = '{}({}) is None.' + raise AttributeError(msg.format(str(name), str(value))) + return value + +def chk_type(name : str, value : Any, type_or_types : Union[type, Set[type]] = set()) -> Any: + if not isinstance(value, type_or_types): + msg = '{}({}) is of a wrong type({}). Accepted type_or_types({}).' + raise AttributeError(msg.format(str(name), str(value), type(value).__name__, str(type_or_types))) + return value + +def chk_string(name, value, allow_empty=False) -> str: + chk_not_none(name, value) + chk_type(name, value, str) + if allow_empty: return value + if len(value) == 0: + msg = '{}({}) string is empty.' + raise AttributeError(msg.format(str(name), str(value))) + return value + +def chk_float(name, value, type_or_types=(int, float), min_value=None, max_value=None) -> float: + chk_not_none(name, value) + chk_type(name, value, type_or_types) + if min_value is not None: + chk_type(name, value, type_or_types) + if value < min_value: + msg = '{}({}) lower than min_value({}).' + raise AttributeError(msg.format(str(name), str(value), str(min_value))) + if max_value is not None: + chk_type(name, value, type_or_types) + if value > max_value: + msg = '{}({}) greater than max_value({}).' + raise AttributeError(msg.format(str(name), str(value), str(max_value))) + return value + +def chk_integer(name, value, min_value=None, max_value=None) -> int: + return int(chk_float(name, value, type_or_types=int, min_value=min_value, max_value=max_value)) + +def chk_options(name, value, options): + chk_not_none(name, value) + if value not in options: + msg = '{}({}) is not one of options({}).' + raise AttributeError(msg.format(str(name), str(value), str(options))) + return value diff --git a/src/common/database/api/context/Context.py b/src/common/database/api/context/Context.py index 6da44e4644cc4a11bf68272be02033eb588e2b72..dfd3a5957b105824d75ae3d087304b96e8b3ae34 100644 --- a/src/common/database/api/context/Context.py +++ b/src/common/database/api/context/Context.py @@ -40,6 +40,7 @@ class Context(_Entity): def delete(self): for topology_uuid in self.topologies.get(): self.topology(topology_uuid).delete() + self.attributes.delete() def dump(self) -> Dict: return {topology_uuid : self.topology(topology_uuid).dump() for topology_uuid in self.topologies.get()} diff --git a/src/common/database/api/context/Device.py b/src/common/database/api/context/Device.py index e9fb5d65d0f66c7132a66e3e7a91d615c215215f..5771c7fe937cd0d0cf60959d4c0e4f42e9c299af 100644 --- a/src/common/database/api/context/Device.py +++ b/src/common/database/api/context/Device.py @@ -20,6 +20,7 @@ VALIDATORS = { TRANSCODERS = { 'device_operational_status': { OperationalStatus: lambda v: v.value, + int : lambda v: to_operationalstatus_enum(v), str : lambda v: to_operationalstatus_enum(v), } } @@ -56,11 +57,11 @@ class Device(_Entity): def endpoint(self, endpoint_uuid : str) -> Endpoint: return Endpoint(endpoint_uuid, self) - def create(self, type : str, config : str, operational_status : OperationalStatus) -> 'Device': + def create(self, device_type : str, device_config : str, device_operational_status : OperationalStatus) -> 'Device': self.update(update_attributes={ - 'device_type': type, - 'device_config': config, - 'device_operational_status': operational_status, + 'device_type': device_type, + 'device_config': device_config, + 'device_operational_status': device_operational_status, }) self.parent.devices.add(self.device_uuid) return self @@ -71,8 +72,7 @@ class Device(_Entity): def delete(self) -> None: for endpoint_uuid in self.endpoints.get(): self.endpoint(endpoint_uuid).delete() - remove_attributes = ['device_type', 'device_config', 'device_operational_status'] - self.update(remove_attributes=remove_attributes) + self.attributes.delete() self.parent.devices.delete(self.device_uuid) def dump(self) -> Dict: diff --git a/src/common/database/api/context/Endpoint.py b/src/common/database/api/context/Endpoint.py index 7f27c71e465bd6499f3b35ae5c0a6beafa58f773..8f165848d93c20c19ab49bd43ad14c4dd3395220 100644 --- a/src/common/database/api/context/Endpoint.py +++ b/src/common/database/api/context/Endpoint.py @@ -42,9 +42,9 @@ class Endpoint(_Entity): @property def attributes(self) -> EntityAttributes: return self._attributes - def create(self, type : str) -> 'Endpoint': + def create(self, port_type : str) -> 'Endpoint': self.update(update_attributes={ - 'port_type': type + 'port_type': port_type }) self.parent.endpoints.add(self._endpoint_uuid) return self @@ -54,8 +54,7 @@ class Endpoint(_Entity): return self def delete(self) -> None: - remove_attributes = ['port_type'] - self.update(remove_attributes=remove_attributes) + self.attributes.delete() self.parent.endpoints.delete(self._endpoint_uuid) def dump_uuid(self) -> Dict: diff --git a/src/common/database/api/context/LinkEndpoint.py b/src/common/database/api/context/LinkEndpoint.py index 0e53e26a74efd0a799b943988512facac92c408b..3607ba5edc262afeb028f2c153cafc400b05a14f 100644 --- a/src/common/database/api/context/LinkEndpoint.py +++ b/src/common/database/api/context/LinkEndpoint.py @@ -57,8 +57,7 @@ class LinkEndpoint(_Entity): return self def delete(self) -> None: - remove_attributes = ['device_uuid', 'endpoint_uuid'] - self.update(remove_attributes=remove_attributes) + self.attributes.delete() self.parent.endpoints.delete(self.link_endpoint_uuid) def dump(self) -> Dict: diff --git a/src/common/database/api/context/OperationalStatus.py b/src/common/database/api/context/OperationalStatus.py index 268c7f35961cdd1fc47b9a7bbbcdb7b51d2308c0..c726b32ef5b03feb8c0a04a586cf7ef4cd250263 100644 --- a/src/common/database/api/context/OperationalStatus.py +++ b/src/common/database/api/context/OperationalStatus.py @@ -1,18 +1,27 @@ from enum import Enum class OperationalStatus(Enum): + KEEP_STATE = 0 # Do not change operational status of device (used in configure) + DISABLED = -1 ENABLED = 1 - DISABLED = 0 -TO_ENUM = { - 1: OperationalStatus.ENABLED, - 0: OperationalStatus.DISABLED, - '1': OperationalStatus.ENABLED, - '0': OperationalStatus.DISABLED, +ANY_TO_ENUM = { + 1: OperationalStatus.ENABLED, + 0: OperationalStatus.KEEP_STATE, + -1: OperationalStatus.DISABLED, + + '1': OperationalStatus.ENABLED, + '0': OperationalStatus.KEEP_STATE, + '-1': OperationalStatus.DISABLED, + 'enabled': OperationalStatus.ENABLED, 'disabled': OperationalStatus.DISABLED, + 'keep_state': OperationalStatus.KEEP_STATE, } +def operationalstatus_enum_values(): + return {m.value for m in OperationalStatus.__members__.values()} + def to_operationalstatus_enum(int_or_str): if isinstance(int_or_str, str): int_or_str = int_or_str.lower() - return TO_ENUM.get(int_or_str) + return ANY_TO_ENUM.get(int_or_str) diff --git a/src/common/database/api/context/Topology.py b/src/common/database/api/context/Topology.py index b4a2e46ad65e5de8bfb62f21f438f606c767ce2e..c95d300781f1b6b2633adae9c3512e3fe878dd5d 100644 --- a/src/common/database/api/context/Topology.py +++ b/src/common/database/api/context/Topology.py @@ -53,6 +53,7 @@ class Topology(_Entity): def delete(self): for device_uuid in self.devices.get(): self.device(device_uuid).delete() for link_uuid in self.links.get(): self.link(link_uuid).delete() + self.attributes.delete() self.parent.topologies.delete(self.topology_uuid) def dump(self) -> Dict: diff --git a/src/common/database/api/entity/EntityAttributes.py b/src/common/database/api/entity/EntityAttributes.py index 3572a44021890aabfb018f99f920ede325108874..244b43bc711cfa67067de47866cee302cfc9138e 100644 --- a/src/common/database/api/entity/EntityAttributes.py +++ b/src/common/database/api/entity/EntityAttributes.py @@ -19,6 +19,7 @@ class EntityAttributes: remove_attributes.discard(attribute_name) value = update_attributes.pop(attribute_name, None) validator = self._validators.get(attribute_name) + if validator is None: return if not validator(value): raise AttributeError('{} is invalid'.format(attribute_name)) def transcode(self, attribute_name, attribute_value): diff --git a/src/common/database/engines/redis/RedisTools.py b/src/common/database/engines/redis/RedisTools.py deleted file mode 100644 index 96e11c6174d92fdd70c08fe185bbf4c30152f2d2..0000000000000000000000000000000000000000 --- a/src/common/database/engines/redis/RedisTools.py +++ /dev/null @@ -1,19 +0,0 @@ -from logging import Logger -from redis.client import Redis - -def dump_keys(client : Redis, logger : Logger) -> None: - logger.info('Dump keys...') - keys = client.keys() - logger.info(' keys = {}'.format(str(keys))) - for key_name in keys: - key_name = key_name.decode('UTF-8') - if not key_name.startswith('context'): continue - key_type = client.type(key_name) - if key_type is not None: key_type = key_type.decode('UTF-8') - key_content = { - 'hash' : lambda key: {k.decode('UTF-8'):v.decode('UTF-8') for k,v in client.hgetall(key).items()}, - 'list' : lambda key: [m.decode('UTF-8') for m in client.lrange(key, 0, -1)], - 'set' : lambda key: {m.decode('UTF-8') for m in client.smembers(key)}, - 'string': lambda key: client.get(key).decode('UTF-8'), - }.get(key_type, lambda key: 'UNSUPPORTED_TYPE') - logger.info(' key {} {}: {}'.format(key_type, key_name, key_content(key_name))) diff --git a/src/common/database/tests/script.py b/src/common/database/tests/script.py index 16ee24d426d47da26bdd77c999d446b3b04e43ec..ca133220176a096de04ffeeb1331a98cab47ca4f 100644 --- a/src/common/database/tests/script.py +++ b/src/common/database/tests/script.py @@ -11,25 +11,29 @@ def sequence(database : Database): context = database.context('ctx-test').create() topology = context.topology('base-topo').create() - device_1 = topology.device('dev1').create(type='ROADM', config='<config/>', operational_status=OperationalStatus.ENABLED) - endpoint_dev1_to_dev2 = device_1.endpoint('to-dev2').create(type='WDM') - endpoint_dev1_to_dev3 = device_1.endpoint('to-dev3').create(type='WDM') - endpoint_dev1_to_dev4 = device_1.endpoint('to-dev4').create(type='WDM') - - device_2 = topology.device('dev2').create(type='ROADM', config='<config/>', operational_status=OperationalStatus.ENABLED) - endpoint_dev2_to_dev1 = device_2.endpoint('to-dev1').create(type='WDM') - endpoint_dev2_to_dev3 = device_2.endpoint('to-dev3').create(type='WDM') - endpoint_dev2_to_dev4 = device_2.endpoint('to-dev4').create(type='WDM') - - device_3 = topology.device('dev3').create(type='ROADM', config='<config/>', operational_status=OperationalStatus.ENABLED) - endpoint_dev3_to_dev1 = device_3.endpoint('to-dev1').create(type='WDM') - endpoint_dev3_to_dev2 = device_3.endpoint('to-dev2').create(type='WDM') - endpoint_dev3_to_dev4 = device_3.endpoint('to-dev4').create(type='WDM') - - device_4 = topology.device('dev4').create(type='ROADM', config='<config/>', operational_status=OperationalStatus.ENABLED) - endpoint_dev4_to_dev1 = device_4.endpoint('to-dev1').create(type='WDM') - endpoint_dev4_to_dev2 = device_4.endpoint('to-dev2').create(type='WDM') - endpoint_dev4_to_dev3 = device_4.endpoint('to-dev3').create(type='WDM') + device_1 = topology.device('dev1').create( + device_type='ROADM', device_config='<config/>', device_operational_status=OperationalStatus.ENABLED) + endpoint_dev1_to_dev2 = device_1.endpoint('to-dev2').create(port_type='WDM') + endpoint_dev1_to_dev3 = device_1.endpoint('to-dev3').create(port_type='WDM') + endpoint_dev1_to_dev4 = device_1.endpoint('to-dev4').create(port_type='WDM') + + device_2 = topology.device('dev2').create( + device_type='ROADM', device_config='<config/>', device_operational_status=OperationalStatus.ENABLED) + endpoint_dev2_to_dev1 = device_2.endpoint('to-dev1').create(port_type='WDM') + endpoint_dev2_to_dev3 = device_2.endpoint('to-dev3').create(port_type='WDM') + endpoint_dev2_to_dev4 = device_2.endpoint('to-dev4').create(port_type='WDM') + + device_3 = topology.device('dev3').create( + device_type='ROADM', device_config='<config/>', device_operational_status=OperationalStatus.ENABLED) + endpoint_dev3_to_dev1 = device_3.endpoint('to-dev1').create(port_type='WDM') + endpoint_dev3_to_dev2 = device_3.endpoint('to-dev2').create(port_type='WDM') + endpoint_dev3_to_dev4 = device_3.endpoint('to-dev4').create(port_type='WDM') + + device_4 = topology.device('dev4').create( + device_type='ROADM', device_config='<config/>', device_operational_status=OperationalStatus.ENABLED) + endpoint_dev4_to_dev1 = device_4.endpoint('to-dev1').create(port_type='WDM') + endpoint_dev4_to_dev2 = device_4.endpoint('to-dev2').create(port_type='WDM') + endpoint_dev4_to_dev3 = device_4.endpoint('to-dev3').create(port_type='WDM') link_dev1_to_dev2 = topology.link('dev1/to-dev2 ==> dev2/to-dev1').create() link_dev1_to_dev2.endpoint('dev1/to-dev2').create(endpoint_dev1_to_dev2) @@ -86,9 +90,9 @@ def sequence(database : Database): with database: t0 = time.time() context = database.context('ctx-test').create() - json_topology = context.topology('base-topo').dump() + json_context = context.dump() t1 = time.time() - LOGGER.info(json.dumps(json_topology)) + LOGGER.info(json.dumps(json_context)) LOGGER.info('Dump elapsed: {}'.format(1000.0 * (t1-t0))) with database: diff --git a/src/common/database/tests/test_redis.py b/src/common/database/tests/test_integration_redis.py similarity index 100% rename from src/common/database/tests/test_redis.py rename to src/common/database/tests/test_integration_redis.py diff --git a/src/common/database/tests/test_inmemory.py b/src/common/database/tests/test_unitary_inmemory.py similarity index 100% rename from src/common/database/tests/test_inmemory.py rename to src/common/database/tests/test_unitary_inmemory.py diff --git a/src/common/tests/Assertions.py b/src/common/tests/Assertions.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f88e4aa2aac645f8cd84a7ee3912071cd2606b --- /dev/null +++ b/src/common/tests/Assertions.py @@ -0,0 +1,24 @@ +def validate_empty(message): + assert type(message) is dict + assert len(message.keys()) == 0 + +def validate_uuid(message, allow_empty=False): + assert type(message) is dict + assert len(message.keys()) == 1 + assert 'uuid' in message + assert type(message['uuid']) is str + if allow_empty: return + assert len(message['uuid']) > 1 + +def validate_device_id(message): + assert type(message) is dict + assert len(message.keys()) == 1 + assert 'device_id' in message + validate_uuid(message['device_id']) + +def validate_topology(message): + assert type(message) is dict + assert len(message.keys()) > 0 + assert 'topoId' in message + assert 'device' in message + assert 'link' in message diff --git a/src/device/tests/tools/__init__.py b/src/common/tests/__init__.py similarity index 100% rename from src/device/tests/tools/__init__.py rename to src/common/tests/__init__.py diff --git a/src/context/tests/test_unitary.py b/src/context/tests/test_unitary.py index 61e580ea704260cd034273e2bd74ae9fbbd606e6..cc22243f09d5fff8ad8ffd089a4fc0b86113b0b4 100644 --- a/src/context/tests/test_unitary.py +++ b/src/context/tests/test_unitary.py @@ -4,8 +4,8 @@ from pathlib import Path sys.path.append(__file__.split('src')[0] + 'src') print(sys.path) +from common.database.Factory import get_database, DatabaseEngineEnum from context.client.ContextClient import ContextClient -from context.database.Factory import get_database, DatabaseEngineEnum from context.proto.context_pb2 import Empty from context.service.ContextService import ContextService from context.Config import SERVICE_PORT, MAX_WORKERS, GRACE_PERIOD diff --git a/src/device/client/DeviceClient.py b/src/device/client/DeviceClient.py index 30b6a53f54c067e5eed8d31f8ca3cf5720152db1..3e9f83f3f5459a738f7863955b78164c81ec21a3 100644 --- a/src/device/client/DeviceClient.py +++ b/src/device/client/DeviceClient.py @@ -1,5 +1,4 @@ import grpc, logging -from google.protobuf.json_format import MessageToDict from common.tools.RetryDecorator import retry, delay_exponential from device.proto.device_pb2_grpc import DeviceServiceStub @@ -30,24 +29,18 @@ class DeviceClient: LOGGER.debug('AddDevice request: {}'.format(request)) response = self.stub.AddDevice(request) LOGGER.debug('AddDevice result: {}'.format(response)) - return MessageToDict( - response, including_default_value_fields=True, preserving_proto_field_name=True, - use_integers_for_enums=False) + return response @retry(exceptions=set(), max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') def ConfigureDevice(self, request): LOGGER.debug('ConfigureDevice request: {}'.format(request)) response = self.stub.ConfigureDevice(request) LOGGER.debug('ConfigureDevice result: {}'.format(response)) - return MessageToDict( - response, including_default_value_fields=True, preserving_proto_field_name=True, - use_integers_for_enums=False) + return response @retry(exceptions=set(), max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect') def DeleteDevice(self, request): LOGGER.debug('DeleteDevice request: {}'.format(request)) response = self.stub.DeleteDevice(request) LOGGER.debug('DeleteDevice result: {}'.format(response)) - return MessageToDict( - response, including_default_value_fields=True, preserving_proto_field_name=True, - use_integers_for_enums=False) + return response diff --git a/src/device/genproto.sh b/src/device/genproto.sh index 72fc512ba9056fd5a1cc1da472f48bab7d802e1a..725f2f95fa4d15b7fa7a5a001131010057dca129 100755 --- a/src/device/genproto.sh +++ b/src/device/genproto.sh @@ -19,13 +19,15 @@ # Make folder containing the script the root folder for its execution cd $(dirname $0) -rm -rf proto/*.py +rm -rf proto/*.py proto/*.proto +rm -rf proto/__pycache__ touch proto/__init__.py python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=proto context.proto python -m grpc_tools.protoc -I../../proto --python_out=proto --grpc_python_out=proto device.proto -sed -i -E 's/(import\ .*)_pb2/from device.proto \1_pb2/g' proto/context_pb2.py -sed -i -E 's/(import\ .*)_pb2/from device.proto \1_pb2/g' proto/context_pb2_grpc.py -sed -i -E 's/(import\ .*)_pb2/from device.proto \1_pb2/g' proto/device_pb2.py -sed -i -E 's/(import\ .*)_pb2/from device.proto \1_pb2/g' proto/device_pb2_grpc.py +rm proto/context_pb2_grpc.py + +sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/context_pb2.py +sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/device_pb2.py +sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' proto/device_pb2_grpc.py diff --git a/src/device/proto/context_pb2.py b/src/device/proto/context_pb2.py index e4acb11a579694017d1ee5572f1f94848731802a..99b6c9defbafb9989bedbb800a5e1ee66c15385b 100644 --- a/src/device/proto/context_pb2.py +++ b/src/device/proto/context_pb2.py @@ -20,7 +20,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\rcontext.proto\x12\x07\x63ontext\"\x07\n\x05\x45mpty\"{\n\x07\x43ontext\x12%\n\tcontextId\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x1f\n\x04topo\x18\x02 \x01(\x0b\x32\x11.context.Topology\x12(\n\x03\x63tl\x18\x03 \x01(\x0b\x32\x1b.context.TeraFlowController\"/\n\tContextId\x12\"\n\x0b\x63ontextUuid\x18\x01 \x01(\x0b\x32\r.context.Uuid\"m\n\x08Topology\x12#\n\x06topoId\x18\x02 \x01(\x0b\x32\x13.context.TopologyId\x12\x1f\n\x06\x64\x65vice\x18\x03 \x03(\x0b\x32\x0f.context.Device\x12\x1b\n\x04link\x18\x04 \x03(\x0b\x32\r.context.Link\"1\n\x04Link\x12)\n\x0c\x65ndpointList\x18\x01 \x03(\x0b\x32\x13.context.EndPointId\"R\n\nTopologyId\x12%\n\tcontextId\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x1d\n\x06topoId\x18\x02 \x01(\x0b\x32\r.context.Uuid\"?\n\nConstraint\x12\x17\n\x0f\x63onstraint_type\x18\x01 \x01(\t\x12\x18\n\x10\x63onstraint_value\x18\x02 \x01(\t\"\xda\x01\n\x06\x44\x65vice\x12$\n\tdevice_id\x18\x01 \x01(\x0b\x32\x11.context.DeviceId\x12\x13\n\x0b\x64\x65vice_type\x18\x02 \x01(\t\x12,\n\rdevice_config\x18\x03 \x01(\x0b\x32\x15.context.DeviceConfig\x12>\n\x14\x64\x65vOperationalStatus\x18\x04 \x01(\x0e\x32 .context.DeviceOperationalStatus\x12\'\n\x0c\x65ndpointList\x18\x05 \x03(\x0b\x32\x11.context.EndPoint\"%\n\x0c\x44\x65viceConfig\x12\x15\n\rdevice_config\x18\x01 \x01(\t\"C\n\x08\x45ndPoint\x12$\n\x07port_id\x18\x01 \x01(\x0b\x32\x13.context.EndPointId\x12\x11\n\tport_type\x18\x02 \x01(\t\"t\n\nEndPointId\x12#\n\x06topoId\x18\x01 \x01(\x0b\x32\x13.context.TopologyId\x12!\n\x06\x64\x65v_id\x18\x02 \x01(\x0b\x32\x11.context.DeviceId\x12\x1e\n\x07port_id\x18\x03 \x01(\x0b\x32\r.context.Uuid\",\n\x08\x44\x65viceId\x12 \n\tdevice_id\x18\x01 \x01(\x0b\x32\r.context.Uuid\"\x14\n\x04Uuid\x12\x0c\n\x04uuid\x18\x01 \x01(\t\"K\n\x12TeraFlowController\x12\"\n\x06\x63tl_id\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x11\n\tipaddress\x18\x02 \x01(\t\"Q\n\x14\x41uthenticationResult\x12\"\n\x06\x63tl_id\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x15\n\rauthenticated\x18\x02 \x01(\x08*4\n\x17\x44\x65viceOperationalStatus\x12\x0c\n\x08\x44ISABLED\x10\x00\x12\x0b\n\x07\x45NABLED\x10\x01\x32\x44\n\x0e\x43ontextService\x12\x32\n\x0bGetTopology\x12\x0e.context.Empty\x1a\x11.context.Topology\"\x00\x62\x06proto3' + serialized_pb=b'\n\rcontext.proto\x12\x07\x63ontext\"\x07\n\x05\x45mpty\"{\n\x07\x43ontext\x12%\n\tcontextId\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x1f\n\x04topo\x18\x02 \x01(\x0b\x32\x11.context.Topology\x12(\n\x03\x63tl\x18\x03 \x01(\x0b\x32\x1b.context.TeraFlowController\"/\n\tContextId\x12\"\n\x0b\x63ontextUuid\x18\x01 \x01(\x0b\x32\r.context.Uuid\"m\n\x08Topology\x12#\n\x06topoId\x18\x02 \x01(\x0b\x32\x13.context.TopologyId\x12\x1f\n\x06\x64\x65vice\x18\x03 \x03(\x0b\x32\x0f.context.Device\x12\x1b\n\x04link\x18\x04 \x03(\x0b\x32\r.context.Link\"S\n\x04Link\x12 \n\x07link_id\x18\x01 \x01(\x0b\x32\x0f.context.LinkId\x12)\n\x0c\x65ndpointList\x18\x02 \x03(\x0b\x32\x13.context.EndPointId\"R\n\nTopologyId\x12%\n\tcontextId\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x1d\n\x06topoId\x18\x02 \x01(\x0b\x32\r.context.Uuid\"?\n\nConstraint\x12\x17\n\x0f\x63onstraint_type\x18\x01 \x01(\t\x12\x18\n\x10\x63onstraint_value\x18\x02 \x01(\t\"\xda\x01\n\x06\x44\x65vice\x12$\n\tdevice_id\x18\x01 \x01(\x0b\x32\x11.context.DeviceId\x12\x13\n\x0b\x64\x65vice_type\x18\x02 \x01(\t\x12,\n\rdevice_config\x18\x03 \x01(\x0b\x32\x15.context.DeviceConfig\x12>\n\x14\x64\x65vOperationalStatus\x18\x04 \x01(\x0e\x32 .context.DeviceOperationalStatus\x12\'\n\x0c\x65ndpointList\x18\x05 \x03(\x0b\x32\x11.context.EndPoint\"%\n\x0c\x44\x65viceConfig\x12\x15\n\rdevice_config\x18\x01 \x01(\t\"C\n\x08\x45ndPoint\x12$\n\x07port_id\x18\x01 \x01(\x0b\x32\x13.context.EndPointId\x12\x11\n\tport_type\x18\x02 \x01(\t\"t\n\nEndPointId\x12#\n\x06topoId\x18\x01 \x01(\x0b\x32\x13.context.TopologyId\x12!\n\x06\x64\x65v_id\x18\x02 \x01(\x0b\x32\x11.context.DeviceId\x12\x1e\n\x07port_id\x18\x03 \x01(\x0b\x32\r.context.Uuid\",\n\x08\x44\x65viceId\x12 \n\tdevice_id\x18\x01 \x01(\x0b\x32\r.context.Uuid\"(\n\x06LinkId\x12\x1e\n\x07link_id\x18\x01 \x01(\x0b\x32\r.context.Uuid\"\x14\n\x04Uuid\x12\x0c\n\x04uuid\x18\x01 \x01(\t\"K\n\x12TeraFlowController\x12\"\n\x06\x63tl_id\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x11\n\tipaddress\x18\x02 \x01(\t\"Q\n\x14\x41uthenticationResult\x12\"\n\x06\x63tl_id\x18\x01 \x01(\x0b\x32\x12.context.ContextId\x12\x15\n\rauthenticated\x18\x02 \x01(\x08*M\n\x17\x44\x65viceOperationalStatus\x12\x0e\n\nKEEP_STATE\x10\x00\x12\x15\n\x08\x44ISABLED\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x12\x0b\n\x07\x45NABLED\x10\x01\x32\xa2\x01\n\x0e\x43ontextService\x12\x32\n\x0bGetTopology\x12\x0e.context.Empty\x1a\x11.context.Topology\"\x00\x12+\n\x07\x41\x64\x64Link\x12\r.context.Link\x1a\x0f.context.LinkId\"\x00\x12/\n\nDeleteLink\x12\x0f.context.LinkId\x1a\x0e.context.Empty\"\x00\x62\x06proto3' ) _DEVICEOPERATIONALSTATUS = _descriptor.EnumDescriptor( @@ -31,25 +31,31 @@ _DEVICEOPERATIONALSTATUS = _descriptor.EnumDescriptor( create_key=_descriptor._internal_create_key, values=[ _descriptor.EnumValueDescriptor( - name='DISABLED', index=0, number=0, + name='KEEP_STATE', index=0, number=0, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), _descriptor.EnumValueDescriptor( - name='ENABLED', index=1, number=1, + name='DISABLED', index=1, number=-1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ENABLED', index=2, number=1, serialized_options=None, type=None, create_key=_descriptor._internal_create_key), ], containing_type=None, serialized_options=None, - serialized_start=1195, - serialized_end=1247, + serialized_start=1271, + serialized_end=1348, ) _sym_db.RegisterEnumDescriptor(_DEVICEOPERATIONALSTATUS) DeviceOperationalStatus = enum_type_wrapper.EnumTypeWrapper(_DEVICEOPERATIONALSTATUS) -DISABLED = 0 +KEEP_STATE = 0 +DISABLED = -1 ENABLED = 1 @@ -212,8 +218,15 @@ _LINK = _descriptor.Descriptor( create_key=_descriptor._internal_create_key, fields=[ _descriptor.FieldDescriptor( - name='endpointList', full_name='context.Link.endpointList', index=0, - number=1, type=11, cpp_type=10, label=3, + name='link_id', full_name='context.Link.link_id', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='endpointList', full_name='context.Link.endpointList', index=1, + number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -231,7 +244,7 @@ _LINK = _descriptor.Descriptor( oneofs=[ ], serialized_start=320, - serialized_end=369, + serialized_end=403, ) @@ -269,8 +282,8 @@ _TOPOLOGYID = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=371, - serialized_end=453, + serialized_start=405, + serialized_end=487, ) @@ -308,8 +321,8 @@ _CONSTRAINT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=455, - serialized_end=518, + serialized_start=489, + serialized_end=552, ) @@ -368,8 +381,8 @@ _DEVICE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=521, - serialized_end=739, + serialized_start=555, + serialized_end=773, ) @@ -400,8 +413,8 @@ _DEVICECONFIG = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=741, - serialized_end=778, + serialized_start=775, + serialized_end=812, ) @@ -439,8 +452,8 @@ _ENDPOINT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=780, - serialized_end=847, + serialized_start=814, + serialized_end=881, ) @@ -485,8 +498,8 @@ _ENDPOINTID = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=849, - serialized_end=965, + serialized_start=883, + serialized_end=999, ) @@ -517,8 +530,40 @@ _DEVICEID = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=967, - serialized_end=1011, + serialized_start=1001, + serialized_end=1045, +) + + +_LINKID = _descriptor.Descriptor( + name='LinkId', + full_name='context.LinkId', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='link_id', full_name='context.LinkId.link_id', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1047, + serialized_end=1087, ) @@ -549,8 +594,8 @@ _UUID = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1013, - serialized_end=1033, + serialized_start=1089, + serialized_end=1109, ) @@ -588,8 +633,8 @@ _TERAFLOWCONTROLLER = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1035, - serialized_end=1110, + serialized_start=1111, + serialized_end=1186, ) @@ -627,8 +672,8 @@ _AUTHENTICATIONRESULT = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1112, - serialized_end=1193, + serialized_start=1188, + serialized_end=1269, ) _CONTEXT.fields_by_name['contextId'].message_type = _CONTEXTID @@ -638,6 +683,7 @@ _CONTEXTID.fields_by_name['contextUuid'].message_type = _UUID _TOPOLOGY.fields_by_name['topoId'].message_type = _TOPOLOGYID _TOPOLOGY.fields_by_name['device'].message_type = _DEVICE _TOPOLOGY.fields_by_name['link'].message_type = _LINK +_LINK.fields_by_name['link_id'].message_type = _LINKID _LINK.fields_by_name['endpointList'].message_type = _ENDPOINTID _TOPOLOGYID.fields_by_name['contextId'].message_type = _CONTEXTID _TOPOLOGYID.fields_by_name['topoId'].message_type = _UUID @@ -650,6 +696,7 @@ _ENDPOINTID.fields_by_name['topoId'].message_type = _TOPOLOGYID _ENDPOINTID.fields_by_name['dev_id'].message_type = _DEVICEID _ENDPOINTID.fields_by_name['port_id'].message_type = _UUID _DEVICEID.fields_by_name['device_id'].message_type = _UUID +_LINKID.fields_by_name['link_id'].message_type = _UUID _TERAFLOWCONTROLLER.fields_by_name['ctl_id'].message_type = _CONTEXTID _AUTHENTICATIONRESULT.fields_by_name['ctl_id'].message_type = _CONTEXTID DESCRIPTOR.message_types_by_name['Empty'] = _EMPTY @@ -664,6 +711,7 @@ DESCRIPTOR.message_types_by_name['DeviceConfig'] = _DEVICECONFIG DESCRIPTOR.message_types_by_name['EndPoint'] = _ENDPOINT DESCRIPTOR.message_types_by_name['EndPointId'] = _ENDPOINTID DESCRIPTOR.message_types_by_name['DeviceId'] = _DEVICEID +DESCRIPTOR.message_types_by_name['LinkId'] = _LINKID DESCRIPTOR.message_types_by_name['Uuid'] = _UUID DESCRIPTOR.message_types_by_name['TeraFlowController'] = _TERAFLOWCONTROLLER DESCRIPTOR.message_types_by_name['AuthenticationResult'] = _AUTHENTICATIONRESULT @@ -754,6 +802,13 @@ DeviceId = _reflection.GeneratedProtocolMessageType('DeviceId', (_message.Messag }) _sym_db.RegisterMessage(DeviceId) +LinkId = _reflection.GeneratedProtocolMessageType('LinkId', (_message.Message,), { + 'DESCRIPTOR' : _LINKID, + '__module__' : 'context_pb2' + # @@protoc_insertion_point(class_scope:context.LinkId) + }) +_sym_db.RegisterMessage(LinkId) + Uuid = _reflection.GeneratedProtocolMessageType('Uuid', (_message.Message,), { 'DESCRIPTOR' : _UUID, '__module__' : 'context_pb2' @@ -784,8 +839,8 @@ _CONTEXTSERVICE = _descriptor.ServiceDescriptor( index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=1249, - serialized_end=1317, + serialized_start=1351, + serialized_end=1513, methods=[ _descriptor.MethodDescriptor( name='GetTopology', @@ -797,6 +852,26 @@ _CONTEXTSERVICE = _descriptor.ServiceDescriptor( serialized_options=None, create_key=_descriptor._internal_create_key, ), + _descriptor.MethodDescriptor( + name='AddLink', + full_name='context.ContextService.AddLink', + index=1, + containing_service=None, + input_type=_LINK, + output_type=_LINKID, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + _descriptor.MethodDescriptor( + name='DeleteLink', + full_name='context.ContextService.DeleteLink', + index=2, + containing_service=None, + input_type=_LINKID, + output_type=_EMPTY, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), ]) _sym_db.RegisterServiceDescriptor(_CONTEXTSERVICE) diff --git a/src/device/proto/context_pb2_grpc.py b/src/device/proto/context_pb2_grpc.py deleted file mode 100644 index f1ff4672a33091ca7fb800aec2af49795050dd03..0000000000000000000000000000000000000000 --- a/src/device/proto/context_pb2_grpc.py +++ /dev/null @@ -1,66 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -from device.proto import context_pb2 as context__pb2 - - -class ContextServiceStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.GetTopology = channel.unary_unary( - '/context.ContextService/GetTopology', - request_serializer=context__pb2.Empty.SerializeToString, - response_deserializer=context__pb2.Topology.FromString, - ) - - -class ContextServiceServicer(object): - """Missing associated documentation comment in .proto file.""" - - def GetTopology(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_ContextServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'GetTopology': grpc.unary_unary_rpc_method_handler( - servicer.GetTopology, - request_deserializer=context__pb2.Empty.FromString, - response_serializer=context__pb2.Topology.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'context.ContextService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class ContextService(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def GetTopology(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/context.ContextService/GetTopology', - context__pb2.Empty.SerializeToString, - context__pb2.Topology.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/device/proto/device_pb2.py b/src/device/proto/device_pb2.py index d561c2a39f02e01bd6584ffa03b848e3d3c1b9c8..62aa8b4467acc0856981de7ab9f6069877c6971a 100644 --- a/src/device/proto/device_pb2.py +++ b/src/device/proto/device_pb2.py @@ -11,7 +11,7 @@ from google.protobuf import symbol_database as _symbol_database _sym_db = _symbol_database.Default() -from device.proto import context_pb2 as context__pb2 +from . import context_pb2 as context__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -20,7 +20,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0c\x64\x65vice.proto\x12\x06\x64\x65vice\x1a\rcontext.proto2\xb6\x01\n\rDeviceService\x12\x31\n\tAddDevice\x12\x0f.context.Device\x1a\x11.context.DeviceId\"\x00\x12=\n\x0f\x43onfigureDevice\x12\x15.context.DeviceConfig\x1a\x11.context.DeviceId\"\x00\x12\x33\n\x0c\x44\x65leteDevice\x12\x11.context.DeviceId\x1a\x0e.context.Empty\"\x00\x62\x06proto3' + serialized_pb=b'\n\x0c\x64\x65vice.proto\x12\x06\x64\x65vice\x1a\rcontext.proto2\xb0\x01\n\rDeviceService\x12\x31\n\tAddDevice\x12\x0f.context.Device\x1a\x11.context.DeviceId\"\x00\x12\x37\n\x0f\x43onfigureDevice\x12\x0f.context.Device\x1a\x11.context.DeviceId\"\x00\x12\x33\n\x0c\x44\x65leteDevice\x12\x11.context.DeviceId\x1a\x0e.context.Empty\"\x00\x62\x06proto3' , dependencies=[context__pb2.DESCRIPTOR,]) @@ -38,7 +38,7 @@ _DEVICESERVICE = _descriptor.ServiceDescriptor( serialized_options=None, create_key=_descriptor._internal_create_key, serialized_start=40, - serialized_end=222, + serialized_end=216, methods=[ _descriptor.MethodDescriptor( name='AddDevice', @@ -55,7 +55,7 @@ _DEVICESERVICE = _descriptor.ServiceDescriptor( full_name='device.DeviceService.ConfigureDevice', index=1, containing_service=None, - input_type=context__pb2._DEVICECONFIG, + input_type=context__pb2._DEVICE, output_type=context__pb2._DEVICEID, serialized_options=None, create_key=_descriptor._internal_create_key, diff --git a/src/device/proto/device_pb2_grpc.py b/src/device/proto/device_pb2_grpc.py index fd8c6bf32128f6390a0bc6a2a38a15d86758158c..f295951893f3ef127a8b42a8c9c94819c61b2f20 100644 --- a/src/device/proto/device_pb2_grpc.py +++ b/src/device/proto/device_pb2_grpc.py @@ -2,7 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -from device.proto import context_pb2 as context__pb2 +from . import context_pb2 as context__pb2 class DeviceServiceStub(object): @@ -21,7 +21,7 @@ class DeviceServiceStub(object): ) self.ConfigureDevice = channel.unary_unary( '/device.DeviceService/ConfigureDevice', - request_serializer=context__pb2.DeviceConfig.SerializeToString, + request_serializer=context__pb2.Device.SerializeToString, response_deserializer=context__pb2.DeviceId.FromString, ) self.DeleteDevice = channel.unary_unary( @@ -62,7 +62,7 @@ def add_DeviceServiceServicer_to_server(servicer, server): ), 'ConfigureDevice': grpc.unary_unary_rpc_method_handler( servicer.ConfigureDevice, - request_deserializer=context__pb2.DeviceConfig.FromString, + request_deserializer=context__pb2.Device.FromString, response_serializer=context__pb2.DeviceId.SerializeToString, ), 'DeleteDevice': grpc.unary_unary_rpc_method_handler( @@ -109,7 +109,7 @@ class DeviceService(object): timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/device.DeviceService/ConfigureDevice', - context__pb2.DeviceConfig.SerializeToString, + context__pb2.Device.SerializeToString, context__pb2.DeviceId.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/src/device/run_integration_tests.sh b/src/device/run_integration_tests.sh deleted file mode 100755 index 616397a898bc49960eeb9988b526968625e6f904..0000000000000000000000000000000000000000 --- a/src/device/run_integration_tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Make folder containing the script the root folder for its execution -cd $(dirname $0) - -ENDPOINT=($(kubectl --namespace teraflow-development get service deviceservice -o 'jsonpath={.spec.clusterIP} {.spec.ports[?(@.name=="grpc")].port}')) -docker run -it --env TEST_TARGET_ADDRESS=${ENDPOINT[0]} --env TEST_TARGET_PORT=${ENDPOINT[1]} device_service:test diff --git a/src/device/run_unitary_tests.sh b/src/device/run_unitary_tests.sh deleted file mode 100755 index 08e941f31502fe8dc32ffcfc1563c2223bb4d8d3..0000000000000000000000000000000000000000 --- a/src/device/run_unitary_tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Make folder containing the script the root folder for its execution -cd $(dirname $0) - -mkdir -p data -pytest -v --log-level=DEBUG tests/test_unitary.py diff --git a/src/device/service/DeviceServiceServicerImpl.py b/src/device/service/DeviceServiceServicerImpl.py index dc946bc972c9e01409e4d3cd178b754100cec5c7..25aaa429678ea9aa78293af5469e3c54ac177f6e 100644 --- a/src/device/service/DeviceServiceServicerImpl.py +++ b/src/device/service/DeviceServiceServicerImpl.py @@ -1,11 +1,18 @@ +from typing import List, Tuple import grpc, logging from prometheus_client import Counter, Histogram -from google.protobuf.json_format import MessageToDict -from context.proto.context_pb2 import DeviceId, Empty +from common.Checkers import chk_options, chk_string +from common.database.api.Database import Database +from common.database.api.context.OperationalStatus import OperationalStatus, operationalstatus_enum_values, \ + to_operationalstatus_enum +from device.proto.context_pb2 import DeviceId, Device, Empty from device.proto.device_pb2_grpc import DeviceServiceServicer LOGGER = logging.getLogger(__name__) +DEFAULT_CONTEXT_ID = 'admin' +DEFAULT_TOPOLOGY_ID = 'admin' + ADDDEVICE_COUNTER_STARTED = Counter ('device_adddevice_counter_started', 'Device:AddDevice counter of requests started' ) ADDDEVICE_COUNTER_COMPLETED = Counter ('device_adddevice_counter_completed', @@ -33,56 +40,240 @@ DELETEDEVICE_COUNTER_FAILED = Counter ('device_deletedevice_counter_failed' DELETEDEVICE_HISTOGRAM_DURATION = Histogram('device_deletedevice_histogram_duration', 'Device:DeleteDevice histogram of request duration') +class ServiceException(Exception): + def __init__(self, code : grpc.StatusCode, details : str) -> None: + self.code = code + self.details = details + super().__init__() + class DeviceServiceServicerImpl(DeviceServiceServicer): - def __init__(self, database): + def __init__(self, database : Database): LOGGER.debug('Creating Servicer...') self.database = database LOGGER.debug('Servicer Created') @ADDDEVICE_HISTOGRAM_DURATION.time() - def AddDevice(self, request, context): - # request=context.Device(), returns=context.DeviceId() + def AddDevice(self, request : Device, grpc_context : grpc.ServicerContext) -> DeviceId: ADDDEVICE_COUNTER_STARTED.inc() try: - LOGGER.info('AddDevice request: {}'.format(str(request))) - reply = DeviceId(**self.database.add_device(MessageToDict(request))) - LOGGER.info('AddDevice reply: {}'.format(str(reply))) + LOGGER.debug('AddDevice request: {}'.format(str(request))) + + # ----- Validate request data and pre-conditions ----------------------------------------------------------- + try: + device_uuid = chk_string('device_uuid', request.device_id.device_id.uuid, allow_empty=False) + device_type = chk_string('device_type', request.device_type, allow_empty=False) + device_config = chk_string('device_config', request.device_config.device_config, allow_empty=True) + device_opstat = chk_options('devOperationalStatus', request.devOperationalStatus, + operationalstatus_enum_values()) + except Exception as e: + LOGGER.exception('Invalid arguments:') + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, str(e)) + + device_opstat = to_operationalstatus_enum(device_opstat) + if device_opstat == OperationalStatus.KEEP_STATE: + msg = ' '.join([ + 'Device has to be created with either ENABLED/DISABLED Operational State.', + 'Use KEEP_STATE only in configure Device methods.', + ]) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + + db_context = self.database.context(DEFAULT_CONTEXT_ID).create() + db_topology = db_context.topology(DEFAULT_TOPOLOGY_ID).create() + + if db_topology.devices.contains(device_uuid): + msg = 'device_uuid({}) already exists.' + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.ALREADY_EXISTS, msg) + + added_endpoint_uuids = set() + endpoint_pairs : List[Tuple[str, str]] = [] + for i,endpoint in enumerate(request.endpointList): + contextId = endpoint.port_id.topoId.contextId.contextUuid.uuid + if (len(contextId) > 0) and (contextId != DEFAULT_CONTEXT_ID): + msg = ' '.join([ + 'Unsupported context_id({}) in endpoint #{}.', + 'Only default context_id({}) is currently supported.', + 'Optionally, leave field empty to use default context_id.', + ]) + msg = msg.format(contextId, i, DEFAULT_CONTEXT_ID) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + elif len(contextId) == 0: + contextId = DEFAULT_CONTEXT_ID + + topoId = endpoint.port_id.topoId.topoId.uuid + if (len(topoId) > 0) and (topoId != DEFAULT_TOPOLOGY_ID): + msg = ' '.join([ + 'Unsupported topology_id({}) in endpoint #{}.', + 'Only default topology_id({}) is currently supported.', + 'Optionally, leave field empty to use default topology_id.', + ]) + msg = msg.format(topoId, i, DEFAULT_TOPOLOGY_ID) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + elif len(topoId) == 0: + topoId = DEFAULT_TOPOLOGY_ID + + dev_id = endpoint.port_id.dev_id.device_id.uuid + if (len(dev_id) > 0) and (dev_id != device_uuid): + msg = ' '.join([ + 'Wrong device_id({}) in endpoint #{}.', + 'Parent specified in message is device_id({}).', + 'Optionally, leave field empty to use parent device_id.', + ]) + msg = msg.format(dev_id, i, device_uuid) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + elif len(dev_id) == 0: + dev_id = device_uuid + + try: + port_id = chk_string('port_uuid', endpoint.port_id.port_id.uuid, allow_empty=False) + port_type = chk_string('port_type', endpoint.port_type, allow_empty=False) + except Exception as e: + LOGGER.exception('Invalid arguments:') + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, str(e)) + + if port_id in added_endpoint_uuids: + msg = 'Duplicated port_id({}) in device_id({}).' + msg = msg.format(port_id, device_uuid) + raise ServiceException(grpc.StatusCode.ALREADY_EXISTS, msg) + + added_endpoint_uuids.add(port_id) + endpoint_pairs.append((port_id, port_type)) + + # ----- Implement changes in database ---------------------------------------------------------------------- + db_device = db_topology.device(device_uuid).create(device_type, device_config, device_opstat) + for port_id,port_type in endpoint_pairs: db_device.endpoint(port_id).create(port_type) + + # ----- Compose reply -------------------------------------------------------------------------------------- + reply = DeviceId(device_id=dict(uuid=device_uuid)) + LOGGER.debug('AddDevice reply: {}'.format(str(reply))) ADDDEVICE_COUNTER_COMPLETED.inc() return reply - except: - LOGGER.exception('AddDevice exception') - ADDDEVICE_COUNTER_FAILED.inc() - context.set_code(grpc.StatusCode.INTERNAL) - return DeviceId() + except ServiceException as e: + grpc_context.abort(e.code, e.details) + except Exception as e: # pragma: no cover + LOGGER.exception('AddDevice exception') # pragma: no cover + ADDDEVICE_COUNTER_FAILED.inc() # pragma: no cover + grpc_context.abort(grpc.StatusCode.INTERNAL, str(e)) # pragma: no cover @CONFIGUREDEVICE_HISTOGRAM_DURATION.time() - def ConfigureDevice(self, request, context): - # request=context.DeviceConfig(), returns=context.DeviceId() + def ConfigureDevice(self, request : Device, grpc_context : grpc.ServicerContext) -> DeviceId: CONFIGUREDEVICE_COUNTER_STARTED.inc() try: LOGGER.info('ConfigureDevice request: {}'.format(str(request))) - reply = DeviceId(**self.database.configure_device(MessageToDict(request))) + + # ----- Validate request data and pre-conditions ----------------------------------------------------------- + try: + device_uuid = chk_string('device_uuid', request.device_id.device_id.uuid, allow_empty=False) + device_type = chk_string('device_type', request.device_type, allow_empty=True) + device_config = chk_string('device_config', request.device_config.device_config, allow_empty=True) + device_opstat = chk_options('devOperationalStatus', request.devOperationalStatus, + operationalstatus_enum_values()) + except Exception as e: + LOGGER.exception('Invalid arguments:') + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, str(e)) + + device_opstat = to_operationalstatus_enum(device_opstat) + if device_opstat is None: + msg = 'Unsupported OperationalStatus({}).' + msg = msg.format(request.devOperationalStatus) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + + db_context = self.database.context(DEFAULT_CONTEXT_ID).create() + db_topology = db_context.topology(DEFAULT_TOPOLOGY_ID).create() + + if not db_topology.devices.contains(device_uuid): + msg = 'device_uuid({}) does not exist.' + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.NOT_FOUND, msg) + + db_device = db_topology.device(device_uuid) + db_device_attributes = db_device.attributes.get(attributes=['device_type']) + if len(db_device_attributes) == 0: + msg = 'attribute device_type for device_uuid({}) does not exist.' + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.FAILED_PRECONDITION, msg) + + db_device_type = db_device_attributes.get('device_type') + if len(db_device_type) == 0: + msg = 'attribute device_type for device_uuid({}) is empty.' + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.FAILED_PRECONDITION, msg) + + if db_device_type != device_type: + msg = 'Device({}) has Type({}). Cannot be changed to Type({}).' + msg = msg.format(device_uuid, db_device_type, device_type) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + + if len(request.endpointList) > 0: + msg = 'Endpoints belonging to Device({}) cannot be modified.' + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, msg) + + update_attributes = {} + + if len(device_config) > 0: + update_attributes['device_config'] = device_config + + if device_opstat != OperationalStatus.KEEP_STATE: + update_attributes['device_operational_status'] = device_opstat + + LOGGER.info('update_attributes={}'.format(str(update_attributes))) + + if len(update_attributes) == 0: + msg = ' '.join([ + 'Any change has been requested for Device({}).', + 'Either specify a new configuration or a new device state.', + ]) + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.ABORTED, msg) + + # ----- Implement changes in database ---------------------------------------------------------------------- + db_device.update(update_attributes=update_attributes) + + # ----- Compose reply -------------------------------------------------------------------------------------- + reply = DeviceId(device_id=dict(uuid=device_uuid)) LOGGER.info('ConfigureDevice reply: {}'.format(str(reply))) CONFIGUREDEVICE_COUNTER_COMPLETED.inc() return reply - except: - LOGGER.exception('ConfigureDevice exception') - CONFIGUREDEVICE_COUNTER_FAILED.inc() - context.set_code(grpc.StatusCode.INTERNAL) - return DeviceId() + except ServiceException as e: + grpc_context.abort(e.code, e.details) + except Exception as e: # pragma: no cover + LOGGER.exception('ConfigureDevice exception') # pragma: no cover + CONFIGUREDEVICE_COUNTER_FAILED.inc() # pragma: no cover + grpc_context.abort(grpc.StatusCode.INTERNAL, str(e)) # pragma: no cover @DELETEDEVICE_HISTOGRAM_DURATION.time() - def DeleteDevice(self, request, context): - # request=context.DeviceId(), returns=context.Empty() + def DeleteDevice(self, request : DeviceId, grpc_context : grpc.ServicerContext) -> Empty: DELETEDEVICE_COUNTER_STARTED.inc() try: - LOGGER.info('DeleteDevice request: {}'.format(str(request))) - reply = Empty(**self.database.delete_device(MessageToDict(request))) - LOGGER.info('DeleteDevice reply: {}'.format(str(reply))) + LOGGER.debug('DeleteDevice request: {}'.format(str(request))) + + # ----- Validate request data and pre-conditions ----------------------------------------------------------- + try: + device_uuid = chk_string('device_uuid', request.device_id.uuid, allow_empty=False) + except Exception as e: + LOGGER.exception('Invalid arguments:') + raise ServiceException(grpc.StatusCode.INVALID_ARGUMENT, str(e)) + + db_context = self.database.context(DEFAULT_CONTEXT_ID).create() + db_topology = db_context.topology(DEFAULT_TOPOLOGY_ID).create() + + if not db_topology.devices.contains(device_uuid): + msg = 'device_uuid({}) does not exist.' + msg = msg.format(device_uuid) + raise ServiceException(grpc.StatusCode.NOT_FOUND, msg) + + # ----- Implement changes in database ---------------------------------------------------------------------- + db_topology.device(device_uuid).delete() + + # ----- Compose reply -------------------------------------------------------------------------------------- + reply = Empty() + LOGGER.debug('DeleteDevice reply: {}'.format(str(reply))) DELETEDEVICE_COUNTER_COMPLETED.inc() return reply - except: - LOGGER.exception('DeleteDevice exception') - DELETEDEVICE_COUNTER_FAILED.inc() - context.set_code(grpc.StatusCode.INTERNAL) - return Empty() + except ServiceException as e: + grpc_context.abort(e.code, e.details) + except Exception as e: # pragma: no cover + LOGGER.exception('DeleteDevice exception') # pragma: no cover + DELETEDEVICE_COUNTER_FAILED.inc() # pragma: no cover + grpc_context.abort(grpc.StatusCode.INTERNAL, str(e)) # pragma: no cover diff --git a/src/device/service/__main__.py b/src/device/service/__main__.py index 81791277e88aa7a4e96ab7f69de43f7011995ca5..8ec270586140ef35c10fa5d5ea112ec4322c5c4c 100644 --- a/src/device/service/__main__.py +++ b/src/device/service/__main__.py @@ -1,6 +1,6 @@ import logging, os, signal, sys, threading from prometheus_client import start_http_server -from device.database.Factory import get_database +from common.database.Factory import get_database from device.service.DeviceService import DeviceService from device.Config import SERVICE_PORT, MAX_WORKERS, GRACE_PERIOD, LOG_LEVEL, METRICS_PORT diff --git a/src/device/tests/test_integration.py b/src/device/tests/test_integration.py index eab068b493a06754ec335ea118fa60e671fddec7..3f1519aba7b1a35972d467279eeb45c2723390b2 100644 --- a/src/device/tests/test_integration.py +++ b/src/device/tests/test_integration.py @@ -1,24 +1,24 @@ -import logging, os, pytest, sys +#import logging, os, pytest, sys -from pathlib import Path -sys.path.append(__file__.split('src')[0] + 'src') -print(sys.path) +#from pathlib import Path +#sys.path.append(__file__.split('src')[0] + 'src') +#print(sys.path) -from context.client.ContextClient import ContextClient -from context.proto.context_pb2 import Empty -from .tools.ValidateTopology import validate_topology_dict +#from context.client.ContextClient import ContextClient +#from context.proto.context_pb2 import Empty +#from .tools.ValidateTopology import validate_topology_dict -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) +#LOGGER = logging.getLogger(__name__) +#LOGGER.setLevel(logging.DEBUG) -@pytest.fixture(scope='session') -def remote_context_client(): - address = os.environ.get('TEST_TARGET_ADDRESS') - if(address is None): raise Exception('EnvironmentVariable(TEST_TARGET_ADDRESS) not specified') - port = os.environ.get('TEST_TARGET_PORT') - if(port is None): raise Exception('EnvironmentVariable(TEST_TARGET_PORT) not specified') - return ContextClient(address=address, port=port) +#@pytest.fixture(scope='session') +#def remote_context_client(): +# address = os.environ.get('TEST_TARGET_ADDRESS') +# if(address is None): raise Exception('EnvironmentVariable(TEST_TARGET_ADDRESS) not specified') +# port = os.environ.get('TEST_TARGET_PORT') +# if(port is None): raise Exception('EnvironmentVariable(TEST_TARGET_PORT) not specified') +# return ContextClient(address=address, port=port) -def test_remote_get_topology(remote_context_client): - response = remote_context_client.GetTopology(Empty()) - validate_topology_dict(response) +#def test_remote_get_topology(remote_context_client): +# response = remote_context_client.GetTopology(Empty()) +# validate_topology_dict(response) diff --git a/src/device/tests/test_unitary.py b/src/device/tests/test_unitary.py index 61e580ea704260cd034273e2bd74ae9fbbd606e6..de7539274352b5b2b1e0397f54cedd2e6e24fbc9 100644 --- a/src/device/tests/test_unitary.py +++ b/src/device/tests/test_unitary.py @@ -1,31 +1,270 @@ -import logging, pytest, sys - -from pathlib import Path -sys.path.append(__file__.split('src')[0] + 'src') -print(sys.path) - -from context.client.ContextClient import ContextClient -from context.database.Factory import get_database, DatabaseEngineEnum -from context.proto.context_pb2 import Empty -from context.service.ContextService import ContextService -from context.Config import SERVICE_PORT, MAX_WORKERS, GRACE_PERIOD -from context.tests.tools.ValidateTopology import validate_topology_dict +import copy, grpc, logging, pytest +from src.common.database.api.context.OperationalStatus import OperationalStatus +from google.protobuf.json_format import MessageToDict +from common.database.Factory import get_database, DatabaseEngineEnum +from common.tests.Assertions import validate_device_id, validate_empty +from device.client.DeviceClient import DeviceClient +from device.proto.context_pb2 import Device, DeviceId +from device.service.DeviceService import DeviceService +from device.Config import SERVICE_PORT, MAX_WORKERS, GRACE_PERIOD LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) +DEVICE_ID = {'device_id': {'uuid': 'test-device-001'}} +DEVICE = { + 'device_id': {'device_id': {'uuid': 'test-device-001'}}, + 'device_type': 'ROADM', + 'device_config': {'device_config': ''}, + 'devOperationalStatus': 1, + 'endpointList' : [ + { + 'port_id': { + 'topoId': { + 'contextId': {'contextUuid': {'uuid': 'admin'}}, + 'topoId': {'uuid': 'admin'} + }, + 'dev_id': {'device_id': {'uuid': 'test-device-001'}}, + 'port_id': {'uuid' : 'port-101'} + }, + 'port_type': 'LINE' + }, + { + 'port_id': { + 'topoId': { + 'contextId': {'contextUuid': {'uuid': 'admin'}}, + 'topoId': {'uuid': 'admin'} + }, + 'dev_id': {'device_id': {'uuid': 'test-device-001'}}, + 'port_id': {'uuid' : 'port-102'} + }, + 'port_type': 'LINE' + }, + ] +} + @pytest.fixture(scope='session') -def local_context_service(): +def service(): database = get_database(engine=DatabaseEngineEnum.INMEMORY, filepath='data/topo_nsfnet.json') - _service = ContextService(database, port=SERVICE_PORT, max_workers=MAX_WORKERS, grace_period=GRACE_PERIOD) + _service = DeviceService(database, port=SERVICE_PORT, max_workers=MAX_WORKERS, grace_period=GRACE_PERIOD) _service.start() yield _service _service.stop() @pytest.fixture(scope='session') -def local_context_client(local_context_service): - return ContextClient(address='127.0.0.1', port=SERVICE_PORT) +def client(service): + _client = DeviceClient(address='127.0.0.1', port=SERVICE_PORT) + yield _client + _client.close() + +def test_create_empty_device_uuid(client : DeviceClient): + # should fail with device uuid is empty + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['device_id']['device_id']['uuid'] = '' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'device_uuid() string is empty.' + +def test_create_empty_device_type(client : DeviceClient): + # should fail with device type is empty + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['device_type'] = '' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'device_type() string is empty.' + +def test_create_wrong_device_operational_status(client : DeviceClient): + # should fail with wrong device operational status + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['devOperationalStatus'] = OperationalStatus.KEEP_STATE.value + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + msg = ' '.join([ + 'Device has to be created with either ENABLED/DISABLED Operational State.', + 'Use KEEP_STATE only in configure Device methods.', + ]) + assert e.value.details() == msg + +def test_create_endpoint_wrong_context(client : DeviceClient): + # should fail with unsupported context + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][0]['port_id']['topoId']['contextId']['contextUuid']['uuid'] = 'wrong-context' + request = Device(**copy_device) + LOGGER.warning('request = {}'.format(request)) + client.AddDevice(request) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + msg = ' '.join([ + 'Unsupported context_id(wrong-context) in endpoint #0.', + 'Only default context_id(admin) is currently supported.', + 'Optionally, leave field empty to use default context_id.', + ]) + assert e.value.details() == msg + +def test_create_endpoint_wrong_topology(client : DeviceClient): + # should fail with unsupported topology + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][0]['port_id']['topoId']['topoId']['uuid'] = 'wrong-topo' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + msg = ' '.join([ + 'Unsupported topology_id(wrong-topo) in endpoint #0.', + 'Only default topology_id(admin) is currently supported.', + 'Optionally, leave field empty to use default topology_id.', + ]) + assert e.value.details() == msg + +def test_create_endpoint_wrong_device(client : DeviceClient): + # should fail with wrong endpoint device + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][0]['port_id']['dev_id']['device_id']['uuid'] = 'wrong-device' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + msg = ' '.join([ + 'Wrong device_id(wrong-device) in endpoint #0.', + 'Parent specified in message is device_id(test-device-001).', + 'Optionally, leave field empty to use parent device_id.', + ]) + assert e.value.details() == msg + +def test_create_empty_port_uuid(client : DeviceClient): + # should fail with port uuid is empty + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][0]['port_id']['port_id']['uuid'] = '' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'port_uuid() string is empty.' + +def test_create_empty_port_type(client : DeviceClient): + # should fail with port type is empty + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][0]['port_type'] = '' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'port_type() string is empty.' + +def test_create_duplicate_port(client : DeviceClient): + # should fail with uplicate port in device + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][1]['port_id']['port_id']['uuid'] = 'port-101' + client.AddDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.ALREADY_EXISTS + assert e.value.details() == 'Duplicated port_id(port-101) in device_id(test-device-001).' + +def test_create(client : DeviceClient): + # should work + validate_device_id(MessageToDict( + client.AddDevice(Device(**DEVICE)), + including_default_value_fields=True, preserving_proto_field_name=True, + use_integers_for_enums=False)) + +def test_create_duplicate(client : DeviceClient): + # should fail with device already exists + with pytest.raises(grpc._channel._InactiveRpcError) as e: + client.AddDevice(Device(**DEVICE)) + assert e.value.code() == grpc.StatusCode.ALREADY_EXISTS + assert e.value.details() == 'device_uuid(test-device-001) already exists.' + +def test_delete_empty_uuid(client : DeviceClient): + # should fail with device uuid is empty + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device_id = copy.deepcopy(DEVICE_ID) + copy_device_id['device_id']['uuid'] = '' + client.DeleteDevice(DeviceId(**copy_device_id)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'device_uuid() string is empty.' + +def test_delete_device_not_found(client : DeviceClient): + # should fail with device not found + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device_id = copy.deepcopy(DEVICE_ID) + copy_device_id['device_id']['uuid'] = 'wrong-device-id' + client.DeleteDevice(DeviceId(**copy_device_id)) + assert e.value.code() == grpc.StatusCode.NOT_FOUND + assert e.value.details() == 'device_uuid(wrong-device-id) does not exist.' + +def test_delete(client : DeviceClient): + # should work + validate_empty(MessageToDict( + client.DeleteDevice(DeviceId(**DEVICE_ID)), + including_default_value_fields=True, preserving_proto_field_name=True, + use_integers_for_enums=False)) + +def test_configure_empty_device_uuid(client : DeviceClient): + # should fail with device uuid is empty + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['device_id']['device_id']['uuid'] = '' + client.ConfigureDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'device_uuid() string is empty.' + +def test_configure_device_not_found(client : DeviceClient): + # should fail with device not found + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['device_id']['device_id']['uuid'] = 'wrong-device-id' + client.ConfigureDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.NOT_FOUND + assert e.value.details() == 'device_uuid(wrong-device-id) does not exist.' + +def test_create_device_default_endpoint_context_topology(client : DeviceClient): + # should work + copy_device = copy.deepcopy(DEVICE) + copy_device['endpointList'][0]['port_id']['topoId']['contextId']['contextUuid']['uuid'] = '' + copy_device['endpointList'][0]['port_id']['topoId']['topoId']['uuid'] = '' + copy_device['endpointList'][0]['port_id']['dev_id']['device_id']['uuid'] = '' + validate_device_id(MessageToDict( + client.AddDevice(Device(**copy_device)), + including_default_value_fields=True, preserving_proto_field_name=True, + use_integers_for_enums=False)) + +def test_configure_wrong_device_type(client : DeviceClient): + # should fail with device type is wrong + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['device_type'] = 'wrong-type' + client.ConfigureDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'Device(test-device-001) has Type(ROADM). Cannot be changed to Type(wrong-type).' + +def test_configure_with_endpoints(client : DeviceClient): + # should fail with endpoints cannot be modified + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + client.ConfigureDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.INVALID_ARGUMENT + assert e.value.details() == 'Endpoints belonging to Device(test-device-001) cannot be modified.' + +def test_configure_no_change(client : DeviceClient): + # should fail with any change detected + with pytest.raises(grpc._channel._InactiveRpcError) as e: + copy_device = copy.deepcopy(DEVICE) + copy_device['devOperationalStatus'] = OperationalStatus.KEEP_STATE.value + copy_device['endpointList'].clear() + client.ConfigureDevice(Device(**copy_device)) + assert e.value.code() == grpc.StatusCode.ABORTED + msg = ' '.join([ + 'Any change has been requested for Device(test-device-001).', + 'Either specify a new configuration or a new device state.', + ]) + assert e.value.details() == msg -def test_local_get_topology(local_context_client): - response = local_context_client.GetTopology(Empty()) - validate_topology_dict(response) +def test_configure(client : DeviceClient): + # should work + copy_device = copy.deepcopy(DEVICE) + copy_device['device_config']['device_config'] = '<new_config/>' + copy_device['devOperationalStatus'] = OperationalStatus.DISABLED.value + copy_device['endpointList'].clear() + validate_device_id(MessageToDict( + client.ConfigureDevice(Device(**copy_device)), + including_default_value_fields=True, preserving_proto_field_name=True, + use_integers_for_enums=False)) diff --git a/src/device/tests/tools/ValidateTopology.py b/src/device/tests/tools/ValidateTopology.py deleted file mode 100644 index b52546e39c27292bec4f11755dade987929e5e71..0000000000000000000000000000000000000000 --- a/src/device/tests/tools/ValidateTopology.py +++ /dev/null @@ -1,6 +0,0 @@ -def validate_topology_dict(topology): - assert type(topology) is dict - assert len(topology.keys()) > 0 - assert 'topoId' in topology - assert 'device' in topology - assert 'link' in topology