Commit dcd19e78 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Context: 

- updated run_tests_locally script
- updated GitLab CI/CD with NATS
- updated ContextModel and TopologyModel with created/updated
- added logic for Context and Topology events created/updated/deleted
- activated dependencies for test_connection
- activated event testing in Context and Topology entities
- corrected conftest for Context component
parent 692fc03e

scripts/run_tests_locally-context.sh

+55 −17
Original line number Diff line number Diff line
@@ -13,28 +13,66 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



########################################################################################################################

# Define your deployment settings here

########################################################################################################################



# If not already set, set the name of the Kubernetes namespace to deploy to.

export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}



########################################################################################################################

# Automated steps start here

########################################################################################################################



PROJECTDIR=`pwd`



cd $PROJECTDIR/src

RCFILE=$PROJECTDIR/coverage/.coveragerc

COVERAGEFILE=$PROJECTDIR/coverage/.coverage



#export CRDB_URI="cockroachdb://tfs:tfs123@127.0.0.1:26257/tfs_test?sslmode=require"

export CRDB_URI="cockroachdb://tfs:tfs123@10.1.7.195:26257/tfs_test?sslmode=require"

export PYTHONPATH=/home/tfs/tfs-ctrl/src

# Destroy old coverage file and configure the correct folder on the .coveragerc file

rm -f $COVERAGEFILE

cat $PROJECTDIR/coverage/.coveragerc.template | sed s+~/tfs-ctrl+$PROJECTDIR+g > $RCFILE



echo

echo "Pre-test clean-up:"

echo "------------------"

docker rm -f crdb nats

docker volume rm -f crdb

docker network rm tfs-br



# Run unitary tests and analyze coverage of code at same time

echo

echo "Pull Docker images:"

echo "-------------------"

docker pull cockroachdb/cockroach:latest-v22.2

docker pull nats:2.9



echo

echo "Create test environment:"

echo "------------------------"

docker network create -d bridge --subnet=172.254.254.0/24 --gateway=172.254.254.1 --ip-range=172.254.254.0/24 tfs-br

docker volume create crdb

docker run --name crdb -d --network=tfs-br --ip 172.254.254.10 -p 26257:26257 -p 8080:8080 \

    --env COCKROACH_DATABASE=tfs_test --env COCKROACH_USER=tfs --env COCKROACH_PASSWORD=tfs123\

    --volume "crdb:/cockroach/cockroach-data" \

    cockroachdb/cockroach:latest-v22.2 start-single-node

docker run --name nats -d --network=tfs-br --ip 172.254.254.11 -p 4222:4222 -p 8222:8222 \

    nats:2.9 --http_port 8222 --user tfs --pass tfs123

echo "Waiting for initialization..."

sleep 10

docker ps -a



echo

echo "Run unitary tests and analyze code coverage:"

echo "--------------------------------------------"

export CRDB_URI="cockroachdb://tfs:tfs123@172.254.254.10:26257/tfs_test?sslmode=require"

export MB_BACKEND="nats"

export NATS_URI="nats://tfs:tfs123@172.254.254.11:4222"

export PYTHONPATH=/home/tfs/tfs-ctrl/src

# helpful pytest flags: --log-level=INFO -o log_cli=true --verbose --maxfail=1 --durations=0

coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose --maxfail=1 \

    context/tests/test_unitary.py \

    context/tests/test_hasher.py
 
    context/tests/test_context.py \

    context/tests/test_topology.py

    #context/tests/test_*.py



echo

echo "Coverage report:"

echo "----------------"

#coverage report --rcfile=$RCFILE --sort cover --show-missing --skip-covered | grep --color -E -i "^context/.*$|$"

coverage report --rcfile=$RCFILE --sort cover --show-missing --skip-covered --include="context/*"



echo

echo "Post-test clean-up:"

echo "-------------------"

docker rm -f crdb nats

docker volume rm -f crdb

docker network rm tfs-br

src/context/.gitlab-ci.yml

+12 −3
Original line number Diff line number Diff line
@@ -51,10 +51,12 @@ unit test context: 
    - if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi

    - if docker container ls | grep crdb; then docker rm -f crdb; else echo "CockroachDB container is not in the system"; fi

    - if docker volume ls | grep crdb; then docker volume rm -f crdb; else echo "CockroachDB volume is not in the system"; fi

    - if docker container ls | grep nats; then docker rm -f nats; else echo "NATS container is not in the system"; fi

    - if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME container is not in the system"; fi

  script:

    - docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker pull "cockroachdb/cockroach:latest-v22.2"

    - docker pull "nats:2.9"

    - docker volume create crdb

    - >

      docker run --name crdb -d --network=teraflowbridge -p 26257:26257 -p 8080:8080
@@ -63,16 +65,24 @@ unit test context: 
      --env COCKROACH_PASSWORD=tfs123

      --volume "crdb:/cockroach/cockroach-data"

      cockroachdb/cockroach:latest-v22.2 start-single-node

    - >

      docker run --name nats -d --network=teraflowbridge -p 4222:4222 -p 8222:8222

      nats:2.9 --http_port 8222 --user tfs --pass tfs123

    - echo "Waiting for initialization..."

    - sleep 10

    - docker ps -a

    - CRDB_ADDRESS=$(docker inspect crdb --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")

    - NATS_ADDRESS=$(docker inspect nats --format "{{.NetworkSettings.Networks.teraflowbridge.IPAddress}}")

    - >

      docker run --name $IMAGE_NAME -d -p 1010:1010

      --env "CRDB_URI=cockroachdb://tfs:tfs123@${CRDB_ADDRESS}:26257/tfs_test?sslmode=require"

      --env "MB_BACKEND=nats"

      --env "NATS_URI=nats://tfs:tfs123@${NATS_ADDRESS}:4222"

      --volume "$PWD/src/$IMAGE_NAME/tests:/opt/results"

      --network=teraflowbridge

      $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG

    - docker ps -a

    - sleep 10

    - sleep 3

    - docker logs $IMAGE_NAME

    - >

      docker exec -i $IMAGE_NAME bash -c
@@ -80,8 +90,7 @@ unit test context: 
    - docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"

  coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'

  after_script:

    - docker rm -f $IMAGE_NAME

    - docker rm -f crdb

    - docker rm -f $IMAGE_NAME crdb nats

    - docker volume rm -f crdb

    - docker network rm teraflowbridge

  rules:

src/context/service/Constants.py

deleted100644 → 0
+0 −30
Original line number Diff line number Diff line 
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



TOPIC_CONNECTION = 'connection'

TOPIC_CONTEXT    = 'context'

TOPIC_DEVICE     = 'device'

TOPIC_LINK       = 'link'

#TOPIC_POLICY     = 'policy'

TOPIC_SERVICE    = 'service'

TOPIC_SLICE      = 'slice'

TOPIC_TOPOLOGY   = 'topology'



TOPICS = {

    TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK,

    #TOPIC_POLICY,

    TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY

}



CONSUME_TIMEOUT = 0.5 # seconds

src/context/service/ContextServiceServicerImpl.py

+25 −23
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ from common.proto.context_pb2 import ( 
    Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList,

    Context, ContextEvent, ContextId, ContextIdList, ContextList,

    Device, DeviceEvent, DeviceId, DeviceIdList, DeviceList,

    Empty,

    Empty, EventTypeEnum,

    Link, LinkEvent, LinkId, LinkIdList, LinkList,

    Service, ServiceEvent, ServiceId, ServiceIdList, ServiceList,

    Slice, SliceEvent, SliceId, SliceIdList, SliceList,
@@ -38,9 +38,9 @@ from .database.PolicyRule import ( 
from .database.Service import service_delete, service_get, service_list_ids, service_list_objs, service_set

from .database.Slice import slice_delete, slice_get, slice_list_ids, slice_list_objs, slice_set, slice_unset

from .database.Topology import topology_delete, topology_get, topology_list_ids, topology_list_objs, topology_set

from .Constants import (

from .Events import (

    CONSUME_TIMEOUT, TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, #TOPIC_POLICY,

    TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY)

    TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY, notify_event)



LOGGER = logging.getLogger(__name__)
@@ -60,28 +60,29 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 


    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def ListContextIds(self, request : Empty, context : grpc.ServicerContext) -> ContextIdList:

        return context_list_ids(self.db_engine)

        return ContextIdList(context_ids=context_list_ids(self.db_engine))



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def ListContexts(self, request : Empty, context : grpc.ServicerContext) -> ContextList:

        return context_list_objs(self.db_engine)

        return ContextList(contexts=context_list_objs(self.db_engine))



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def GetContext(self, request : ContextId, context : grpc.ServicerContext) -> Context:

        return context_get(self.db_engine, request)

        return Context(**context_get(self.db_engine, request))



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def SetContext(self, request : Context, context : grpc.ServicerContext) -> ContextId:

        context_id,updated = context_set(self.db_engine, request) # pylint: disable=unused-variable

        #event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

        #notify_event(self.messagebroker, TOPIC_CONTEXT, event_type, {'context_id': context_id})

        return context_id

        context_id,updated = context_set(self.db_engine, request)

        event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

        notify_event(self.messagebroker, TOPIC_CONTEXT, event_type, {'context_id': context_id})

        return ContextId(**context_id)



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def RemoveContext(self, request : ContextId, context : grpc.ServicerContext) -> Empty:

        deleted = context_delete(self.db_engine, request) # pylint: disable=unused-variable

        #if deleted:

        #    notify_event(self.messagebroker, TOPIC_CONTEXT, EventTypeEnum.EVENTTYPE_REMOVE, {'context_id': request})

        context_id,deleted = context_delete(self.db_engine, request)

        if deleted:

            event_type = EventTypeEnum.EVENTTYPE_REMOVE

            notify_event(self.messagebroker, TOPIC_CONTEXT, event_type, {'context_id': context_id})

        return Empty()



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
@@ -94,28 +95,29 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer 


    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def ListTopologyIds(self, request : ContextId, context : grpc.ServicerContext) -> TopologyIdList:

        return topology_list_ids(self.db_engine, request)

        return TopologyIdList(topology_ids=topology_list_ids(self.db_engine, request))



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def ListTopologies(self, request : ContextId, context : grpc.ServicerContext) -> TopologyList:

        return topology_list_objs(self.db_engine, request)

        return TopologyList(topologies=topology_list_objs(self.db_engine, request))



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def GetTopology(self, request : TopologyId, context : grpc.ServicerContext) -> Topology:

        return topology_get(self.db_engine, request)

        return Topology(**topology_get(self.db_engine, request))



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def SetTopology(self, request : Topology, context : grpc.ServicerContext) -> TopologyId:

        topology_id,updated = topology_set(self.db_engine, request) # pylint: disable=unused-variable

        #event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

        #notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': topology_id})

        return topology_id

        topology_id,updated = topology_set(self.db_engine, request)

        event_type = EventTypeEnum.EVENTTYPE_UPDATE if updated else EventTypeEnum.EVENTTYPE_CREATE

        notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': topology_id})

        return TopologyId(**topology_id)



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

    def RemoveTopology(self, request : TopologyId, context : grpc.ServicerContext) -> Empty:

        deleted = topology_delete(self.db_engine, request) # pylint: disable=unused-variable

        #if deleted:

        #    notify_event(self.messagebroker, TOPIC_TOPOLOGY, EventTypeEnum.EVENTTYPE_REMOVE, {'topology_id': request})

        topology_id,deleted = topology_delete(self.db_engine, request)

        if deleted:

            event_type = EventTypeEnum.EVENTTYPE_REMOVE

            notify_event(self.messagebroker, TOPIC_TOPOLOGY, event_type, {'topology_id': topology_id})

        return Empty()



    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)

src/context/service/Events.py

+18 −2
Original line number Diff line number Diff line
@@ -18,9 +18,25 @@ from common.message_broker.Message import Message 
from common.message_broker.MessageBroker import MessageBroker

from common.proto.context_pb2 import EventTypeEnum



def notify_event(

    messagebroker : MessageBroker, topic_name : str, event_type : EventTypeEnum, fields : Dict[str, str]) -> None:

TOPIC_CONNECTION = 'connection'

TOPIC_CONTEXT    = 'context'

TOPIC_DEVICE     = 'device'

TOPIC_LINK       = 'link'

#TOPIC_POLICY     = 'policy'

TOPIC_SERVICE    = 'service'

TOPIC_SLICE      = 'slice'

TOPIC_TOPOLOGY   = 'topology'



TOPICS = {

    TOPIC_CONNECTION, TOPIC_CONTEXT, TOPIC_DEVICE, TOPIC_LINK, #TOPIC_POLICY,

    TOPIC_SERVICE, TOPIC_SLICE, TOPIC_TOPOLOGY

}



CONSUME_TIMEOUT = 0.5 # seconds



def notify_event(

    messagebroker : MessageBroker, topic_name : str, event_type : EventTypeEnum, fields : Dict[str, str]

) -> None:

    event = {'event': {'timestamp': {'timestamp': time.time()}, 'event_type': event_type}}

    for field_name, field_value in fields.items():

        event[field_name] = field_value