Commit 0a1b6178 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

DLT: 

MockBlockchain:
- Implemented mock-blockchain module to facilitate testing and debugging of DLT connector & workflows
- Added manifests, Dockerfile, etc.

DLT Connector:
- corrected requirements.in
- updated Dockerfile
- WIP: implementation of test_unitary.py

DLT Gateway:
- Improved/extended Dockerfile
- Moved Dockerfile to correct folder
- Corrected import path for proto files
parent abce48da

.gitignore

+1 −0
Original line number Diff line number Diff line
@@ -162,6 +162,7 @@ cython_debug/ 


# TeraFlowSDN-generated files

tfs_runtime_env_vars.sh

tfs_bchain_runtime_env_vars.sh

delete_local_deployment.sh

local_docker_deployment.sh

local_k8s_deployment.sh

deploy_mock_blockchain.sh

0 → 100755
+121 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.





########################################################################################################################

# Read deployment settings

########################################################################################################################



# Set the URL of your local Docker registry where the images will be uploaded to.

REGISTRY_IMAGE="http://localhost:32000/tfs/"



# Set the tag you want to use for your images.

IMAGE_TAG="dev"



# Set the name of the Kubernetes namespace to deploy to.

K8S_NAMESPACE="tfs-bchain"



COMPONENT="mock_blockchain"



########################################################################################################################

# Automated steps start here

########################################################################################################################



# Constants

GITLAB_REPO_URL="registry.gitlab.com/teraflow-h2020/controller"

TMP_FOLDER="./tmp"



# Create a tmp folder for files modified during the deployment

TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"

mkdir -p $TMP_MANIFESTS_FOLDER

TMP_LOGS_FOLDER="$TMP_FOLDER/logs"

mkdir -p $TMP_LOGS_FOLDER



echo "Deleting and Creating a new namespace..."

kubectl delete namespace $K8S_NAMESPACE

kubectl create namespace $K8S_NAMESPACE

printf "\n"



echo "Deploying components and collecting environment variables..."

ENV_VARS_SCRIPT=tfs_bchain_runtime_env_vars.sh

echo "# Environment variables for TeraFlow Mock-Blockchain deployment" > $ENV_VARS_SCRIPT

PYTHONPATH=$(pwd)/src



echo "Processing '$COMPONENT' component..."

IMAGE_NAME="$COMPONENT:$IMAGE_TAG"

IMAGE_URL=$(echo "$REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g')



echo "  Building Docker image..."

BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"

docker build -t "$IMAGE_NAME" -f ./src/dlt/mock_blockchain/Dockerfile . > "$BUILD_LOG"



if [ -n "$REGISTRY_IMAGE" ]; then

    echo "  Pushing Docker image to '$REGISTRY_IMAGE'..."



    TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"

    docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG"



    PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"

    docker push "$IMAGE_URL" > "$PUSH_LOG"

fi



echo "  Adapting '$COMPONENT' manifest file..."

MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}.yaml"

cp ./manifests/"${COMPONENT}".yaml "$MANIFEST"



if [ -n "$REGISTRY_IMAGE" ]; then

    # Registry is set

    VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)

    sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

    sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"

else

    # Registry is not set

    VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)

    sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST"

    sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"

fi



echo "  Deploying '$COMPONENT' component to Kubernetes..."

DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"

kubectl --namespace $K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"

COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/")

kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG"

kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG"



echo "  Collecting env-vars for '$COMPONENT' component..."

SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME} --namespace $K8S_NAMESPACE -o json)



# Env vars for service's host address

SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP')

ENVVAR_HOST=$(echo "${COMPONENT}_SERVICE_HOST" | tr '[:lower:]' '[:upper:]')

echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT



# Env vars for service's 'grpc' port

SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port')

ENVVAR_PORT_GRPC=$(echo "${COMPONENT}_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]')

echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT



printf "\n"



echo "Waiting for '$COMPONENT' component..."

kubectl wait --namespace $K8S_NAMESPACE \

    --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME}

printf "\n"



echo "Deployment Resources:"

kubectl --namespace $K8S_NAMESPACE get all

printf "\n"



echo "Done!"

manifests/mock_blockchain.yaml

0 → 100644
+64 −0
Original line number Diff line number Diff line 
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



apiVersion: apps/v1

kind: Deployment

metadata:

  name: mock-blockchain

spec:

  selector:

    matchLabels:

      app: mock-blockchain

  template:

    metadata:

      labels:

        app: mock-blockchain

    spec:

      terminationGracePeriodSeconds: 5

      containers:

      - name: server

        image: registry.gitlab.com/teraflow-h2020/controller/mock_blockchain:latest

        imagePullPolicy: Always

        ports:

        - containerPort: 50051

        env:

        - name: LOG_LEVEL

          value: "DEBUG"

        readinessProbe:

          exec:

            command: ["/bin/grpc_health_probe", "-addr=:50051"]

        livenessProbe:

          exec:

            command: ["/bin/grpc_health_probe", "-addr=:50051"]

        resources:

          requests:

            cpu: 250m

            memory: 512Mi

          limits:

            cpu: 700m

            memory: 1024Mi

---

apiVersion: v1

kind: Service

metadata:

  name: mock-blockchain

spec:

  type: ClusterIP

  selector:

    app: mock-blockchain

  ports:

  - name: grpc

    protocol: TCP

    port: 50051

    targetPort: 50051

src/common/Constants.py

+6 −0
Original line number Diff line number Diff line
@@ -49,6 +49,9 @@ class ServiceNameEnum(Enum): 
    PATHCOMP      = 'pathcomp'

    WEBUI         = 'webui'



    # Used for test and debugging only

    DLT_GATEWAY   = 'dlt-gateway'



# Default gRPC service ports

DEFAULT_SERVICE_GRPC_PORTS = {

    ServiceNameEnum.CONTEXT      .value :  1010,
@@ -63,6 +66,9 @@ DEFAULT_SERVICE_GRPC_PORTS = { 
    ServiceNameEnum.CYBERSECURITY.value : 10000,

    ServiceNameEnum.INTERDOMAIN  .value : 10010,

    ServiceNameEnum.PATHCOMP     .value : 10020,



    # Used for test and debugging only

    ServiceNameEnum.DLT_GATEWAY  .value : 50051,

}



# Default HTTP/REST-API service ports

src/common/tests/MockServicerImpl_DltGateway.py

+3 −3
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ 
# limitations under the License.



import grpc, itertools, json, logging, time

from typing import Dict, Iterator, Optional, Tuple

from typing import Any, Dict, Iterator, Optional, Tuple

from common.tests.MockMessageBroker import Message, MockMessageBroker

from common.tools.grpc.Tools import grpc_message_to_json_string

from common.proto.context_pb2 import EVENTTYPE_CREATE, EVENTTYPE_REMOVE, EVENTTYPE_UPDATE, Empty, TeraFlowController
@@ -27,7 +27,7 @@ from common.proto.dlt_gateway_pb2_grpc import DltGatewayServiceServicer 


LOGGER = logging.getLogger(__name__)



DltRecordKey  = Tuple[str, DltRecordOperationEnum, str]     # domain_uuid, operation, record_uuid

DltRecordKey  = Tuple[str, Any, str]            # domain_uuid, DltRecordOperationEnum, record_uuid

DltRecordDict = Dict[DltRecordKey, DltRecord]   # dlt_record_key => dlt_record



class AlreadyExistsException(Exception):