Commit 752bbe52 authored by Luis de la Cal's avatar Luis de la Cal
Browse files

Merge branch 'develop' into feat/l3-components

parents 45b6c0b4 cd94fed3

deploy_component.sh

0 → 100755
+186 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.





########################################################################################################################

# Read deployment settings

########################################################################################################################



# If not already set, set the URL of your local Docker registry where the images will be uploaded to.

# Leave it blank if you do not want to use any Docker registry.

export TFS_REGISTRY_IMAGE=${TFS_REGISTRY_IMAGE:-""}

#export TFS_REGISTRY_IMAGE="http://my-container-registry.local/"



TFS_COMPONENTS=$1



# If not already set, set the tag you want to use for your images.

export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}



# If not already set, set the name of the Kubernetes namespace to deploy to.

export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}



# If not already set, set additional manifest files to be applied after the deployment

export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}



# If not already set, set the neew Grafana admin password

export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}



########################################################################################################################

# Automated steps start here

########################################################################################################################



# Constants

GITLAB_REPO_URL="registry.gitlab.com/teraflow-h2020/controller"

TMP_FOLDER="./tmp"



# Create a tmp folder for files modified during the deployment

TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"

TMP_LOGS_FOLDER="$TMP_FOLDER/logs"



echo "Deploying component and collecting environment variables..."

ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh



for COMPONENT in $TFS_COMPONENTS; do

    echo "Processing '$COMPONENT' component..."

    IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG"

    IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g')



    echo "  Building Docker image..."

    BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"



    if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then

        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"

    elif [ "$COMPONENT" == "pathcomp" ]; then

        BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"

        docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG"



        BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"

        docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

        # next command is redundant, but helpful to keep cache updated between rebuilds

        docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"

    else

        docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"

    fi



    if [ -n "$TFS_REGISTRY_IMAGE" ]; then

        echo "  Pushing Docker image to '$TFS_REGISTRY_IMAGE'..."



        if [ "$COMPONENT" == "pathcomp" ]; then

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"

            docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL-frontend" > "$TAG_LOG"



            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"

            docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL-backend" > "$TAG_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"

            docker push "$IMAGE_URL-frontend" > "$PUSH_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"

            docker push "$IMAGE_URL-backend" > "$PUSH_LOG"

        else

            TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"

            docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG"



            PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"

            docker push "$IMAGE_URL" > "$PUSH_LOG"

        fi

    fi



    echo "  Adapting '$COMPONENT' manifest file..."

    MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"

    cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"



    if [ -n "$TFS_REGISTRY_IMAGE" ]; then

        # Registry is set

        if [ "$COMPONENT" == "pathcomp" ]; then

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL-frontend#g" "$MANIFEST"



            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL-backend#g" "$MANIFEST"



            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"

        else

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"

            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"

        fi

    else

        # Registry is not set

        if [ "$COMPONENT" == "pathcomp" ]; then

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_NAME-frontend#g" "$MANIFEST"



            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_NAME-backend#g" "$MANIFEST"



            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"

        else

            VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)

            sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST"

            sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"

        fi

    fi



    # TODO: harmonize names of the monitoring component



    echo "  Deploying '$COMPONENT' component to Kubernetes..."

    DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"

    kubectl --namespace $TFS_K8S_NAMESPACE delete -f "$MANIFEST" > "$DEPLOY_LOG"

    kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"

    COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/")

    kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"

    kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"



    echo "  Collecting env-vars for '$COMPONENT' component..."



    SERVICE_DATA=$(kubectl get service ${COMPONENT}service --namespace $TFS_K8S_NAMESPACE -o json)

    if [ -z "${SERVICE_DATA}" ]; then continue; fi



    # Env vars for service's host address

    SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP')

    if [ -z "${SERVICE_HOST}" ]; then continue; fi

    # TODO: remove previous value from file

    ENVVAR_HOST=$(echo "${COMPONENT}service_SERVICE_HOST" | tr '[:lower:]' '[:upper:]')

    echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT



    # Env vars for service's 'grpc' port (if any)

    SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port')

    if [ -n "${SERVICE_PORT_GRPC}" ]; then

        ENVVAR_PORT_GRPC=$(echo "${COMPONENT}service_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]')

        echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT

    fi



    # Env vars for service's 'http' port (if any)

    SERVICE_PORT_HTTP=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="http") | .port')

    if [ -n "${SERVICE_PORT_HTTP}" ]; then

        ENVVAR_PORT_HTTP=$(echo "${COMPONENT}service_SERVICE_PORT_HTTP" | tr '[:lower:]' '[:upper:]')

        echo "export ${ENVVAR_PORT_HTTP}=${SERVICE_PORT_HTTP}" >> $ENV_VARS_SCRIPT

    fi



    printf "\n"

done



# By now, leave this control here. Some component dependencies are not well handled

for COMPONENT in $TFS_COMPONENTS; do

    echo "Waiting for '$COMPONENT' component..."

    kubectl wait --namespace $TFS_K8S_NAMESPACE \

        --for='condition=available' --timeout=300s deployment/${COMPONENT}service

    printf "\n"

done



./show_deploy.sh



echo "Done!"

src/compute/tests/mock_osm/WimconnectorIETFL2VPN.py

+1 −1
Original line number Diff line number Diff line
@@ -371,7 +371,7 @@ class WimconnectorIETFL2VPN(SdnConnectorBase): 
                    self.delete_connectivity_service(vpn_service["vpn-id"])



                    raise SdnConnectorError(

                        "Request no accepted",

                        "Request not accepted",

                        http_code=response_endpoint_site_network_access_creation.status_code,

                    )

            except requests.exceptions.ConnectionError:

src/service/service/service_handlers/l3nm_emulated/ConfigRules.py

0 → 100644
+137 −0
Original line number Diff line number Diff line 
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



from typing import Dict, List

from common.tools.object_factory.ConfigRule import json_config_rule_delete, json_config_rule_set

from service.service.service_handler_api.AnyTreeTools import TreeNode



def setup_config_rules(

    service_uuid : str, connection_uuid : str, device_uuid : str, endpoint_uuid : str,

    service_settings : TreeNode, endpoint_settings : TreeNode

) -> List[Dict]:



    json_settings          : Dict = {} if service_settings  is None else service_settings.value

    json_endpoint_settings : Dict = {} if endpoint_settings is None else endpoint_settings.value



    service_short_uuid        = service_uuid.split('-')[-1]

    network_instance_name     = '{:s}-NetInst'.format(service_short_uuid)

    network_interface_desc    = '{:s}-NetIf'.format(service_uuid)

    network_subinterface_desc = '{:s}-NetSubIf'.format(service_uuid)



    mtu                 = json_settings.get('mtu',                 1450 )    # 1512

    bgp_as              = json_settings.get('bgp_as',              0    )    # 65000

    bgp_route_target    = json_settings.get('bgp_route_target',    '0:0')    # 65000:333



    #router_id           = json_endpoint_settings.get('router_id',           '0.0.0.0')  # '10.95.0.10'

    route_distinguisher = json_endpoint_settings.get('route_distinguisher', '0:0'    )  # '60001:801'

    sub_interface_index = json_endpoint_settings.get('sub_interface_index', 0        )  # 1

    vlan_id             = json_endpoint_settings.get('vlan_id',             1        )  # 400

    address_ip          = json_endpoint_settings.get('address_ip',          '0.0.0.0')  # '2.2.2.1'

    address_prefix      = json_endpoint_settings.get('address_prefix',      24       )  # 30

    if_subif_name       = '{:s}.{:d}'.format(endpoint_uuid, vlan_id)



    json_config_rules = [

        json_config_rule_set(

            '/network_instance[{:s}]'.format(network_instance_name), {

                'name': network_instance_name, 'description': network_interface_desc, 'type': 'L3VRF',

                'route_distinguisher': route_distinguisher,

                #'router_id': router_id, 'address_families': address_families,

        }),

        json_config_rule_set(

            '/interface[{:s}]'.format(endpoint_uuid), {

                'name': endpoint_uuid, 'description': network_interface_desc, 'mtu': mtu,

        }),

        json_config_rule_set(

            '/interface[{:s}]/subinterface[{:d}]'.format(endpoint_uuid, sub_interface_index), {

                'name': endpoint_uuid, 'index': sub_interface_index,

                'description': network_subinterface_desc, 'vlan_id': vlan_id,

                'address_ip': address_ip, 'address_prefix': address_prefix,

        }),

        json_config_rule_set(

            '/network_instance[{:s}]/interface[{:s}]'.format(network_instance_name, if_subif_name), {

                'name': network_instance_name, 'id': if_subif_name, 'interface': endpoint_uuid,

                'subinterface': sub_interface_index,

        }),

        json_config_rule_set(

            '/network_instance[{:s}]/protocols[BGP]'.format(network_instance_name), {

                'name': network_instance_name, 'identifier': 'BGP', 'protocol_name': 'BGP', 'as': bgp_as,

        }),

        json_config_rule_set(

            '/network_instance[{:s}]/table_connections[STATIC][BGP][IPV4]'.format(network_instance_name), {

                'name': network_instance_name, 'src_protocol': 'STATIC', 'dst_protocol': 'BGP',

                'address_family': 'IPV4', #'default_import_policy': 'REJECT_ROUTE',

        }),

        json_config_rule_set(

            '/network_instance[{:s}]/table_connections[DIRECTLY_CONNECTED][BGP][IPV4]'.format(

                network_instance_name), {

                'name': network_instance_name, 'src_protocol': 'DIRECTLY_CONNECTED', 'dst_protocol': 'BGP',

                'address_family': 'IPV4', #'default_import_policy': 'REJECT_ROUTE',

        }),

        json_config_rule_set(

            '/routing_policy/bgp_defined_set[{:s}_rt_import]'.format(network_instance_name), {

                'ext_community_set_name': '{:s}_rt_import'.format(network_instance_name),

        }),

        json_config_rule_set(

            '/routing_policy/bgp_defined_set[{:s}_rt_import][route-target:{:s}]'.format(

                network_instance_name, bgp_route_target), {

                'ext_community_set_name': '{:s}_rt_import'.format(network_instance_name),

                'ext_community_member'  : 'route-target:{:s}'.format(bgp_route_target),

        }),

        json_config_rule_set(

            '/routing_policy/policy_definition[{:s}_import]'.format(network_instance_name), {

                'policy_name': '{:s}_import'.format(network_instance_name),

        }),

        json_config_rule_set(

            '/routing_policy/policy_definition[{:s}_import]/statement[{:s}]'.format(

                network_instance_name, '3'), {

                'policy_name': '{:s}_import'.format(network_instance_name), 'statement_name': '3',

                'ext_community_set_name': '{:s}_rt_import'.format(network_instance_name),

                'match_set_options': 'ANY', 'policy_result': 'ACCEPT_ROUTE',

        }),

        json_config_rule_set(

            # pylint: disable=duplicate-string-formatting-argument

            '/network_instance[{:s}]/inter_instance_policies[{:s}_import]'.format(

                network_instance_name, network_instance_name), {

                'name': network_instance_name, 'import_policy': '{:s}_import'.format(network_instance_name),

        }),

        json_config_rule_set(

            '/routing_policy/bgp_defined_set[{:s}_rt_export]'.format(network_instance_name), {

                'ext_community_set_name': '{:s}_rt_export'.format(network_instance_name),

        }),

        json_config_rule_set(

            '/routing_policy/bgp_defined_set[{:s}_rt_export][route-target:{:s}]'.format(

                network_instance_name, bgp_route_target), {

                'ext_community_set_name': '{:s}_rt_export'.format(network_instance_name),

                'ext_community_member'  : 'route-target:{:s}'.format(bgp_route_target),

        }),

        json_config_rule_set(

            '/routing_policy/policy_definition[{:s}_export]'.format(network_instance_name), {

                'policy_name': '{:s}_export'.format(network_instance_name),

        }),

        json_config_rule_set(

            '/routing_policy/policy_definition[{:s}_export]/statement[{:s}]'.format(

                network_instance_name, '3'), {

                'policy_name': '{:s}_export'.format(network_instance_name), 'statement_name': '3',

                'ext_community_set_name': '{:s}_rt_export'.format(network_instance_name),

                'match_set_options': 'ANY', 'policy_result': 'ACCEPT_ROUTE',

        }),

        json_config_rule_set(

            # pylint: disable=duplicate-string-formatting-argument

            '/network_instance[{:s}]/inter_instance_policies[{:s}_export]'.format(

                network_instance_name, network_instance_name), {

                'name': network_instance_name, 'export_policy': '{:s}_export'.format(network_instance_name),

        }),

    ]



    return json_config_rules
 
 No newline at end of file

src/service/service/service_handlers/l3nm_emulated/L3NMEmulatedServiceHandler.py

+158 −136

File changed.

Preview size limit exceeded, changes collapsed.