Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • tfs/controller
1 result
Show changes
Commits on Source (69)
Hide whitespace changes
Inline Side-by-side
Showing
with 1372 additions and 2 deletions
.gitlab-ci.yml
View file @ c5b1bbeb
......@@ -35,12 +35,13 @@ include:
- local: '/src/ztp/.gitlab-ci.yml'
- local: '/src/policy/.gitlab-ci.yml'
- local: '/src/forecaster/.gitlab-ci.yml'
#- local: '/src/webui/.gitlab-ci.yml'
- local: '/src/webui/.gitlab-ci.yml'
#- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'
#- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml'
#- local: '/src/l3_attackmitigator/.gitlab-ci.yml'
#- local: '/src/slice/.gitlab-ci.yml'
- local: '/src/slice/.gitlab-ci.yml'
#- local: '/src/interdomain/.gitlab-ci.yml'
- local: '/src/pathcomp/.gitlab-ci.yml'
#- local: '/src/dlt/.gitlab-ci.yml'
- local: '/src/load_generator/.gitlab-ci.yml'
- local: '/src/bgpls_speaker/.gitlab-ci.yml'
manifests/bgpls_speakerservice.yaml 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: bgpls-speakerservice
spec:
selector:
matchLabels:
app: bgpls-speakerservice
replicas: 1
template:
metadata:
labels:
app: bgpls-speakerservice
spec:
terminationGracePeriodSeconds: 5
containers:
- name: server
image: localhost:32000/tfs/bgpls_speaker:dev
imagePullPolicy: Always
ports:
- containerPort: 20030
- containerPort: 9192
env:
- name: LOG_LEVEL
value: "INFO"
readinessProbe:
exec:
command: ["/bin/grpc_health_probe", "-addr=:20030"]
livenessProbe:
exec:
command: ["/bin/grpc_health_probe", "-addr=:20030"]
resources:
requests:
cpu: 250m
memory: 128Mi
limits:
cpu: 1000m
memory: 1024Mi
---
apiVersion: v1
kind: Service
metadata:
name: bgpls-speakerservice
labels:
app: bgpls-speakerservice
spec:
type: ClusterIP
selector:
app: bgpls-speakerservice
ports:
- name: grpc
protocol: TCP
port: 20030
targetPort: 20030
- name: metrics
protocol: TCP
port: 9192
targetPort: 9192
my_deploy.sh
View file @ c5b1bbeb
......@@ -25,6 +25,9 @@ export TFS_COMPONENTS="context device pathcomp service slice nbi webui load_gene
# Uncomment to activate Monitoring
#export TFS_COMPONENTS="${TFS_COMPONENTS} monitoring"
# Uncomment to activate bgpls_speaker
#export TFS_COMPONENTS="${TFS_COMPONENTS} bgpls_speaker"
# Uncomment to activate ZTP
#export TFS_COMPONENTS="${TFS_COMPONENTS} ztp"
......
proto/bgpls.proto 0 → 100644
View file @ c5b1bbeb
// Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package bgpls;
import "context.proto";
service BgplsService {
rpc ListDiscoveredDevices (context.Empty ) returns (DiscoveredDeviceList) {}
rpc ListDiscoveredLinks (context.Empty ) returns (DiscoveredLinkList ) {}
rpc AddBgplsSpeaker (BgplsSpeaker ) returns (BgplsSpeakerId ) {}
rpc ListBgplsSpeakers (context.Empty ) returns (BgplsSpeakerList ) {}
rpc DisconnectFromSpeaker (BgplsSpeaker ) returns (context.Empty ) {}
rpc GetSpeakerInfoFromId (BgplsSpeakerId ) returns (BgplsSpeaker ) {}
rpc NotifyAddNodeToContext(NodeDescriptors) returns (context.Empty ) {}
}
message DiscoveredDevice {
string nodeName = 1;
string ip = 2;
string igpID = 3;
string learntFrom = 4;
}
message DiscoveredDeviceList {
repeated DiscoveredDevice discovereddevices = 1;
}
message DiscoveredLinkList{
repeated DiscoveredLink discoveredlinks = 1;
}
message DiscoveredLink{
NodeDescriptors local = 1;
NodeDescriptors remote = 2;
string learntFrom = 3;
string local_ipv4 = 4;
string remote_ipv4 = 5;
}
message NodeDescriptors{
string asNumber = 1;
string igp_id = 2;
string nodeName = 3;
}
message BgplsSpeaker{
string address = 1;
string port = 2;
string asNumber = 3;
}
message BgplsSpeakerId{
uint32 id = 1;
}
message BgplsSpeakerList{
repeated BgplsSpeakerId speakers = 1;
}
scripts/show_logs_bgp.sh 0 → 100755
View file @ c5b1bbeb
#!/bin/bash
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
########################################################################################################################
# Define your deployment settings here
########################################################################################################################
# If not already set, set the name of the Kubernetes namespace to deploy to.
export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}
########################################################################################################################
# Automated steps start here
########################################################################################################################
kubectl --namespace $TFS_K8S_NAMESPACE logs deployment/bgpls_speakerservice -c server
src/bgpls_speaker/.gitlab-ci.yml 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Build, tag, and push the Docker image to the GitLab Docker registry
build bgpls_speaker:
variables:
IMAGE_NAME: 'bgpls_speaker' # name of the microservice
IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
stage: build
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
script:
- docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile .
- docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
- docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
after_script:
- docker images --filter="dangling=true" --quiet | xargs -r docker rmi
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
- changes:
- src/common/**/*.py
- proto/*.proto
- src/$IMAGE_NAME/**/*.{py,in,yml}
- src/$IMAGE_NAME/Dockerfile
- src/$IMAGE_NAME/tests/*.py
- manifests/${IMAGE_NAME}service.yaml
- .gitlab-ci.yml
# Apply unit test to the component
unit_test bgpls_speaker:
variables:
IMAGE_NAME: 'bgpls_speaker' # name of the microservice
IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
stage: unit_test
needs:
- build bgpls_speaker
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
- if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create --driver=bridge teraflowbridge; fi
- if docker container ls | grep $IMAGE_NAME; then docker rm -f $IMAGE_NAME; else echo "$IMAGE_NAME image is not in the system"; fi
script:
- docker pull "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
- docker run --name $IMAGE_NAME -d -p 20030:20030 -v "$PWD/src/$IMAGE_NAME/tests:/opt/results" --network=teraflowbridge $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
- sleep 5
- docker ps -a
- docker logs $IMAGE_NAME
- docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $IMAGE_NAME/tests/test_unitary.py --junitxml=/opt/results/${IMAGE_NAME}_report.xml"
- docker exec -i $IMAGE_NAME bash -c "coverage report --include='${IMAGE_NAME}/*' --show-missing"
coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'
after_script:
- docker rm -f $IMAGE_NAME
- docker network rm teraflowbridge
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
- changes:
- src/common/**/*.py
- proto/*.proto
- src/$IMAGE_NAME/**/*.{py,in,yml}
- src/$IMAGE_NAME/Dockerfile
- src/$IMAGE_NAME/tests/*.py
- manifests/${IMAGE_NAME}service.yaml
- .gitlab-ci.yml
artifacts:
when: always
reports:
junit: src/$IMAGE_NAME/tests/${IMAGE_NAME}_report.xml
## Deployment of the service in Kubernetes Cluster
#deploy bgpls_speaker:
# variables:
# IMAGE_NAME: 'bgpls_speaker' # name of the microservice
# IMAGE_TAG: 'latest' # tag of the container image (production, development, etc)
# stage: deploy
# needs:
# - unit test bgpls_speaker
# # - integ_test execute
# script:
# - 'sed -i "s/$IMAGE_NAME:.*/$IMAGE_NAME:$IMAGE_TAG/" manifests/${IMAGE_NAME}service.yaml'
# - kubectl version
# - kubectl get all
# - kubectl apply -f "manifests/${IMAGE_NAME}service.yaml"
# - kubectl get all
# # environment:
# # name: test
# # url: https://example.com
# # kubernetes:
# # namespace: test
# rules:
# - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'
# when: manual
# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'
# when: manual
src/bgpls_speaker/Config.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
src/bgpls_speaker/Dockerfile 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Maven install stage
#
# ----------------------------------------------
# FROM alpine/git:latest AS repo
# WORKDIR /usr/src/app
# RUN git clone https://github.com/telefonicaid/netphony-network-protocols.git .
FROM maven:3.8.8-eclipse-temurin-17 AS build
WORKDIR /
COPY src/bgpls_speaker/service/java/netphony-topology/ netphony-topology/
COPY src/bgpls_speaker/service/java/netphony-topology/pom.xml netphony-topology/pom.xml
WORKDIR /netphony-topology/
RUN mvn clean compile -DskipTests -X
RUN mvn package assembly:single -P bgp-ls-speaker -DskipTests
WORKDIR /netphony-topology/target/
# ENTRYPOINT [ "ls" ,"-a"]
# -------------------------------------------
# jar created in /netphony-topology/target/bgp-ls-speaker-jar-with-dependencies.jar
#
# Stage 2
#
FROM python:3.9-slim
# Install dependencies
RUN apt-get --yes --quiet --quiet update && \
apt-get --yes --quiet --quiet install wget g++ && \
rm -rf /var/lib/apt/lists/*
# Set Python to show logs as they occur
ENV PYTHONUNBUFFERED=0
# Download the gRPC health probe
RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
chmod +x /bin/grpc_health_probe
# Get generic Python packages
RUN python3 -m pip install --upgrade pip
RUN python3 -m pip install --upgrade setuptools wheel
RUN python3 -m pip install --upgrade pip-tools
# Install OpenJDK-11
RUN apt-get update && \
apt-get install -y openjdk-17-jre && \
apt-get clean;
# Get common Python packages
# Note: this step enables sharing the previous Docker build steps among all the Python components
WORKDIR /var/teraflow
COPY common_requirements.in common_requirements.in
RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in
RUN python3 -m pip install -r common_requirements.txt
# Add common files into working directory
WORKDIR /var/teraflow/common
COPY src/common/. ./
RUN rm -rf proto
# Create proto sub-folder, copy .proto files, and generate Python code
RUN mkdir -p /var/teraflow/common/proto
WORKDIR /var/teraflow/common/proto
RUN touch __init__.py
COPY proto/*.proto ./
RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto
RUN rm *.proto
RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;
# Create component sub-folders, get specific Python packages
RUN mkdir -p /var/teraflow/bgpls_speaker
WORKDIR /var/teraflow/bgpls_speaker
COPY src/bgpls_speaker/requirements.in requirements.in
RUN pip-compile --quiet --output-file=requirements.txt requirements.in
RUN python3 -m pip install -r requirements.txt
# Java module necessary config files
WORKDIR /var/teraflow/bgpls_speaker
RUN mkdir -p /java
COPY src/bgpls_speaker/service/java/* /java/
COPY --from=build /netphony-topology/target/bgp-ls-speaker-jar-with-dependencies.jar /var/teraflow/bgpls_speaker/service/java/bgp_ls.jar
# Add component files into working directory
WORKDIR /var/teraflow
COPY src/context/__init__.py context/__init__.py
COPY src/context/client/. context/client/
COPY src/bgpls_speaker/. bgpls_speaker/
# Start the service
ENTRYPOINT ["python", "-m", "bgpls_speaker.service"]
# ENTRYPOINT [ "ls","-R" ]
\ No newline at end of file
src/bgpls_speaker/__init__.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
src/bgpls_speaker/client/BgplsClient.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import grpc, logging
from common.Constants import ServiceNameEnum
from common.Settings import get_service_host, get_service_port_grpc
from common.proto.context_pb2 import Empty, Service, ServiceId
from common.proto.service_pb2_grpc import ServiceServiceStub
from common.proto.bgpls_pb2_grpc import BgplsServiceStub
from common.proto.bgpls_pb2 import BgplsSpeaker, DiscoveredDeviceList,DiscoveredLinkList,BgplsSpeakerId, NodeDescriptors
from common.tools.client.RetryDecorator import retry, delay_exponential
from common.tools.grpc.Tools import grpc_message_to_json_string
LOGGER = logging.getLogger(__name__)
MAX_RETRIES = 15
DELAY_FUNCTION = delay_exponential(initial=0.01, increment=2.0, maximum=5.0)
RETRY_DECORATOR = retry(max_retries=MAX_RETRIES, delay_function=DELAY_FUNCTION, prepare_method_name='connect')
class BgplsClient:
def __init__(self, host=None, port=None):
if not host: host = get_service_host(ServiceNameEnum.BGPLS)
if not port: port = get_service_port_grpc(ServiceNameEnum.BGPLS)
self.endpoint = '{:s}:{:s}'.format(str(host), str(port))
LOGGER.info('Creating channel to {:s}...'.format(str(self.endpoint)))
self.channel = None
self.stub = None
self.connect()
LOGGER.info('Channel created')
def connect(self):
self.channel = grpc.insecure_channel(self.endpoint)
self.stub = BgplsServiceStub(self.channel)
def close(self):
if self.channel is not None: self.channel.close()
self.channel = None
self.stub = None
@RETRY_DECORATOR
def ListDiscoveredDevices(self, request: Empty) -> DiscoveredDeviceList:
LOGGER.info('ListDiscoveredDevices request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.ListDiscoveredDevices(request)
LOGGER.info('ListDiscoveredDevices result: {:s}'.format(grpc_message_to_json_string(response)))
return response
@RETRY_DECORATOR
def ListDiscoveredLinks(self, request: Empty) -> DiscoveredLinkList:
LOGGER.info('ListDiscoveredDevices request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.ListDiscoveredLinks(request)
LOGGER.info('ListDiscoveredDevices result: {:s}'.format(grpc_message_to_json_string(response)))
return response
@RETRY_DECORATOR
def AddBgplsSpeaker(self, request: BgplsSpeaker) -> str:
LOGGER.info('AddBgplsSpeaker request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.AddBgplsSpeaker(request)
LOGGER.info('AddBgplsSpeaker result: {:s}'.format(grpc_message_to_json_string(response)))
return response
@RETRY_DECORATOR
def ListBgplsSpeakers(self, request: Empty) -> BgplsSpeakerId:
LOGGER.info('ListBgplsSpeakers request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.ListBgplsSpeakers(request)
LOGGER.info('ListBgplsSpeakers result: {:s}'.format(grpc_message_to_json_string(response)))
return response
@RETRY_DECORATOR
def DisconnectFromSpeaker(self, request: BgplsSpeaker) -> bool:
LOGGER.info('DisconnectFromSpeaker request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.DisconnectFromSpeaker(request)
LOGGER.info('DisconnectFromSpeaker result: {:s}'.format(grpc_message_to_json_string(response)))
return response
@RETRY_DECORATOR
def GetSpeakerInfoFromId(self, request: BgplsSpeakerId) -> BgplsSpeaker:
LOGGER.info('GetSpeakerInfoFromId request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.GetSpeakerInfoFromId(request)
LOGGER.info('GetSpeakerInfoFromId result: {:s}'.format(grpc_message_to_json_string(response)))
return response
@RETRY_DECORATOR
def NotifyAddNodeToContext(self, request: NodeDescriptors) -> str:
LOGGER.info('NotifyAddNodeToContext request: {:s}'.format(grpc_message_to_json_string(request)))
response = self.stub.NotifyAddNodeToContext(request)
LOGGER.info('NotifyAddNodeToContext result: {:s}'.format(grpc_message_to_json_string(response)))
return response
src/bgpls_speaker/client/__init__.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
src/bgpls_speaker/quick_deploy.sh 0 → 100644
View file @ c5b1bbeb
#!/bin/bash
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
########################################################################################################################
# Read deployment settings
########################################################################################################################
# ----- TeraFlowSDN ------------------------------------------------------------
# If not already set, set the URL of the Docker registry where the images will be uploaded to.
# By default, assume internal MicroK8s registry is used.
export TFS_REGISTRY_IMAGES=${TFS_REGISTRY_IMAGES:-"http://localhost:32000/tfs/"}
# If not already set, set the list of components, separated by spaces, you want to build images for, and deploy.
# By default, only basic components are deployed
export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device pathcomp service slice nbi webui load_generator bgpls_speaker"}
# If not already set, set the tag you want to use for your images.
export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}
# If not already set, set the name of the Kubernetes namespace to deploy TFS to.
export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}
# If not already set, set additional manifest files to be applied after the deployment
export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}
# If not already set, set the new Grafana admin password
export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}
# If not already set, disable skip-build flag to rebuild the Docker images.
# If TFS_SKIP_BUILD is "YES", the containers are not rebuilt-retagged-repushed and existing ones are used.
export TFS_SKIP_BUILD=${TFS_SKIP_BUILD:-"YES"}
# If TFS_SKIP_BUILD is "YES", select the containers to be build
# Any other container will use previous docker images
export TFS_QUICK_COMPONENTS="bgpls_speaker"
# ----- CockroachDB ------------------------------------------------------------
# If not already set, set the namespace where CockroackDB will be deployed.
export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"}
# If not already set, set the database username to be used by Context.
export CRDB_USERNAME=${CRDB_USERNAME:-"tfs"}
# If not already set, set the database user's password to be used by Context.
export CRDB_PASSWORD=${CRDB_PASSWORD:-"tfs123"}
# If not already set, set the database name to be used by Context.
export CRDB_DATABASE=${CRDB_DATABASE:-"tfs"}
# ----- NATS -------------------------------------------------------------------
# If not already set, set the namespace where NATS will be deployed.
export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"}
# ----- QuestDB ----------------------------------------------------------------
# If not already set, set the namespace where QuestDB will be deployed.
export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"}
# If not already set, set the database username to be used for QuestDB.
export QDB_USERNAME=${QDB_USERNAME:-"admin"}
# If not already set, set the database user's password to be used for QuestDB.
export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}
# If not already set, set the table name to be used by Monitoring for KPIs.
export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}
# If not already set, set the table name to be used by Slice for plotting groups.
export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}
########################################################################################################################
# Automated steps start here
########################################################################################################################
# Constants
GITLAB_REPO_URL="labs.etsi.org:5050/tfs/controller"
TMP_FOLDER="./tmp"
# Create a tmp folder for files modified during the deployment
TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"
mkdir -p $TMP_MANIFESTS_FOLDER
TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
mkdir -p $TMP_LOGS_FOLDER
echo "Deleting and Creating a new namespace..."
kubectl delete namespace $TFS_K8S_NAMESPACE --ignore-not-found
kubectl create namespace $TFS_K8S_NAMESPACE
printf "\n"
echo "Create secret with CockroachDB data"
CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
--from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \
--from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \
--from-literal=CRDB_DATABASE=${CRDB_DATABASE} \
--from-literal=CRDB_USERNAME=${CRDB_USERNAME} \
--from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \
--from-literal=CRDB_SSLMODE=require
printf "\n"
echo "Create secret with NATS data"
NATS_CLIENT_PORT=$(kubectl --namespace ${NATS_NAMESPACE} get service nats -o 'jsonpath={.spec.ports[?(@.name=="client")].port}')
kubectl create secret generic nats-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
--from-literal=NATS_NAMESPACE=${NATS_NAMESPACE} \
--from-literal=NATS_CLIENT_PORT=${NATS_CLIENT_PORT}
printf "\n"
echo "Create secret with QuestDB data"
QDB_HTTP_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="http")].port}')
QDB_ILP_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="ilp")].port}')
QDB_SQL_PORT=$(kubectl --namespace ${QDB_NAMESPACE} get service questdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')
METRICSDB_HOSTNAME="questdb-public.${QDB_NAMESPACE}.svc.cluster.local"
kubectl create secret generic qdb-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \
--from-literal=QDB_NAMESPACE=${QDB_NAMESPACE} \
--from-literal=METRICSDB_HOSTNAME=${METRICSDB_HOSTNAME} \
--from-literal=METRICSDB_REST_PORT=${QDB_HTTP_PORT} \
--from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \
--from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \
--from-literal=METRICSDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS} \
--from-literal=METRICSDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS} \
--from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \
--from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}
printf "\n"
echo "Deploying components and collecting environment variables..."
ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh
echo "# Environment variables for TeraFlowSDN deployment" > $ENV_VARS_SCRIPT
PYTHONPATH=$(pwd)/src
echo "export PYTHONPATH=${PYTHONPATH}" >> $ENV_VARS_SCRIPT
for COMPONENT in $TFS_COMPONENTS; do
echo "Processing '$COMPONENT' component..."
if [ "$TFS_SKIP_BUILD" != "YES" ]; then
echo " Building Docker image..."
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then
docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
elif [ "$COMPONENT" == "pathcomp" ]; then
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG"
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"
docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG"
# next command is redundant, but helpful to keep cache updated between rebuilds
IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder"
docker build -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
elif [ "$COMPONENT" == "dlt" ]; then
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log"
docker build -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG"
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log"
docker build -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG"
else
docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"
fi
echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."
if [ "$COMPONENT" == "pathcomp" ]; then
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"
docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
elif [ "$COMPONENT" == "dlt" ]; then
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log"
docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log"
docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
else
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
fi
else
for QUICK_COMPONENT in $TFS_QUICK_COMPONENTS; do
if [ "$COMPONENT" == "$QUICK_COMPONENT" ]; then
echo " Building Docker image..."
BUILD_LOG="$TMP_LOGS_FOLDER/build_${QUICK_COMPONENT}.log"
docker build -t "$QUICK_COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$QUICK_COMPONENT"/Dockerfile . > "$BUILD_LOG"
echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..."
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$QUICK_COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
TAG_LOG="$TMP_LOGS_FOLDER/tag_${QUICK_COMPONENT}.log"
docker tag "$QUICK_COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${QUICK_COMPONENT}.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
fi
done
fi
echo " Adapting '$COMPONENT' manifest file..."
MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"
cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"
if [ "$COMPONENT" == "pathcomp" ]; then
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f4)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f4)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
elif [ "$COMPONENT" == "dlt" ]; then
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f4)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f4)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
else
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGES/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g')
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f4)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
fi
sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"
# TODO: harmonize names of the monitoring component
echo " Deploying '$COMPONENT' component to Kubernetes..."
DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"
kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"
COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/")
#kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"
#kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"
echo " Collecting env-vars for '$COMPONENT' component..."
SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME}service --namespace $TFS_K8S_NAMESPACE -o json)
if [ -z "${SERVICE_DATA}" ]; then continue; fi
# Env vars for service's host address
SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP')
if [ -z "${SERVICE_HOST}" ]; then continue; fi
ENVVAR_HOST=$(echo "${COMPONENT}service_SERVICE_HOST" | tr '[:lower:]' '[:upper:]')
echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT
# Env vars for service's 'grpc' port (if any)
SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port')
if [ -n "${SERVICE_PORT_GRPC}" ]; then
ENVVAR_PORT_GRPC=$(echo "${COMPONENT}service_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]')
echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT
fi
# Env vars for service's 'http' port (if any)
SERVICE_PORT_HTTP=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="http") | .port')
if [ -n "${SERVICE_PORT_HTTP}" ]; then
ENVVAR_PORT_HTTP=$(echo "${COMPONENT}service_SERVICE_PORT_HTTP" | tr '[:lower:]' '[:upper:]')
echo "export ${ENVVAR_PORT_HTTP}=${SERVICE_PORT_HTTP}" >> $ENV_VARS_SCRIPT
fi
printf "\n"
done
echo "Deploying extra manifests..."
for EXTRA_MANIFEST in $TFS_EXTRA_MANIFESTS; do
echo "Processing manifest '$EXTRA_MANIFEST'..."
if [[ "$EXTRA_MANIFEST" == *"servicemonitor"* ]]; then
kubectl apply -f $EXTRA_MANIFEST
else
kubectl --namespace $TFS_K8S_NAMESPACE apply -f $EXTRA_MANIFEST
fi
printf "\n"
done
printf "\n"
for COMPONENT in $TFS_COMPONENTS; do
echo "Waiting for '$COMPONENT' component..."
COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/")
kubectl wait --namespace $TFS_K8S_NAMESPACE \
--for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME}service
printf "\n"
done
if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring"* ]]; then
echo "Configuring WebUI DataStores and Dashboards..."
sleep 5
# Exposed through the ingress controller "tfs-ingress"
GRAFANA_URL="127.0.0.1:80/grafana"
# Default Grafana credentials
GRAFANA_USERNAME="admin"
GRAFANA_PASSWORD="admin"
# Configure Grafana Admin Password
# Ref: https://grafana.com/docs/grafana/latest/http_api/user/#change-password
GRAFANA_URL_DEFAULT="http://${GRAFANA_USERNAME}:${GRAFANA_PASSWORD}@${GRAFANA_URL}"
echo ">> Updating Grafana 'admin' password..."
curl -X PUT -H "Content-Type: application/json" -d '{
"oldPassword": "'${GRAFANA_PASSWORD}'",
"newPassword": "'${TFS_GRAFANA_PASSWORD}'",
"confirmNew" : "'${TFS_GRAFANA_PASSWORD}'"
}' ${GRAFANA_URL_DEFAULT}/api/user/password
echo
echo
# Updated Grafana API URL
GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_URL}"
echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT
echo ">> Installing Scatter Plot plugin..."
curl -X POST -H "Content-Type: application/json" -H "Content-Length: 0" \
${GRAFANA_URL_UPDATED}/api/plugins/michaeldmoore-scatter-panel/install
echo
# Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/
QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}"
echo ">> Creating datasources..."
curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{
"access" : "proxy",
"type" : "postgres",
"name" : "questdb-mon-kpi",
"url" : "'${QDB_HOST_PORT}'",
"database" : "'${QDB_TABLE_MONITORING_KPIS}'",
"user" : "'${QDB_USERNAME}'",
"basicAuth": false,
"isDefault": true,
"jsonData" : {
"sslmode" : "disable",
"postgresVersion" : 1100,
"maxOpenConns" : 0,
"maxIdleConns" : 2,
"connMaxLifetime" : 14400,
"tlsAuth" : false,
"tlsAuthWithCACert" : false,
"timescaledb" : false,
"tlsConfigurationMethod": "file-path",
"tlsSkipVerify" : true
},
"secureJsonData": {"password": "'${QDB_PASSWORD}'"}
}' ${GRAFANA_URL_UPDATED}/api/datasources
echo
curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{
"access" : "proxy",
"type" : "postgres",
"name" : "questdb-slc-grp",
"url" : "'${QDB_HOST_PORT}'",
"database" : "'${QDB_TABLE_SLICE_GROUPS}'",
"user" : "'${QDB_USERNAME}'",
"basicAuth": false,
"isDefault": false,
"jsonData" : {
"sslmode" : "disable",
"postgresVersion" : 1100,
"maxOpenConns" : 0,
"maxIdleConns" : 2,
"connMaxLifetime" : 14400,
"tlsAuth" : false,
"tlsAuthWithCACert" : false,
"timescaledb" : false,
"tlsConfigurationMethod": "file-path",
"tlsSkipVerify" : true
},
"secureJsonData": {"password": "'${QDB_PASSWORD}'"}
}' ${GRAFANA_URL_UPDATED}/api/datasources
printf "\n\n"
echo ">> Creating dashboards..."
# Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/
curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_mon_kpis_psql.json' \
${GRAFANA_URL_UPDATED}/api/dashboards/db
echo
curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_slc_grps_psql.json' \
${GRAFANA_URL_UPDATED}/api/dashboards/db
printf "\n\n"
echo ">> Staring dashboards..."
DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-l3-monit"
DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')
curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
echo
DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-slice-grps"
DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')
curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
echo
printf "\n\n"
fi
src/bgpls_speaker/requirements.in 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
anytree==2.8.0
APScheduler==3.8.1
ncclient==0.6.13
python-json-logger==2.0.2
lxml==4.9.1
pytz==2021.3
xmltodict==0.12.0
grpcio==1.47.*
protobuf==3.20.*
\ No newline at end of file
src/bgpls_speaker/service/BgplsService.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from bgpls_speaker.service.tools.DiscoveredDBManager import DiscoveredDBManager
from bgpls_speaker.service.tools.GrpcServer import GrpcServer
from common.Constants import ServiceNameEnum
from common.Settings import get_service_port_grpc
from common.proto.bgpls_pb2_grpc import add_BgplsServiceServicer_to_server
from common.tools.service.GenericGrpcService import GenericGrpcService
from .BgplsServiceServicerImpl import BgplsServiceServicerImpl
class BgplsService(GenericGrpcService):
def __init__(self, discoveredDB : DiscoveredDBManager,
speakerServer : GrpcServer,cls_name: str = __name__) -> None:
port = get_service_port_grpc(ServiceNameEnum.BGPLS) # El enum en common.constants
super().__init__(port, cls_name=cls_name)
self.bgpls_servicer = BgplsServiceServicerImpl(discoveredDB,speakerServer)
def install_servicers(self):
add_BgplsServiceServicer_to_server(self.bgpls_servicer, self.server)
src/bgpls_speaker/service/BgplsServiceServicerImpl.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import grpc, json, logging
from typing import List, Tuple, Union
from bgpls_speaker.service.tools.DiscoveredDBManager import DiscoveredDBManager, GetContextDevices
from bgpls_speaker.service.tools.GrpcServer import GrpcServer
from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_method
from common.proto.context_pb2 import DeviceId, Empty, EndPointId, Link, LinkId, Uuid
from context.client.ContextClient import ContextClient
from common.proto.bgpls_pb2 import (
BgplsSpeaker, BgplsSpeakerId, DiscoveredDeviceList, DiscoveredDevice,
DiscoveredLink, DiscoveredLinkList, NodeDescriptors, BgplsSpeakerList
)
from common.proto.bgpls_pb2_grpc import BgplsServiceServicer
def json_to_list(json_str : str) -> List[Union[str, Tuple[str, str]]]:
try:
data = json.loads(json_str)
except: # pylint: disable=bare-except
return [('item', str(json_str))]
if isinstance(data, dict):
return [('kv', (key, value)) for key, value in data.items()]
elif isinstance(data, list):
return [('item', ', '.join(data))]
else:
return [('item', str(data))]
LOGGER = logging.getLogger(__name__)
METRICS_POOL = MetricsPool('Service', 'RPC')
class BgplsServiceServicerImpl(BgplsServiceServicer):
def __init__(self,discoveredDB : DiscoveredDBManager,
speakerServer : GrpcServer) -> None:
LOGGER.debug('Creating Servicer...')
self.speaker_handler_factory = 1
self.speaker_server=speakerServer
self.discoveredDB=discoveredDB
LOGGER.debug('Servicer Created')
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def ListDiscoveredDevices(self, request : Empty, context : grpc.ServicerContext) -> DiscoveredDeviceList:
"""
Get devices discovered from bgpls protocol
"""
device_names=self.discoveredDB.GetNodeNamesFromDiscoveredDB()
nodes = self.discoveredDB.GetNodesFromDiscoveredDB()
devices = [DiscoveredDevice(nodeName=node.node_name,igpID=node.igp_id,learntFrom=node.learnt_from) for node in nodes]
return DiscoveredDeviceList(discovereddevices=devices)
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def ListDiscoveredLinks(self, request : Empty, context : grpc.ServicerContext) -> DiscoveredLinkList:
"""
Get links discovered from bgpls protocol
"""
self.discoveredDB.UpdateNodeNameInLink()
links = self.discoveredDB.GetLinksFromDiscoveredDB()
links_info=[]
for link in links:
local=NodeDescriptors(igp_id=link.local_id,nodeName=link.local_id)
remote=NodeDescriptors(igp_id=link.remote_id,nodeName=link.remote_id)
links_info.append(DiscoveredLink(local=local,remote=remote,learntFrom=link.learnt_from,
local_ipv4=link.local_ipv4_id,remote_ipv4=link.remote_ipv4_id))
return DiscoveredLinkList(discoveredlinks=links_info)
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def AddBgplsSpeaker(self, request : BgplsSpeaker, context : grpc.ServicerContext) -> BgplsSpeakerId:
"""
Creates a new connection with an speaker with the given ip address, port and as.
Returns de id of the speaker created (to kill proccess¿)
"""
LOGGER.debug("(AddBgplsSpeaker) Create speaker instance %s",request)
speaker_id=self.speaker_server.connectToJavaBgpls(request.address,request.port,request.asNumber)
return BgplsSpeakerId(id=speaker_id)
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def ListBgplsSpeakers(self, request : Empty, context : grpc.ServicerContext) -> BgplsSpeakerId:
"""
Returns a list of the IDs of the BGP-LS speakers with open connections.
"""
speaker_list=[]
bgpls_speaker_list=[]
speaker_list=self.speaker_server.getSpeakerListIds()
for speaker in speaker_list:
bgpls_speaker_list.append(BgplsSpeakerId(id=speaker))
return BgplsSpeakerList(speakers=bgpls_speaker_list)
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def DisconnectFromSpeaker(self, request : BgplsSpeaker, context : grpc.ServicerContext) -> bool:
"""
Disconencts from the BGP-LS speaker given its ipv4 address.
"""
speaker_id=self.speaker_server.getSpeakerIdFromIpAddr(request.address)
self.speaker_server.terminateRunnerById(speaker_id)
return Empty()
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def GetSpeakerInfoFromId(self, request : BgplsSpeakerId, context : grpc.ServicerContext) -> BgplsSpeaker:
"""
Get the address, port and as number of the speaker given its id.
"""
address,as_number,port=self.speaker_server.getSpeakerFromId(request.id)
return BgplsSpeaker(address=address,port=port,asNumber=as_number)
@safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
def NotifyAddNodeToContext(self, request : DiscoveredDevice, context : grpc.ServicerContext) :
"""
When a node is added to context via bgpls module this function checks if there are other nodes in the
topology connected by links discovered via bgpls. Then, if the link exist adds it to the context.
TODO: get endpoints from pce module
"""
node_name=request.nodeName
node_igp=self.discoveredDB.GetIgpIdFromNodeName(node_name)
LOGGER.debug("(NotifyAddNodeToContext) Find links to nodes ")
nodes_conected=self.discoveredDB.FindConnectedNodes(node_igp)
# Check if nodes are in context
context_client=ContextClient()
context_client.connect()
# devices=context_client.ListDevices(Empty())
device_names,devices_ips=GetContextDevices(context_client)
LOGGER.debug("(NotifyAddNodeToContext) Devices in context: %s", device_names)
LOGGER.debug("(NotifyAddNodeToContext) Nodes conected in context: %s", nodes_conected)
nodes_conected_in_context=list(set(nodes_conected) & set(device_names))
LOGGER.debug("(NotifyAddNodeToContext) nodes_conected_in_context: %s", nodes_conected_in_context)
# TODO: next to function
for remote_node in nodes_conected_in_context:
# TODO: get endpoints connected to remote ip (pce¿)
end_point1="eth-1/0/20"
end_point2="eth-1/0/20"
end_point_uuid1=Uuid(uuid=end_point1)
end_point_uuid2=Uuid(uuid=end_point2)
link_name_src_dest=node_name+"/"+end_point1+"=="+remote_node+"/"+end_point2
device_uuid_src=DeviceId(device_uuid=Uuid(uuid=node_name))
device_src=context_client.GetDevice(device_uuid_src)
link_name_dest_src=remote_node+"/"+end_point2+"=="+node_name+"/"+end_point1
device_uuid_dest=DeviceId(device_uuid=Uuid(uuid=remote_node))
device_dest=context_client.GetDevice(device_uuid_dest)
self.getEndpointFromIpInterface(device_src,link.local_ipv4_id)
self.getEndpointFromIpInterface(device_dest,link.remote_ipv4_id)
# LOGGER.debug("(NotifyAddNodeToContext) Source: %s Destination: %s", device_src,device_dest)
end_point_id1=EndPointId(endpoint_uuid=end_point_uuid1,device_id=device_uuid_src)
end_point_id2=EndPointId(endpoint_uuid=end_point_uuid2,device_id=device_uuid_dest)
end_point_ids_src_dest=[end_point_id1,end_point_id2]
end_point_ids_dest_src=[end_point_id2,end_point_id1]
link_id_src=context_client.SetLink(Link(link_id=LinkId(link_uuid=Uuid(uuid=link_name_src_dest)),
link_endpoint_ids=end_point_ids_src_dest))
link_id_dst=context_client.SetLink(Link(link_id=LinkId(link_uuid=Uuid(uuid=link_name_dest_src)),
link_endpoint_ids=end_point_ids_dest_src))
LOGGER.debug("(NotifyAddNodeToContext) Link set id src--->dst: %s", link_id_src)
context_client.close()
return Empty()
def getEndpointFromIpInterface(self,device,ipv4):
"""
Get TFS endpoint from interface IPv4.
"""
for config in device.device_config.config_rules:
if config.WhichOneof('config_rule') == 'custom':
for item_type, item in json_to_list(config.custom.resource_value):
if item_type == 'kv':
# LOGGER.debug("(getEndpointFromIpInterface) item: %s",item)
endpoint=item
LOGGER.debug("(getEndpointFromIpInterface) config: %s",config.custom.resource_key)
if "/interface" in config.custom.resource_key:
interface=config.custom.resource_key.split("/interface")[1].strip("[]")
LOGGER.debug("(getEndpointFromIpInterface) interface: %s",interface)
if ipv4 in config.custom.resource_value:
LOGGER.debug("(getEndpointFromIpInterface) value: %s",config.custom.resource_value)
return endpoint
\ No newline at end of file
src/bgpls_speaker/service/__init__.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
\ No newline at end of file
src/bgpls_speaker/service/__main__.py 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging, signal, sys, threading
from prometheus_client import start_http_server
from common.Settings import get_log_level, get_metrics_port
from .tools.DiscoveredDBManager import DiscoveredDBManager
from .BgplsService import BgplsService
from .tools.GrpcServer import GrpcServer
terminate = threading.Event()
LOGGER : logging.Logger = None
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
def signal_handler(signal, frame):
LOGGER.warning('Terminate signal received')
LOGGER.warning(signal)
terminate.set()
def main():
global LOGGER
log_level = get_log_level()
logging.basicConfig(level=log_level, format="[%(asctime)s] %(levelname)s:%(name)s:%(message)s")
LOGGER = logging.getLogger(__name__)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
LOGGER.info('Starting...')
# Start metrics server
metrics_port = get_metrics_port()
start_http_server(metrics_port)
# One common database for all bgpls_speakers connection
DB=DiscoveredDBManager()
speaker_server = GrpcServer(DB)
speaker_server.Connect()
grpc_service = BgplsService(DB,speaker_server)
grpc_service.start()
# Wait for termination signal
while not terminate.wait(timeout=0.1): pass
LOGGER.info('Terminating...')
speaker_server.terminateGrpcServer()
grpc_service.stop()
LOGGER.info('Bye')
return 0
if __name__ == '__main__':
sys.exit(main())
src/bgpls_speaker/service/java/BGP4Parameters_3.xml 0 → 100644
View file @ c5b1bbeb
<!-- Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. -->
<config>
<!-- TCP port where the BGP is listening for incoming bgp4 connections. Optional Parameter. Default value: 179 (BGP Port) -->
<BGP4Port>12179</BGP4Port>
<BGPIdentifier>7.7.7.7</BGPIdentifier>
<!-- TCP port to connect to manage the BGP connection. Default value: 1112 -->
<BGP4ManagementPort>1112</BGP4ManagementPort>
<!-- Peers to which this Peer is going to establish connection -->
<configPeer>
<peer>10.95.90.43</peer>
<export>false</export>
<import>true</import>
<peerPort>179</peerPort>
</configPeer>
<!-- Ficheros log (servidor, protocolo PCEP y OSPF). Campos opcionales-->
<BGP4LogFile>BGP4Parser2.log</BGP4LogFile><!-- Default value: BGP4Parser.log -->
<BGP4LogFileClient>BGP4Client2.log</BGP4LogFileClient><!-- Default value: BGP4Client.log-->
<BGP4LogFileServer>BGP4Server2.log</BGP4LogFileServer><!-- Default value: BGP4Server.log-->
<!-- If the tcp no delay option is used or not. Optional Parameter. Default value: false. -->
<nodelay>true</nodelay>
<!-- Waiting Time to re-connect to clients. Default value: 6000 ms. -->
<delay>40000</delay>
<setTraces>true</setTraces>
<!-- OPEN Parameters -->
<!-- RFC 4271. This 2-octet unsigned integer indicates the number of seconds the sender proposes for the value of the Hold Timer.
Upon receipt of an OPEN message, a BGP speaker MUST calculate the value of the Hold Timer by using the smaller of its configured
Hold Time and the Hold Time received in the OPEN message. The Hold Time MUST be either zero or at least three seconds. An
implementation MAY reject connections on the basis of the Hold Time. The calculated value indicates the maximum number of
seconds that may elapse between the receipt of successive KEEPALIVE and/or UPDATE messages from the sender. -->
<holdTime>180</holdTime><!-- Optional Parameter. Default value: 3. -->
<!-- RFC 4271. This 1-octet unsigned integer indicates the protocol version number of the message. The current BGP version number is 4. -->
<version>4</version><!-- Optional Parameter. Default value: 4. -->
<!-- RFC 4271. This 2-octet unsigned integer indicates the Autonomous System number of the sender.-->
<myAutonomousSystem>65006</myAutonomousSystem>
<!-- RFC 4271. This 4-octet unsigned integer indicates the BGP Identifier of the sender. A given BGP speaker sets the value of its BGP
Identifier to an IP address that is assigned to that BGP speaker. The value of the BGP Identifier is determined upon
startup and is the same for every local interface and BGP peer. -->
<!--<BGPIdentifier>192.168.1.200</BGPIdentifier> -->
<!-- If the peer is in charge of sending its topology (only the interdomain Links) to the other BGP peer it is connected to. Default: false -->
<sendTopology>false</sendTopology>
<!-- If the peer is in charge of sending its whole topology to the other BGP peer it is connected to. Default: false -->
<sendIntradomainLinks>true</sendIntradomainLinks>
<!-- Optional Parameter. How to learn the topology. Possibilities: fromXML, fromBGP. Default: fromBGP -->
<learnTopology>fromBGP</learnTopology>
<!-- Topology network to read. It is mandatory if and only if learnTopology parameter is fromXML. -->
<!--<topologyFile>src/test/resources/network1.xml</topologyFile>-->
<!-- Optional Parameter. Instance Identifier for node and link NLRI. See rfc 6549. Default value: 0-->
<!--<instanceID>0</instanceID>-->
<!-- Optional Parameter. Default value: localhost -->
<localBGPAddress>0.0.0.0</localBGPAddress>
</config>
\ No newline at end of file
src/bgpls_speaker/service/java/exec_speakear_java.sh 0 → 100644
View file @ c5b1bbeb
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#!/bin/bash
cd netphony-topology/
"/home/ubuntu/downloads/apache-maven-3.8.8/bin/mvn" clean package -P bgp-ls-speaker assembly:single -DskipTests
cd ..
sudo java -jar netphony-topology/target/bgp-ls-speaker-jar-with-dependencies.jar BGP4Parameters_3.xml
\ No newline at end of file
src/bgpls_speaker/service/java/netphony-topology/.settings/org.eclipse.jdt.core.prefs 0 → 100644
View file @ c5b1bbeb
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8