Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • tfs/controller
1 result
Show changes
Commits on Source (176)
Showing
with 654 additions and 76 deletions
......@@ -53,6 +53,7 @@ coverage.xml
.pytest_cache/
.benchmarks/
cover/
*_report.xml
# Translations
*.mo
......@@ -161,3 +162,7 @@ cython_debug/
# TeraFlowSDN-generated files
tfs_runtime_env_vars.sh
delete_local_deployment.sh
local_docker_deployment.sh
local_k8s_deployment.sh
.tool-versions
......@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
########################################################################################################################
# Read deployment settings
......@@ -25,7 +26,7 @@ export TFS_REGISTRY_IMAGE=${TFS_REGISTRY_IMAGE:-""}
# If not already set, set the list of components you want to build images for, and deploy.
# By default, only basic components are deployed
export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui"}
export TFS_COMPONENTS=${TFS_COMPONENTS:-"context device monitoring service compute webui te"}
# If not already set, set the tag you want to use for your images.
export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}
......@@ -54,22 +55,10 @@ TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
mkdir -p $TMP_LOGS_FOLDER
echo "Deleting and Creating a new namespace..."
kubectl delete namespace $TFS_K8S_NAMESPACE
kubectl delete --ignore-not-found=true namespace $TFS_K8S_NAMESPACE
kubectl create namespace $TFS_K8S_NAMESPACE
printf "\n"
if [[ "$TFS_COMPONENTS" == *"monitoring"* ]]; then
echo "Creating secrets for InfluxDB..."
#TODO: make sure to change this when having a production deployment
kubectl create secret generic influxdb-secrets --namespace=$TFS_K8S_NAMESPACE \
--from-literal=INFLUXDB_DB="monitoring" --from-literal=INFLUXDB_ADMIN_USER="teraflow" \
--from-literal=INFLUXDB_ADMIN_PASSWORD="teraflow" --from-literal=INFLUXDB_HTTP_AUTH_ENABLED="True"
kubectl create secret generic monitoring-secrets --namespace=$TFS_K8S_NAMESPACE \
--from-literal=INFLUXDB_DATABASE="monitoring" --from-literal=INFLUXDB_USER="teraflow" \
--from-literal=INFLUXDB_PASSWORD="teraflow" --from-literal=INFLUXDB_HOSTNAME="localhost"
printf "\n"
fi
echo "Deploying components and collecting environment variables..."
ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh
echo "# Environment variables for TeraFlowSDN deployment" > $ENV_VARS_SCRIPT
......@@ -158,6 +147,8 @@ for COMPONENT in $TFS_COMPONENTS; do
fi
fi
# TODO: harmonize names of the monitoring component
echo " Deploying '$COMPONENT' component to Kubernetes..."
DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"
kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"
......@@ -212,12 +203,12 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
echo "Configuring WebUI DataStores and Dashboards..."
sleep 3
INFLUXDB_HOST="monitoringservice"
INFLUXDB_PORT=$(kubectl --namespace $TFS_K8S_NAMESPACE get service/monitoringservice -o jsonpath='{.spec.ports[?(@.name=="influxdb")].port}')
INFLUXDB_URL="http://${INFLUXDB_HOST}:${INFLUXDB_PORT}"
INFLUXDB_USER=$(kubectl --namespace $TFS_K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_ADMIN_USER}' | base64 --decode)
INFLUXDB_PASSWORD=$(kubectl --namespace $TFS_K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_ADMIN_PASSWORD}' | base64 --decode)
INFLUXDB_DATABASE=$(kubectl --namespace $TFS_K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_DB}' | base64 --decode)
# INFLUXDB_HOST="monitoringservice"
# INFLUXDB_PORT=$(kubectl --namespace $TFS_K8S_NAMESPACE get service/monitoringservice -o jsonpath='{.spec.ports[?(@.name=="influxdb")].port}')
# INFLUXDB_URL="http://${INFLUXDB_HOST}:${INFLUXDB_PORT}"
# INFLUXDB_USER=$(kubectl --namespace $TFS_K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_ADMIN_USER}' | base64 --decode)
# INFLUXDB_PASSWORD=$(kubectl --namespace $TFS_K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_ADMIN_PASSWORD}' | base64 --decode)
# INFLUXDB_DATABASE=$(kubectl --namespace $TFS_K8S_NAMESPACE get secrets influxdb-secrets -o jsonpath='{.data.INFLUXDB_DB}' | base64 --decode)
# Exposed through the ingress controller "tfs-ingress"
GRAFANA_HOSTNAME="127.0.0.1"
......@@ -234,6 +225,8 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
# Updated Grafana API URL
GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_HOSTNAME}:${GRAFANA_PORT}${GRAFANA_BASEURL}"
echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT
echo "Connecting to grafana at URL: ${GRAFANA_URL_DEFAULT}..."
# Configure Grafana Admin Password
......@@ -245,26 +238,38 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
}' ${GRAFANA_URL_DEFAULT}/api/user/password
echo
# Create InfluxDB DataSource
# Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/
curl -X POST -H "Content-Type: application/json" -d '{
"type" : "influxdb",
"name" : "InfluxDB",
"url" : "'"$INFLUXDB_URL"'",
# TODO: replace user, password and database by variables to be saved
echo "Creating a datasource..."
curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{
"access" : "proxy",
"basicAuth": false,
"user" : "'"$INFLUXDB_USER"'",
"password" : "'"$INFLUXDB_PASSWORD"'",
"type" : "postgres",
"name" : "monitoringdb",
"url" : "monitoringservice:8812",
"database" : "monitoring",
"user" : "admin",
"password" : "quest",
"basicAuth" : false,
"isDefault": true,
"database" : "'"$INFLUXDB_DATABASE"'"
"jsonData" : {
"sslmode" : "disable",
"postgresVersion" : 1100,
"tlsAuth" : false,
"tlsAuthWithCACert": false,
"tlsConfigurationMethod": "file-path",
"tlsSkipVerify": true
},
"secureJsonFields" : {
"password" : true
}
}' ${GRAFANA_URL_UPDATED}/api/datasources
echo
# Create Monitoring Dashboard
# Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/
curl -X POST -H "Content-Type: application/json" \
-d '@src/webui/grafana_dashboard.json' \
${GRAFANA_URL_UPDATED}/api/dashboards/db
-d '@src/webui/grafana_dashboard.json' \
${GRAFANA_URL_UPDATED}/api/dashboards/db
echo
DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tf-l3-monit"
......
#!/bin/bash
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
########################################################################################################################
# Read deployment settings
########################################################################################################################
# If not already set, set the URL of your local Docker registry where the images will be uploaded to.
# Leave it blank if you do not want to use any Docker registry.
export TFS_REGISTRY_IMAGE=${TFS_REGISTRY_IMAGE:-""}
#export TFS_REGISTRY_IMAGE="http://my-container-registry.local/"
TFS_COMPONENTS=$1
# If not already set, set the tag you want to use for your images.
export TFS_IMAGE_TAG=${TFS_IMAGE_TAG:-"dev"}
# If not already set, set the name of the Kubernetes namespace to deploy to.
export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}
# If not already set, set additional manifest files to be applied after the deployment
export TFS_EXTRA_MANIFESTS=${TFS_EXTRA_MANIFESTS:-""}
# If not already set, set the neew Grafana admin password
export TFS_GRAFANA_PASSWORD=${TFS_GRAFANA_PASSWORD:-"admin123+"}
########################################################################################################################
# Automated steps start here
########################################################################################################################
# Constants
GITLAB_REPO_URL="registry.gitlab.com/teraflow-h2020/controller"
TMP_FOLDER="./tmp"
# Create a tmp folder for files modified during the deployment
TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests"
TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
echo "Deploying component and collecting environment variables..."
ENV_VARS_SCRIPT=tfs_runtime_env_vars.sh
for COMPONENT in $TFS_COMPONENTS; do
echo "Processing '$COMPONENT' component..."
IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG"
IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g')
echo " Building Docker image..."
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then
docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
elif [ "$COMPONENT" == "pathcomp" ]; then
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log"
docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG"
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log"
docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
# next command is redundant, but helpful to keep cache updated between rebuilds
docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG"
else
docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG"
fi
if [ -n "$TFS_REGISTRY_IMAGE" ]; then
echo " Pushing Docker image to '$TFS_REGISTRY_IMAGE'..."
if [ "$COMPONENT" == "pathcomp" ]; then
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log"
docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL-frontend" > "$TAG_LOG"
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log"
docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL-backend" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log"
docker push "$IMAGE_URL-frontend" > "$PUSH_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log"
docker push "$IMAGE_URL-backend" > "$PUSH_LOG"
else
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
fi
fi
echo " Adapting '$COMPONENT' manifest file..."
MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}service.yaml"
cp ./manifests/"${COMPONENT}"service.yaml "$MANIFEST"
if [ -n "$TFS_REGISTRY_IMAGE" ]; then
# Registry is set
if [ "$COMPONENT" == "pathcomp" ]; then
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL-frontend#g" "$MANIFEST"
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL-backend#g" "$MANIFEST"
sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"
else
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST"
sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST"
fi
else
# Registry is not set
if [ "$COMPONENT" == "pathcomp" ]; then
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_NAME-frontend#g" "$MANIFEST"
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_NAME-backend#g" "$MANIFEST"
sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"
else
VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3)
sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST"
sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST"
fi
fi
# TODO: harmonize names of the monitoring component
echo " Deploying '$COMPONENT' component to Kubernetes..."
DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log"
kubectl --namespace $TFS_K8S_NAMESPACE delete -f "$MANIFEST" > "$DEPLOY_LOG"
kubectl --namespace $TFS_K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG"
COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/")
kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"
kubectl --namespace $TFS_K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME}service >> "$DEPLOY_LOG"
echo " Collecting env-vars for '$COMPONENT' component..."
SERVICE_DATA=$(kubectl get service ${COMPONENT}service --namespace $TFS_K8S_NAMESPACE -o json)
if [ -z "${SERVICE_DATA}" ]; then continue; fi
# Env vars for service's host address
SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP')
if [ -z "${SERVICE_HOST}" ]; then continue; fi
# TODO: remove previous value from file
ENVVAR_HOST=$(echo "${COMPONENT}service_SERVICE_HOST" | tr '[:lower:]' '[:upper:]')
echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT
# Env vars for service's 'grpc' port (if any)
SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port')
if [ -n "${SERVICE_PORT_GRPC}" ]; then
ENVVAR_PORT_GRPC=$(echo "${COMPONENT}service_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]')
echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT
fi
# Env vars for service's 'http' port (if any)
SERVICE_PORT_HTTP=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="http") | .port')
if [ -n "${SERVICE_PORT_HTTP}" ]; then
ENVVAR_PORT_HTTP=$(echo "${COMPONENT}service_SERVICE_PORT_HTTP" | tr '[:lower:]' '[:upper:]')
echo "export ${ENVVAR_PORT_HTTP}=${SERVICE_PORT_HTTP}" >> $ENV_VARS_SCRIPT
fi
printf "\n"
done
# By now, leave this control here. Some component dependencies are not well handled
for COMPONENT in $TFS_COMPONENTS; do
echo "Waiting for '$COMPONENT' component..."
kubectl wait --namespace $TFS_K8S_NAMESPACE \
--for='condition=available' --timeout=300s deployment/${COMPONENT}service
printf "\n"
done
./show_deploy.sh
echo "Done!"
src/tests/ecoc22/
\ No newline at end of file
......@@ -34,7 +34,7 @@ spec:
- containerPort: 2020
env:
- name: LOG_LEVEL
value: "INFO"
value: "DEBUG"
readinessProbe:
exec:
command: ["/bin/grpc_health_probe", "-addr=:2020"]
......
......@@ -13,13 +13,14 @@
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
kind: StatefulSet
metadata:
name: monitoringservice
name: monitoringdb
spec:
selector:
matchLabels:
app: monitoringservice
serviceName: "monitoringservice"
replicas: 1
template:
metadata:
......@@ -32,35 +33,50 @@ spec:
- name: metricsdb
image: questdb/questdb
ports:
- containerPort: 9000
- containerPort: 9009
- containerPort: 9003
- name: http
containerPort: 9000
protocol: TCP
- name: influxdb
containerPort: 9009
protocol: TCP
- name: postgre
containerPort: 8812
protocol: TCP
env:
- name: QDB_CAIRO_COMMIT_LAG
value: "1000"
- name: QDB_CAIRO_MAX_UNCOMMITTED_ROWS
value: "100000"
readinessProbe:
exec:
command: ["curl", "-XGET", "localhost:9000"]
livenessProbe:
exec:
command: ["curl", "-XGET", "localhost:9003/metrics"]
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 700m
memory: 1024Mi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: monitoringserver
spec:
selector:
matchLabels:
app: monitoringservice
replicas: 1
template:
metadata:
labels:
app: monitoringservice
spec:
terminationGracePeriodSeconds: 5
restartPolicy: Always
containers:
- name: server
image: registry.gitlab.com/teraflow-h2020/controller/monitoring:latest
imagePullPolicy: Always
ports:
- containerPort: 7070
- name: grpc
containerPort: 7070
protocol: TCP
env:
- name: LOG_LEVEL
value: "INFO"
- name: METRICSDB_HOSTNAME
value: "localhost"
value: "monitoringservice"
- name: METRICSDB_ILP_PORT
value: "9009"
- name: METRICSDB_REST_PORT
......@@ -73,14 +89,6 @@ spec:
livenessProbe:
exec:
command: ["/bin/grpc_health_probe", "-addr=:7070"]
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 700m
memory: 1024Mi
---
apiVersion: v1
kind: Service
......@@ -95,7 +103,37 @@ spec:
protocol: TCP
port: 7070
targetPort: 7070
- name: questdb
- name: http
protocol: TCP
port: 9000
targetPort: 9000
\ No newline at end of file
targetPort: 9000
- name: influxdb
protocol: TCP
port: 9009
targetPort: 9009
- name: postgre
protocol: TCP
port: 8812
targetPort: 8812
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: access-monitoring
spec:
podSelector:
matchLabels:
app: monitoringservice
ingress:
- from: []
ports:
- port: 7070
- port: 8812
- from:
- podSelector:
matchLabels:
app: monitoringservice
ports:
- port: 9009
- port: 9000
......@@ -34,7 +34,7 @@ spec:
- containerPort: 3030
env:
- name: LOG_LEVEL
value: "INFO"
value: "DEBUG"
readinessProbe:
exec:
command: ["/bin/grpc_health_probe", "-addr=:3030"]
......
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: teservice
spec:
selector:
matchLabels:
app: teservice
template:
metadata:
labels:
app: teservice
spec:
terminationGracePeriodSeconds: 5
shareProcessNamespace: true
containers:
- name: server
image: registry.gitlab.com/teraflow-h2020/controller/te:latest
imagePullPolicy: Always
ports:
- containerPort: 11010
env:
- name: ERLANG_LOGGER_LEVEL
value: "debug"
- name: ERLANG_COOKIE
value: "tfte-unsafe-cookie"
- name: ERLANG_NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: ERLANG_NODE_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
readinessProbe:
exec:
command: ["/tfte/bin/tfte", "status"]
livenessProbe:
grpc:
port: 11010
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 700m
memory: 1024Mi
---
apiVersion: v1
kind: Service
metadata:
name: teservice
spec:
type: ClusterIP
selector:
app: teservice
ports:
- name: grpc
protocol: TCP
port: 11010
targetPort: 11010
- name: pcep
protocol: TCP
port: 4189
targetPort: 4189
......@@ -35,12 +35,12 @@ spec:
image: registry.gitlab.com/teraflow-h2020/controller/webui:latest
imagePullPolicy: Always
ports:
- containerPort: 8004 # TODO: define the real port
- containerPort: 8004
env:
- name: LOG_LEVEL
value: "DEBUG"
- name: WEBUISERVICE_SERVICE_BASEURL_HTTP
value: "/webui"
value: "/webui/"
readinessProbe:
httpGet:
path: /healthz/ready
......@@ -61,7 +61,7 @@ spec:
cpu: 700m
memory: 1024Mi
- name: grafana
image: grafana/grafana:8.2.6
image: grafana/grafana:8.5.11
imagePullPolicy: IfNotPresent
ports:
- containerPort: 3000
......
# Set the URL of your local Docker registry where the images will be uploaded to.
export TFS_REGISTRY_IMAGE="http://localhost:32000/tfs/"
# Set the list of components, separated by comas, you want to build images for, and deploy.
# Set the list of components, separated by spaces, you want to build images for, and deploy.
# Supported components are:
# context device automation policy service compute monitoring webui
# interdomain slice pathcomp dlt
# dbscanserving opticalattackmitigator opticalcentralizedattackdetector
# dbscanserving opticalattackmitigator opticalattackdetector
# l3_attackmitigator l3_centralizedattackdetector l3_distributedattackdetector
export TFS_COMPONENTS="context device automation service compute monitoring webui"
export TFS_COMPONENTS="context device automation pathcomp service slice compute monitoring webui te"
# Set the tag you want to use for your images.
export TFS_IMAGE_TAG="dev"
# Set the name of the Kubernetes namespace to deploy to.
export TFS_K8S_NAMESPACE="tfs"
# Set additional manifest files to be applied after the deployment
export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml"
# Set the neew Grafana admin password
export TFS_GRAFANA_PASSWORD="admin123+"
......@@ -3,5 +3,8 @@ src/*/*
# used to prevent breaking symbolic links from source code folders
!src/*/.gitignore
!src/python/__init__.py
!src/erlang/rebar.config
!src/erlang/rebar.lock
!src/erlang/src/tfpb.app.src
uml/generated
......@@ -51,6 +51,7 @@ service ContextService {
rpc ListServices (ContextId ) returns ( ServiceList ) {}
rpc GetService (ServiceId ) returns ( Service ) {}
rpc SetService (Service ) returns ( ServiceId ) {}
rpc UnsetService (Service ) returns ( ServiceId ) {}
rpc RemoveService (ServiceId ) returns ( Empty ) {}
rpc GetServiceEvents (Empty ) returns (stream ServiceEvent ) {}
......@@ -58,6 +59,7 @@ service ContextService {
rpc ListSlices (ContextId ) returns ( SliceList ) {}
rpc GetSlice (SliceId ) returns ( Slice ) {}
rpc SetSlice (Slice ) returns ( SliceId ) {}
rpc UnsetSlice (Slice ) returns ( SliceId ) {}
rpc RemoveSlice (SliceId ) returns ( Empty ) {}
rpc GetSliceEvents (Empty ) returns (stream SliceEvent ) {}
......@@ -188,6 +190,7 @@ message DeviceList {
message DeviceEvent {
Event event = 1;
DeviceId device_id = 2;
DeviceConfig device_config = 3;
}
......@@ -236,6 +239,7 @@ enum ServiceTypeEnum {
SERVICETYPE_L3NM = 1;
SERVICETYPE_L2NM = 2;
SERVICETYPE_TAPI_CONNECTIVITY_SERVICE = 3;
SERVICETYPE_TE = 4;
}
enum ServiceStatusEnum {
......
#!/bin/bash -eu
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
FORCE=0
DEFAULT_ACTION="generate"
usage() {
echo "Usage: $0 [-f] [clean|generate]" 1>&2
echo "Options:"
echo " -f: Force regeneration of all protocol buffers"
exit 1;
}
while getopts "fc" o; do
case "${o}" in
f)
FORCE=1
;;
*)
usage
;;
esac
done
shift $((OPTIND-1))
ACTION=${1:-$DEFAULT_ACTION}
cd $(dirname $0)
ROOT=$(pwd)
ERLANG_PROTO_DIR="$ROOT/src/erlang"
BUILD_CHECK="$ERLANG_PROTO_DIR/.generated"
tfpb_clean() {
rm -f "$BUILD_CHECK"
rm -rf "$ERLANG_PROTO_DIR/src/"*.erl
rm -rf "$ERLANG_PROTO_DIR/src/erlang/_build"
}
tfpb_generate() {
if [[ -f "$BUILD_CHECK" && $FORCE != 1 ]]; then
echo "Protocol buffer code for Erlang already generated, use -f to force"
exit 0
fi
tfpb_clean
mkdir -p "$ERLANG_PROTO_DIR"
cd "$ERLANG_PROTO_DIR"
rebar3 compile
rebar3 grpc gen
rebar3 compile
touch "$BUILD_CHECK"
echo "Protocol buffer code for Erlang generated"
}
case "$ACTION" in
clean) tfpb_clean;;
generate) tfpb_generate;;
*) usage;;
esac
*
!rebar.config
!rebar.lock
!src/tfpb.app.src
{erl_opts, [debug_info]}.
{deps, [grpcbox]}.
{grpc, [{protos, "../.."},
{gpb_opts, [{i, "../.."}, {strbin, true}, {descriptor, true}, {module_name_suffix, "_pb"}]}]}.
{plugins, [grpcbox_plugin]}.
{"1.2.0",
[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},1},
{<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.12.0">>},1},
{<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},1},
{<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
{<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.15.0">>},0},
{<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},2}]}.
[
{pkg_hash,[
{<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>},
{<<"chatterbox">>, <<"4E54F199E15C0320B85372A24E35554A2CCFC4342E0B7CD8DAED9A04F9B8EF4A">>},
{<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>},
{<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
{<<"grpcbox">>, <<"97C7126296A091602D372EBF5860A04F7BC795B45B33A984CAD2B8E362774FD8">>},
{<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}]},
{pkg_hash_ext,[
{<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>},
{<<"chatterbox">>, <<"6478C161BC60244F41CD5847CC3ACCD26D997883E9F7FACD36FF24533B2FA579">>},
{<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>},
{<<"gproc">>, <<"580ADAFA56463B75263EF5A5DF4C86AF321F68694E7786CB057FD805D1E2A7DE">>},
{<<"grpcbox">>, <<"161ABE9E17E7D1982EFA6488ADEAA13C3E847A07984A6E6B224E553368918647">>},
{<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}]}
].
#!/bin/bash
./report_coverage_all.sh | grep --color -E -i "^slice/.*$|$"
#!/bin/bash
########################################################################################################################
# Define your deployment settings here
########################################################################################################################
# Set the URL of your local Docker registry where the images will be uploaded to. Leave it blank if you do not want to
# use any Docker registry.
REGISTRY_IMAGE=""
#REGISTRY_IMAGE="http://my-container-registry.local/"
# Set the list of components you want to build images for, and deploy.
COMPONENTS="context device automation policy service compute monitoring centralizedattackdetector"
# Set the tag you want to use for your images.
IMAGE_TAG="tf-dev"
# Constants
TMP_FOLDER="./tmp"
TMP_LOGS_FOLDER="$TMP_FOLDER/logs"
mkdir -p $TMP_LOGS_FOLDER
for COMPONENT in $COMPONENTS; do
echo "Processing '$COMPONENT' component..."
IMAGE_NAME="$COMPONENT:$IMAGE_TAG"
IMAGE_URL="$REGISTRY_IMAGE/$IMAGE_NAME"
echo " Building Docker image..."
BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log"
if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then
docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG"
else
docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/ > "$BUILD_LOG"
fi
if [ -n "$REGISTRY_IMAGE" ]; then
echo "Pushing Docker image to '$REGISTRY_IMAGE'..."
TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log"
docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG"
PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log"
docker push "$IMAGE_URL" > "$PUSH_LOG"
fi
done
echo "Preparing for running the tests..."
if docker network list | grep teraflowbridge; then echo "teraflowbridge is already created"; else docker network create -d bridge teraflowbridge; fi
for COMPONENT in $COMPONENTS; do
IMAGE_NAME="$COMPONENT:$IMAGE_TAG"
echo " Running tests for $COMPONENT:"
docker run -it -d --name $COMPONENT $IMAGE_NAME --network=teraflowbridge
docker exec -it $COMPONENT bash -c "pytest --log-level=DEBUG --verbose $COMPONENT/tests/test_unitary.py"
docker stop $COMPONENT
done
#!/bin/bash
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
die () {
echo >&2 "$@"
exit 1
}
[ "$#" -eq 1 ] || die "component name required but not provided"
COMPONENT_NAME=$1 # parameter
IMAGE_NAME="${COMPONENT_NAME}-local"
IMAGE_TAG="latest"
if docker ps | grep $IMAGE_NAME
then
docker stop $IMAGE_NAME
fi
if docker network list | grep teraflowbridge
then
echo "teraflowbridge is already created"
else
docker network create -d bridge teraflowbridge
fi
docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$COMPONENT_NAME/Dockerfile .
docker run --name $IMAGE_NAME -d -v "${PWD}/src/${COMPONENT_NAME}/tests:/home/${COMPONENT_NAME}/results" --network=teraflowbridge --rm $IMAGE_NAME:$IMAGE_TAG
docker exec -i $IMAGE_NAME bash -c "coverage run -m pytest --log-level=INFO --verbose $COMPONENT_NAME/tests/ --junitxml=/home/${COMPONENT_NAME}/results/${COMPONENT_NAME}_report.xml"
PROJECTDIR=`pwd`
cd $PROJECTDIR/src
RCFILE=$PROJECTDIR/coverage/.coveragerc
echo
echo "Coverage report:"
echo "----------------"
docker exec -i $IMAGE_NAME bash -c "coverage report --include='${COMPONENT_NAME}/*' --show-missing"
# docker stop $IMAGE_NAME
docker rm -f $IMAGE_NAME
docker network rm teraflowbridge
#!/bin/bash
# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
########################################################################################################################
# Define your deployment settings here
########################################################################################################################
# If not already set, set the name of the Kubernetes namespace to deploy to.
export TFS_K8S_NAMESPACE=${TFS_K8S_NAMESPACE:-"tfs"}
########################################################################################################################
# Automated steps start here
########################################################################################################################
mkdir -p tmp/exec_logs/$TFS_K8S_NAMESPACE/
rm tmp/exec_logs/$TFS_K8S_NAMESPACE/*
PODS=$(kubectl get pods --namespace $TFS_K8S_NAMESPACE --no-headers --output=custom-columns=":metadata.name")
for POD in $PODS; do
CONTAINERS=$(kubectl get pods --namespace $TFS_K8S_NAMESPACE $POD -o jsonpath='{.spec.containers[*].name}')
for CONTAINER in $CONTAINERS; do
kubectl --namespace $TFS_K8S_NAMESPACE logs pod/${POD} --container ${CONTAINER} \
> tmp/exec_logs/$TFS_K8S_NAMESPACE/$POD\_\_$CONTAINER.log
done
done