Loading .gitignore +1 −0 Original line number Diff line number Diff line Loading @@ -162,6 +162,7 @@ cython_debug/ # TeraFlowSDN-generated files tfs_runtime_env_vars.sh tfs_bchain_runtime_env_vars.sh delete_local_deployment.sh local_docker_deployment.sh local_k8s_deployment.sh Loading .gitlab-ci.yml +1 −0 Original line number Diff line number Diff line Loading @@ -42,3 +42,4 @@ include: #- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml' deploy.sh +70 −26 Original line number Diff line number Diff line Loading @@ -66,44 +66,71 @@ echo "export PYTHONPATH=${PYTHONPATH}" >> $ENV_VARS_SCRIPT for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" elif [ "$COMPONENT" == "pathcomp" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG" docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" # next command is redundant, but helpful to keep cache updated between rebuilds docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" docker build -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" elif [ "$COMPONENT" == "dlt" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" docker build -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" docker build -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" else docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" fi if [ -n "$TFS_REGISTRY_IMAGE" ]; then echo " Pushing Docker image to '$TFS_REGISTRY_IMAGE'..." if [ "$COMPONENT" == "pathcomp" ]; then TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL-frontend" > "$TAG_LOG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL-backend" > "$TAG_LOG" TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log" docker push "$IMAGE_URL-frontend" > "$PUSH_LOG" docker push "$IMAGE_URL" > "$PUSH_LOG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log" docker push "$IMAGE_URL-backend" > "$PUSH_LOG" docker push "$IMAGE_URL" > "$PUSH_LOG" elif [ "$COMPONENT" == "dlt" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log" docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log" docker push "$IMAGE_URL" > "$PUSH_LOG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log" docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log" docker push "$IMAGE_URL" > "$PUSH_LOG" else IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" Loading @@ -117,33 +144,48 @@ for COMPONENT in $TFS_COMPONENTS; do if [ -n "$TFS_REGISTRY_IMAGE" ]; then # Registry is set if [ "$COMPONENT" == "pathcomp" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL-frontend#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL-backend#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" elif [ "$COMPONENT" == "dlt" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" else IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" fi sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" else # Registry is not set if [ "$COMPONENT" == "pathcomp" ]; then VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_NAME-frontend#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $COMPONENT-frontend:$TFS_IMAGE_TAG#g" "$MANIFEST" VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_NAME-backend#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $COMPONENT-backend:$TFS_IMAGE_TAG#g" "$MANIFEST" elif [ "$COMPONENT" == "dlt" ]; then VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $COMPONENT-connector:$TFS_IMAGE_TAG#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $COMPONENT-gateway:$TFS_IMAGE_TAG#g" "$MANIFEST" else VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $COMPONENT:$TFS_IMAGE_TAG#g" "$MANIFEST" fi sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" fi # TODO: harmonize names of the monitoring component Loading @@ -157,7 +199,7 @@ for COMPONENT in $TFS_COMPONENTS; do echo " Collecting env-vars for '$COMPONENT' component..." SERVICE_DATA=$(kubectl get service ${COMPONENT}service --namespace $TFS_K8S_NAMESPACE -o json) SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME}service --namespace $TFS_K8S_NAMESPACE -o json) if [ -z "${SERVICE_DATA}" ]; then continue; fi # Env vars for service's host address Loading Loading @@ -189,6 +231,7 @@ for EXTRA_MANIFEST in $TFS_EXTRA_MANIFESTS; do kubectl --namespace $TFS_K8S_NAMESPACE apply -f $EXTRA_MANIFEST printf "\n" done printf "\n" # By now, leave these controls here. Some component dependencies are not well handled. Loading @@ -203,8 +246,9 @@ fi for COMPONENT in $TFS_COMPONENTS; do echo "Waiting for '$COMPONENT' component..." COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl wait --namespace $TFS_K8S_NAMESPACE \ --for='condition=available' --timeout=300s deployment/${COMPONENT}service --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME}service printf "\n" done Loading deploy_mock_blockchain.sh 0 → 100755 +121 −0 Original line number Diff line number Diff line #!/bin/bash # Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ######################################################################################################################## # Read deployment settings ######################################################################################################################## # Set the URL of your local Docker registry where the images will be uploaded to. REGISTRY_IMAGE="http://localhost:32000/tfs/" # Set the tag you want to use for your images. IMAGE_TAG="dev" # Set the name of the Kubernetes namespace to deploy to. K8S_NAMESPACE="tfs-bchain" COMPONENT="mock_blockchain" ######################################################################################################################## # Automated steps start here ######################################################################################################################## # Constants GITLAB_REPO_URL="registry.gitlab.com/teraflow-h2020/controller" TMP_FOLDER="./tmp" # Create a tmp folder for files modified during the deployment TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests" mkdir -p $TMP_MANIFESTS_FOLDER TMP_LOGS_FOLDER="$TMP_FOLDER/logs" mkdir -p $TMP_LOGS_FOLDER echo "Deleting and Creating a new namespace..." kubectl delete namespace $K8S_NAMESPACE kubectl create namespace $K8S_NAMESPACE printf "\n" echo "Deploying components and collecting environment variables..." ENV_VARS_SCRIPT=tfs_bchain_runtime_env_vars.sh echo "# Environment variables for TeraFlow Mock-Blockchain deployment" > $ENV_VARS_SCRIPT PYTHONPATH=$(pwd)/src echo "Processing '$COMPONENT' component..." IMAGE_NAME="$COMPONENT:$IMAGE_TAG" IMAGE_URL=$(echo "$REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" docker build -t "$IMAGE_NAME" -f ./src/dlt/mock_blockchain/Dockerfile . > "$BUILD_LOG" if [ -n "$REGISTRY_IMAGE" ]; then echo " Pushing Docker image to '$REGISTRY_IMAGE'..." TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" fi echo " Adapting '$COMPONENT' manifest file..." MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}.yaml" cp ./manifests/"${COMPONENT}".yaml "$MANIFEST" if [ -n "$REGISTRY_IMAGE" ]; then # Registry is set VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" else # Registry is not set VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" fi echo " Deploying '$COMPONENT' component to Kubernetes..." DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log" kubectl --namespace $K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG" COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG" kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG" echo " Collecting env-vars for '$COMPONENT' component..." SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME} --namespace $K8S_NAMESPACE -o json) # Env vars for service's host address SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP') ENVVAR_HOST=$(echo "${COMPONENT}_SERVICE_HOST" | tr '[:lower:]' '[:upper:]') echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT # Env vars for service's 'grpc' port SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port') ENVVAR_PORT_GRPC=$(echo "${COMPONENT}_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]') echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT printf "\n" echo "Waiting for '$COMPONENT' component..." kubectl wait --namespace $K8S_NAMESPACE \ --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME} printf "\n" echo "Deployment Resources:" kubectl --namespace $K8S_NAMESPACE get all printf "\n" echo "Done!" manifests/dltservice.yaml 0 → 100644 +86 −0 Original line number Diff line number Diff line # Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: apps/v1 kind: Deployment metadata: name: dltservice spec: selector: matchLabels: app: dltservice template: metadata: labels: app: dltservice spec: terminationGracePeriodSeconds: 5 containers: - name: connector image: registry.gitlab.com/teraflow-h2020/controller/dlt-connector:latest imagePullPolicy: Always ports: - containerPort: 8080 env: - name: LOG_LEVEL value: "INFO" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:8080"] livenessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:8080"] resources: requests: cpu: 250m memory: 512Mi limits: cpu: 700m memory: 1024Mi - name: gateway image: registry.gitlab.com/teraflow-h2020/controller/dlt-gateway:latest imagePullPolicy: Always #readinessProbe: # httpGet: # path: /health # port: 8081 # initialDelaySeconds: 5 # timeoutSeconds: 5 #livenessProbe: # httpGet: # path: /health # port: 8081 # initialDelaySeconds: 5 # timeoutSeconds: 5 resources: requests: cpu: 250m memory: 512Mi limits: cpu: 700m memory: 1024Mi --- apiVersion: v1 kind: Service metadata: name: dltservice spec: type: ClusterIP selector: app: dltservice ports: - name: grpc protocol: TCP port: 8080 targetPort: 8080 Loading
.gitignore +1 −0 Original line number Diff line number Diff line Loading @@ -162,6 +162,7 @@ cython_debug/ # TeraFlowSDN-generated files tfs_runtime_env_vars.sh tfs_bchain_runtime_env_vars.sh delete_local_deployment.sh local_docker_deployment.sh local_k8s_deployment.sh Loading
.gitlab-ci.yml +1 −0 Original line number Diff line number Diff line Loading @@ -42,3 +42,4 @@ include: #- local: '/src/slice/.gitlab-ci.yml' #- local: '/src/interdomain/.gitlab-ci.yml' - local: '/src/pathcomp/.gitlab-ci.yml' #- local: '/src/dlt/.gitlab-ci.yml'
deploy.sh +70 −26 Original line number Diff line number Diff line Loading @@ -66,44 +66,71 @@ echo "export PYTHONPATH=${PYTHONPATH}" >> $ENV_VARS_SCRIPT for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." IMAGE_NAME="$COMPONENT:$TFS_IMAGE_TAG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" elif [ "$COMPONENT" == "pathcomp" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . >> "$BUILD_LOG" docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" # next command is redundant, but helpful to keep cache updated between rebuilds docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG-builder" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" docker build -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" elif [ "$COMPONENT" == "dlt" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" docker build -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" docker build -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" else docker build -t "$IMAGE_NAME" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" fi if [ -n "$TFS_REGISTRY_IMAGE" ]; then echo " Pushing Docker image to '$TFS_REGISTRY_IMAGE'..." if [ "$COMPONENT" == "pathcomp" ]; then TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL-frontend" > "$TAG_LOG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL-backend" > "$TAG_LOG" TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-frontend.log" docker tag "$COMPONENT-frontend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-frontend.log" docker push "$IMAGE_URL-frontend" > "$PUSH_LOG" docker push "$IMAGE_URL" > "$PUSH_LOG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-backend.log" docker tag "$COMPONENT-backend:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-backend.log" docker push "$IMAGE_URL-backend" > "$PUSH_LOG" docker push "$IMAGE_URL" > "$PUSH_LOG" elif [ "$COMPONENT" == "dlt" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-connector.log" docker tag "$COMPONENT-connector:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-connector.log" docker push "$IMAGE_URL" > "$PUSH_LOG" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}-gateway.log" docker tag "$COMPONENT-gateway:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}-gateway.log" docker push "$IMAGE_URL" > "$PUSH_LOG" else IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" docker tag "$COMPONENT:$TFS_IMAGE_TAG" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" Loading @@ -117,33 +144,48 @@ for COMPONENT in $TFS_COMPONENTS; do if [ -n "$TFS_REGISTRY_IMAGE" ]; then # Registry is set if [ "$COMPONENT" == "pathcomp" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-frontend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL-frontend#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-backend:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL-backend#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" elif [ "$COMPONENT" == "dlt" ]; then IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-connector:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT-gateway:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" else IMAGE_URL=$(echo "$TFS_REGISTRY_IMAGE/$COMPONENT:$TFS_IMAGE_TAG" | sed 's,//,/,g' | sed 's,http:/,,g') VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" fi sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" else # Registry is not set if [ "$COMPONENT" == "pathcomp" ]; then VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-frontend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $IMAGE_NAME-frontend#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-frontend:${VERSION}#image: $COMPONENT-frontend:$TFS_IMAGE_TAG#g" "$MANIFEST" VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-backend:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $IMAGE_NAME-backend#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-backend:${VERSION}#image: $COMPONENT-backend:$TFS_IMAGE_TAG#g" "$MANIFEST" elif [ "$COMPONENT" == "dlt" ]; then VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-connector:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-connector:${VERSION}#image: $COMPONENT-connector:$TFS_IMAGE_TAG#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}-gateway:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT-gateway:${VERSION}#image: $COMPONENT-gateway:$TFS_IMAGE_TAG#g" "$MANIFEST" else VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $COMPONENT:$TFS_IMAGE_TAG#g" "$MANIFEST" fi sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" fi # TODO: harmonize names of the monitoring component Loading @@ -157,7 +199,7 @@ for COMPONENT in $TFS_COMPONENTS; do echo " Collecting env-vars for '$COMPONENT' component..." SERVICE_DATA=$(kubectl get service ${COMPONENT}service --namespace $TFS_K8S_NAMESPACE -o json) SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME}service --namespace $TFS_K8S_NAMESPACE -o json) if [ -z "${SERVICE_DATA}" ]; then continue; fi # Env vars for service's host address Loading Loading @@ -189,6 +231,7 @@ for EXTRA_MANIFEST in $TFS_EXTRA_MANIFESTS; do kubectl --namespace $TFS_K8S_NAMESPACE apply -f $EXTRA_MANIFEST printf "\n" done printf "\n" # By now, leave these controls here. Some component dependencies are not well handled. Loading @@ -203,8 +246,9 @@ fi for COMPONENT in $TFS_COMPONENTS; do echo "Waiting for '$COMPONENT' component..." COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl wait --namespace $TFS_K8S_NAMESPACE \ --for='condition=available' --timeout=300s deployment/${COMPONENT}service --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME}service printf "\n" done Loading
deploy_mock_blockchain.sh 0 → 100755 +121 −0 Original line number Diff line number Diff line #!/bin/bash # Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ######################################################################################################################## # Read deployment settings ######################################################################################################################## # Set the URL of your local Docker registry where the images will be uploaded to. REGISTRY_IMAGE="http://localhost:32000/tfs/" # Set the tag you want to use for your images. IMAGE_TAG="dev" # Set the name of the Kubernetes namespace to deploy to. K8S_NAMESPACE="tfs-bchain" COMPONENT="mock_blockchain" ######################################################################################################################## # Automated steps start here ######################################################################################################################## # Constants GITLAB_REPO_URL="registry.gitlab.com/teraflow-h2020/controller" TMP_FOLDER="./tmp" # Create a tmp folder for files modified during the deployment TMP_MANIFESTS_FOLDER="$TMP_FOLDER/manifests" mkdir -p $TMP_MANIFESTS_FOLDER TMP_LOGS_FOLDER="$TMP_FOLDER/logs" mkdir -p $TMP_LOGS_FOLDER echo "Deleting and Creating a new namespace..." kubectl delete namespace $K8S_NAMESPACE kubectl create namespace $K8S_NAMESPACE printf "\n" echo "Deploying components and collecting environment variables..." ENV_VARS_SCRIPT=tfs_bchain_runtime_env_vars.sh echo "# Environment variables for TeraFlow Mock-Blockchain deployment" > $ENV_VARS_SCRIPT PYTHONPATH=$(pwd)/src echo "Processing '$COMPONENT' component..." IMAGE_NAME="$COMPONENT:$IMAGE_TAG" IMAGE_URL=$(echo "$REGISTRY_IMAGE/$IMAGE_NAME" | sed 's,//,/,g' | sed 's,http:/,,g') echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" docker build -t "$IMAGE_NAME" -f ./src/dlt/mock_blockchain/Dockerfile . > "$BUILD_LOG" if [ -n "$REGISTRY_IMAGE" ]; then echo " Pushing Docker image to '$REGISTRY_IMAGE'..." TAG_LOG="$TMP_LOGS_FOLDER/tag_${COMPONENT}.log" docker tag "$IMAGE_NAME" "$IMAGE_URL" > "$TAG_LOG" PUSH_LOG="$TMP_LOGS_FOLDER/push_${COMPONENT}.log" docker push "$IMAGE_URL" > "$PUSH_LOG" fi echo " Adapting '$COMPONENT' manifest file..." MANIFEST="$TMP_MANIFESTS_FOLDER/${COMPONENT}.yaml" cp ./manifests/"${COMPONENT}".yaml "$MANIFEST" if [ -n "$REGISTRY_IMAGE" ]; then # Registry is set VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_URL#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Always#g" "$MANIFEST" else # Registry is not set VERSION=$(grep -i "${GITLAB_REPO_URL}/${COMPONENT}:" "$MANIFEST" | cut -d ":" -f3) sed -E -i "s#image: $GITLAB_REPO_URL/$COMPONENT:${VERSION}#image: $IMAGE_NAME#g" "$MANIFEST" sed -E -i "s#imagePullPolicy: .*#imagePullPolicy: Never#g" "$MANIFEST" fi echo " Deploying '$COMPONENT' component to Kubernetes..." DEPLOY_LOG="$TMP_LOGS_FOLDER/deploy_${COMPONENT}.log" kubectl --namespace $K8S_NAMESPACE apply -f "$MANIFEST" > "$DEPLOY_LOG" COMPONENT_OBJNAME=$(echo "${COMPONENT}" | sed "s/\_/-/") kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=0 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG" kubectl --namespace $K8S_NAMESPACE scale deployment --replicas=1 ${COMPONENT_OBJNAME} >> "$DEPLOY_LOG" echo " Collecting env-vars for '$COMPONENT' component..." SERVICE_DATA=$(kubectl get service ${COMPONENT_OBJNAME} --namespace $K8S_NAMESPACE -o json) # Env vars for service's host address SERVICE_HOST=$(echo ${SERVICE_DATA} | jq -r '.spec.clusterIP') ENVVAR_HOST=$(echo "${COMPONENT}_SERVICE_HOST" | tr '[:lower:]' '[:upper:]') echo "export ${ENVVAR_HOST}=${SERVICE_HOST}" >> $ENV_VARS_SCRIPT # Env vars for service's 'grpc' port SERVICE_PORT_GRPC=$(echo ${SERVICE_DATA} | jq -r '.spec.ports[] | select(.name=="grpc") | .port') ENVVAR_PORT_GRPC=$(echo "${COMPONENT}_SERVICE_PORT_GRPC" | tr '[:lower:]' '[:upper:]') echo "export ${ENVVAR_PORT_GRPC}=${SERVICE_PORT_GRPC}" >> $ENV_VARS_SCRIPT printf "\n" echo "Waiting for '$COMPONENT' component..." kubectl wait --namespace $K8S_NAMESPACE \ --for='condition=available' --timeout=300s deployment/${COMPONENT_OBJNAME} printf "\n" echo "Deployment Resources:" kubectl --namespace $K8S_NAMESPACE get all printf "\n" echo "Done!"
manifests/dltservice.yaml 0 → 100644 +86 −0 Original line number Diff line number Diff line # Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. apiVersion: apps/v1 kind: Deployment metadata: name: dltservice spec: selector: matchLabels: app: dltservice template: metadata: labels: app: dltservice spec: terminationGracePeriodSeconds: 5 containers: - name: connector image: registry.gitlab.com/teraflow-h2020/controller/dlt-connector:latest imagePullPolicy: Always ports: - containerPort: 8080 env: - name: LOG_LEVEL value: "INFO" readinessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:8080"] livenessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:8080"] resources: requests: cpu: 250m memory: 512Mi limits: cpu: 700m memory: 1024Mi - name: gateway image: registry.gitlab.com/teraflow-h2020/controller/dlt-gateway:latest imagePullPolicy: Always #readinessProbe: # httpGet: # path: /health # port: 8081 # initialDelaySeconds: 5 # timeoutSeconds: 5 #livenessProbe: # httpGet: # path: /health # port: 8081 # initialDelaySeconds: 5 # timeoutSeconds: 5 resources: requests: cpu: 250m memory: 512Mi limits: cpu: 700m memory: 1024Mi --- apiVersion: v1 kind: Service metadata: name: dltservice spec: type: ClusterIP selector: app: dltservice ports: - name: grpc protocol: TCP port: 8080 targetPort: 8080