From 10abfbdddcb56a703bdd35baec5b8ac13e2b2fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Wed, 11 Oct 2023 23:10:15 +0100 Subject: [PATCH 1/3] docker buildx verification --- deploy/tfs.sh | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) mode change 100755 => 100644 deploy/tfs.sh diff --git a/deploy/tfs.sh b/deploy/tfs.sh old mode 100755 new mode 100644 index 95d882c8b..2e5baec07 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -184,32 +184,46 @@ kubectl create secret generic redis-secrets --namespace=$TFS_K8S_NAMESPACE \ --from-literal=REDIS_PASSWORD=$REDIS_PASSWORD echo "export REDIS_PASSWORD=${REDIS_PASSWORD}" >> $ENV_VARS_SCRIPT +DOCKER_BUILD="docker build" +DOCKER_MAJOR_VERSION=$(docker --version | grep -o -E "[ ][0-9]+[.]" | cut -c 1-3) +if [[ $DOCKER_MAJOR_VERSION -ge 23 ]]; then + DOCKER_BUILDX_VERSION=$(docker buildx version) + if [[ $? -ne 0 ]]; then + echo "Docker buildx command is not installed. Check: https://docs.docker.com/build/architecture/#install-buildx" + exit 0; + fi + DOCKER_BUILD="docker buildx build" +fi +echo ${DOCKER_BUILD} + for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." + + if [ "$TFS_SKIP_BUILD" != "YES" ]; then echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" if [ "$COMPONENT" == "automation" ] || [ "$COMPONENT" == "policy" ]; then - docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile ./src/"$COMPONENT"/ > "$BUILD_LOG" elif [ "$COMPONENT" == "pathcomp" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-frontend.log" - docker build -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-frontend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/frontend/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-backend.log" - docker build -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-backend:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/backend/Dockerfile . > "$BUILD_LOG" # next command is redundant, but helpful to keep cache updated between rebuilds IMAGE_NAME="$COMPONENT-backend:$TFS_IMAGE_TAG-builder" - docker build -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" + $DOCKER_BUILD -t "$IMAGE_NAME" --target builder -f ./src/"$COMPONENT"/backend/Dockerfile . >> "$BUILD_LOG" elif [ "$COMPONENT" == "dlt" ]; then BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-connector.log" - docker build -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-connector:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/connector/Dockerfile . > "$BUILD_LOG" BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}-gateway.log" - docker build -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT-gateway:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/gateway/Dockerfile . > "$BUILD_LOG" else - docker build -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" + $DOCKER_BUILD -t "$COMPONENT:$TFS_IMAGE_TAG" -f ./src/"$COMPONENT"/Dockerfile . > "$BUILD_LOG" fi echo " Pushing Docker image to '$TFS_REGISTRY_IMAGES'..." -- GitLab From 2a7ec43f55fb950471009c3b81ad6d04ca9f4ef6 Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 7 Nov 2023 10:58:54 +0000 Subject: [PATCH 2/3] Multiple components: - Homogenize Dockerfile --- src/compute/Dockerfile | 4 ++-- src/context/Dockerfile | 4 ++-- src/dbscanserving/Dockerfile | 2 +- src/dlt/connector/Dockerfile | 2 +- src/dlt/mock_blockchain/Dockerfile | 2 +- src/interdomain/Dockerfile | 2 +- src/l3_attackmitigator/Dockerfile | 2 +- src/l3_centralizedattackdetector/Dockerfile | 2 +- src/l3_distributedattackdetector/Dockerfile | 2 +- src/load_generator/Dockerfile | 2 +- src/monitoring/Dockerfile | 2 +- src/opticalattackdetector/Dockerfile | 2 +- src/opticalattackmanager/Dockerfile | 2 +- src/opticalattackmitigator/Dockerfile | 2 +- src/pathcomp/frontend/Dockerfile | 2 +- src/service/Dockerfile | 2 +- src/slice/Dockerfile | 2 +- src/tests/tools/mock_mw_sdn_ctrl/ssl_not_working/Dockerfile | 2 +- src/webui/Dockerfile | 2 +- 19 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/compute/Dockerfile b/src/compute/Dockerfile index 7a0c9bf6d..9717ea5f7 100644 --- a/src/compute/Dockerfile +++ b/src/compute/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur @@ -53,7 +53,7 @@ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -# Create component sub-folder, get specific Python packages +# Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/compute WORKDIR /var/teraflow/compute COPY src/compute/requirements.in requirements.in diff --git a/src/context/Dockerfile b/src/context/Dockerfile index d6fa045c2..18a7fbc54 100644 --- a/src/context/Dockerfile +++ b/src/context/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur @@ -53,7 +53,7 @@ RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto RUN rm *.proto RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \; -# Create component sub-folder, get specific Python packages +# Create component sub-folders, get specific Python packages RUN mkdir -p /var/teraflow/context WORKDIR /var/teraflow/context COPY src/context/requirements.in requirements.in diff --git a/src/dbscanserving/Dockerfile b/src/dbscanserving/Dockerfile index 81e3fb28a..2f3f62d6b 100644 --- a/src/dbscanserving/Dockerfile +++ b/src/dbscanserving/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/dlt/connector/Dockerfile b/src/dlt/connector/Dockerfile index f4d85ec8c..8e3a2f953 100644 --- a/src/dlt/connector/Dockerfile +++ b/src/dlt/connector/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/dlt/mock_blockchain/Dockerfile b/src/dlt/mock_blockchain/Dockerfile index 09fe5d399..af1edb3d5 100644 --- a/src/dlt/mock_blockchain/Dockerfile +++ b/src/dlt/mock_blockchain/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/interdomain/Dockerfile b/src/interdomain/Dockerfile index 66c6e938d..c10fd8220 100644 --- a/src/interdomain/Dockerfile +++ b/src/interdomain/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/l3_attackmitigator/Dockerfile b/src/l3_attackmitigator/Dockerfile index 99b7e7a94..2333bfa30 100644 --- a/src/l3_attackmitigator/Dockerfile +++ b/src/l3_attackmitigator/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/l3_centralizedattackdetector/Dockerfile b/src/l3_centralizedattackdetector/Dockerfile index 377ecd21b..376d8904d 100644 --- a/src/l3_centralizedattackdetector/Dockerfile +++ b/src/l3_centralizedattackdetector/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/l3_distributedattackdetector/Dockerfile b/src/l3_distributedattackdetector/Dockerfile index e78c41803..37fb71fba 100644 --- a/src/l3_distributedattackdetector/Dockerfile +++ b/src/l3_distributedattackdetector/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ libpcap-dev libtool && \ + apt-get --yes --quiet --quiet install wget g++ git libpcap-dev libtool && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/load_generator/Dockerfile b/src/load_generator/Dockerfile index c9297eb2f..6f7bb0eac 100644 --- a/src/load_generator/Dockerfile +++ b/src/load_generator/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/monitoring/Dockerfile b/src/monitoring/Dockerfile index 946f2cd1b..36aa7eb09 100644 --- a/src/monitoring/Dockerfile +++ b/src/monitoring/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/opticalattackdetector/Dockerfile b/src/opticalattackdetector/Dockerfile index fd903a616..2b55ae7b0 100644 --- a/src/opticalattackdetector/Dockerfile +++ b/src/opticalattackdetector/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/opticalattackmanager/Dockerfile b/src/opticalattackmanager/Dockerfile index 9920d6cef..38637861a 100644 --- a/src/opticalattackmanager/Dockerfile +++ b/src/opticalattackmanager/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/opticalattackmitigator/Dockerfile b/src/opticalattackmitigator/Dockerfile index e364cbee1..278990306 100644 --- a/src/opticalattackmitigator/Dockerfile +++ b/src/opticalattackmitigator/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/pathcomp/frontend/Dockerfile b/src/pathcomp/frontend/Dockerfile index 9384b3e19..08fe50e0f 100644 --- a/src/pathcomp/frontend/Dockerfile +++ b/src/pathcomp/frontend/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/service/Dockerfile b/src/service/Dockerfile index 5988374e0..fc431deca 100644 --- a/src/service/Dockerfile +++ b/src/service/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/slice/Dockerfile b/src/slice/Dockerfile index 4c434e212..94d5fc040 100644 --- a/src/slice/Dockerfile +++ b/src/slice/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.9-slim # Install dependencies RUN apt-get --yes --quiet --quiet update && \ - apt-get --yes --quiet --quiet install wget g++ && \ + apt-get --yes --quiet --quiet install wget g++ git && \ rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur diff --git a/src/tests/tools/mock_mw_sdn_ctrl/ssl_not_working/Dockerfile b/src/tests/tools/mock_mw_sdn_ctrl/ssl_not_working/Dockerfile index ad214b97c..70fc81e54 100644 --- a/src/tests/tools/mock_mw_sdn_ctrl/ssl_not_working/Dockerfile +++ b/src/tests/tools/mock_mw_sdn_ctrl/ssl_not_working/Dockerfile @@ -22,7 +22,7 @@ RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade setuptools wheel RUN python3 -m pip install --upgrade pip-tools -# Create component sub-folder, and copy content +# Create component sub-folders, and copy content RUN mkdir -p /var/teraflow/mock_mw_sdn_ctrl WORKDIR /var/teraflow/mock_mw_sdn_ctrl COPY . . diff --git a/src/webui/Dockerfile b/src/webui/Dockerfile index 2a1510954..ab0b2c441 100644 --- a/src/webui/Dockerfile +++ b/src/webui/Dockerfile @@ -18,7 +18,7 @@ FROM python:3.9-slim # Install dependencies # RUN apt-get --yes --quiet --quiet update && \ -# apt-get --yes --quiet --quiet install wget g++ && \ +# apt-get --yes --quiet --quiet install wget g++ git && \ # rm -rf /var/lib/apt/lists/* # Set Python to show logs as they occur -- GitLab From 230af993f4aba32009a99682229a1e761ea913da Mon Sep 17 00:00:00 2001 From: gifrerenom Date: Tue, 7 Nov 2023 10:59:35 +0000 Subject: [PATCH 3/3] Deploy Scripts: - Updated tfs.sh to choose between classical docker build command or new docker buildx framework - Minor cosmetic improvements --- deploy/tfs.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) mode change 100644 => 100755 deploy/tfs.sh diff --git a/deploy/tfs.sh b/deploy/tfs.sh old mode 100644 new mode 100755 index 2e5baec07..1ffc0cb98 --- a/deploy/tfs.sh +++ b/deploy/tfs.sh @@ -183,24 +183,26 @@ REDIS_PASSWORD=`uuidgen` kubectl create secret generic redis-secrets --namespace=$TFS_K8S_NAMESPACE \ --from-literal=REDIS_PASSWORD=$REDIS_PASSWORD echo "export REDIS_PASSWORD=${REDIS_PASSWORD}" >> $ENV_VARS_SCRIPT +printf "\n" DOCKER_BUILD="docker build" -DOCKER_MAJOR_VERSION=$(docker --version | grep -o -E "[ ][0-9]+[.]" | cut -c 1-3) +DOCKER_MAJOR_VERSION=$(docker --version | grep -o -E "Docker version [0-9]+\." | grep -o -E "[0-9]+" | cut -c 1-3) if [[ $DOCKER_MAJOR_VERSION -ge 23 ]]; then - DOCKER_BUILDX_VERSION=$(docker buildx version) + # If Docker version >= 23, build command was migrated to docker-buildx + # In Ubuntu, in practice, means to install package docker-buildx together with docker.io + # Check if docker-buildx plugin is installed + docker buildx version 1>/dev/null 2>/dev/null if [[ $? -ne 0 ]]; then echo "Docker buildx command is not installed. Check: https://docs.docker.com/build/architecture/#install-buildx" - exit 0; + echo "If you installed docker through APT package docker.io, consider installing also package docker-buildx" + exit 1; fi DOCKER_BUILD="docker buildx build" fi -echo ${DOCKER_BUILD} for COMPONENT in $TFS_COMPONENTS; do echo "Processing '$COMPONENT' component..." - - if [ "$TFS_SKIP_BUILD" != "YES" ]; then echo " Building Docker image..." BUILD_LOG="$TMP_LOGS_FOLDER/build_${COMPONENT}.log" -- GitLab