diff --git a/deploy/all.sh b/deploy/all.sh index 876928c4b3a0bd4d61ba5ec568bcb49c6d825dc9..50a6c0816c9a5a25fa4ceaebb3730408ce66e4cc 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -107,6 +107,15 @@ export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} # If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to. export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"} +# TESTING +# If not already set, set NATS installation mode. Accepted values are: 'single' and 'cluster'. +# - If NATS_DEPLOY_MODE is "single", NATS is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# - If NATS_DEPLOY_MODE is "cluster", NATS is deployed in cluster mode, and an entire NATS cluster +# with 3 replicas (set by default) will be deployed. It is convenient for production and +# provides scalability features. +export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"} + # If not already set, disable flag for re-deploying NATS from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! # If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS. diff --git a/deploy/crdb.sh b/deploy/crdb.sh index d81c7c70389cba11d00c63ddd579ffe9019bd14b..2a8bd88d390846e8543d9043324455250ab616dc 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -228,7 +228,7 @@ function crdb_deploy_cluster() { kubectl create namespace ${CRDB_NAMESPACE} echo - echo "CockroachDB" + echo "CockroachDB (cluster-mode)" echo ">>> Checking if CockroachDB is deployed..." if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then echo ">>> CockroachDB is present; skipping step." diff --git a/deploy/nats.sh b/deploy/nats.sh index 3e2374fc714261ed308c660627766be91d904125..d6922d86b9be5dbc535468b83a50221781cc5092 100755 --- a/deploy/nats.sh +++ b/deploy/nats.sh @@ -31,16 +31,32 @@ export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} # If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to. export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"} +# TESTING +# If not already set, set NATS installation mode. Accepted values are: 'single' and 'cluster'. +# - If NATS_DEPLOY_MODE is "single", NATS is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# - If NATS_DEPLOY_MODE is "cluster", NATS is deployed in cluster mode, and an entire NATS cluster +# with 3 replicas (set by default) will be deployed. It is convenient for production and +# provides scalability features. +export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"} + # If not already set, disable flag for re-deploying NATS from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! # If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS. export NATS_REDEPLOY=${NATS_REDEPLOY:-""} - ######################################################################################################################## # Automated steps start here ######################################################################################################################## +# Constants +TMP_FOLDER="./tmp" +NATS_MANIFESTS_PATH="manifests/nats" + +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${NATS_NAMESPACE}/manifests" +mkdir -p $TMP_MANIFESTS_FOLDER + function nats_deploy_single() { echo "NATS Namespace" echo ">>> Create NATS Namespace (if missing)" @@ -51,20 +67,85 @@ function nats_deploy_single() { helm3 repo add nats https://nats-io.github.io/k8s/helm/charts/ echo + echo "Install NATS (single-node)" + echo ">>> Checking if NATS is deployed..." + if kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; then + echo ">>> NATS is present; skipping step." + else + echo ">>> Deploy NATS" + helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} --set nats.image=nats:2.9-alpine + + echo ">>> Waiting NATS statefulset to be created..." + while ! kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; do + printf "%c" "." + sleep 1 + done + + # Wait for statefulset condition "Available=True" does not work + # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error: + # "error: readyReplicas is not found" + # Workaround: Check the pods are ready + #echo ">>> NATS statefulset created. Waiting for readiness condition..." + #kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/nats + #kubectl wait --namespace ${NATS_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \ + # statefulset/nats + echo ">>> NATS statefulset created. Waiting NATS pods to be created..." + while ! kubectl get --namespace ${NATS_NAMESPACE} pod/${NATS_NAMESPACE}-0 &> /dev/null; do + printf "%c" "." + sleep 1 + done + kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-0 + fi + echo + + echo "NATS Port Mapping" + echo ">>> Expose NATS Client port (4222->${NATS_EXT_PORT_CLIENT})" + NATS_PORT_CLIENT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}') + PATCH='{"data": {"'${NATS_EXT_PORT_CLIENT}'": "'${NATS_NAMESPACE}'/'${NATS_NAMESPACE}':'${NATS_PORT_CLIENT}'"}}' + kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" + + PORT_MAP='{"containerPort": '${NATS_EXT_PORT_CLIENT}', "hostPort": '${NATS_EXT_PORT_CLIENT}'}' + CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' + PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' + kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" + echo + + echo ">>> Expose NATS HTTP Mgmt GUI port (8222->${NATS_EXT_PORT_HTTP})" + NATS_PORT_HTTP=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="monitor")].port}') + PATCH='{"data": {"'${NATS_EXT_PORT_HTTP}'": "'${NATS_NAMESPACE}'/'${NATS_NAMESPACE}':'${NATS_PORT_HTTP}'"}}' + kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" + + PORT_MAP='{"containerPort": '${NATS_EXT_PORT_HTTP}', "hostPort": '${NATS_EXT_PORT_HTTP}'}' + CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' + PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' + kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" + echo +} + + +function nats_deploy_cluster() { + echo "NATS Namespace" + echo ">>> Create NATS Namespace (if missing)" + kubectl create namespace ${NATS_NAMESPACE} + echo + + echo "Add NATS Helm Chart" + helm3 repo add nats https://nats-io.github.io/k8s/helm/charts/ + echo + echo "Upgrade NATS Helm Chart" helm3 repo update nats echo - echo "Install NATS (single-node)" + echo "Install NATS (cluster-mode)" echo ">>> Checking if NATS is deployed..." if kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; then echo ">>> NATS is present; skipping step." else - # TESTING - Experimenting with cluster echo ">>> Deploy NATS" - helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} --set nats.image=nats:2.9-alpine --set config.cluster.enabled=true --set config.cluster.tls.enabled=true - - + cp "${NATS_MANIFESTS_PATH}/cluster.yaml" "${TMP_MANIFESTS_FOLDER}/nats_cluster.yaml" + helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/nats_cluster.yaml" + echo ">>> Waiting NATS statefulset to be created..." while ! kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; do printf "%c" "." @@ -84,7 +165,17 @@ function nats_deploy_single() { printf "%c" "." sleep 1 done + while ! kubectl get --namespace ${NATS_NAMESPACE} pod/${NATS_NAMESPACE}-1 &> /dev/null; do + printf "%c" "." + sleep 1 + done + while ! kubectl get --namespace ${NATS_NAMESPACE} pod/${NATS_NAMESPACE}-2 &> /dev/null; do + printf "%c" "." + sleep 1 + done kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-0 + kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-1 + kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-2 fi echo @@ -116,7 +207,7 @@ function nats_deploy_single() { echo } -function nats_undeploy_single() { +function nats_undeploy() { echo "NATS" echo ">>> Checking if NATS is deployed..." if kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; then @@ -134,7 +225,13 @@ function nats_undeploy_single() { } if [ "$NATS_REDEPLOY" == "YES" ] || [ "$REDEPLOYALL" == "YES" ]; then - nats_undeploy_single + nats_undeploy fi -nats_deploy_single +if [ "$NATS_DEPLOY_MODE" == "single" ]; then + nats_deploy_single +elif [ "$NATS_DEPLOY_MODE" == "cluster" ]; then + nats_deploy_cluster +else + echo "Unsupported value: NATS_DEPLOY_MODE=$NATS_DEPLOY_MODE" +fi \ No newline at end of file diff --git a/manifests/cockroachdb/cluster.yaml b/manifests/cockroachdb/cluster.yaml index 4d9ef0f844b5ffb02753b6cc7a7be7d03928896c..73875ca3f5edef43208b8ac79cdc1ec3c38d9ec6 100644 --- a/manifests/cockroachdb/cluster.yaml +++ b/manifests/cockroachdb/cluster.yaml @@ -33,14 +33,16 @@ spec: resources: requests: # This is intentionally low to make it work on local k3d clusters. - cpu: 4 - memory: 4Gi + # TESTING + cpu: 1 #4 + memory: 500Mi #4Gi limits: - cpu: 8 - memory: 8Gi + # TESTING + cpu: 1 #8 + memory: 1Gi #8Gi tlsEnabled: true -# You can set either a version of the db or a specific image name -# cockroachDBVersion: v22.2.8 + # You can set either a version of the db or a specific image name + # cockroachDBVersion: v22.2.8 image: name: cockroachdb/cockroach:v22.2.8 # nodes refers to the number of crdb pods that are created @@ -49,21 +51,17 @@ spec: additionalLabels: crdb: is-cool # affinity is a new API field that is behind a feature gate that is - # disabled by default. To enable please see the operator.yaml file. + # disabled by default. To enable please see the operator.yaml file. # The affinity field will accept any podSpec affinity rule. - # affinity: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - weight: 100 - # podAffinityTerm: - # labelSelector: - # matchExpressions: - # - key: app.kubernetes.io/instance - # operator: In - # values: - # - cockroachdb - # topologyKey: kubernetes.io/hostname + # TESTING: Force one pod per node, if possible + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: cockroachdb # nodeSelectors used to match against # nodeSelector: diff --git a/manifests/cockroachdb/operator.yaml b/manifests/cockroachdb/operator.yaml index 59d515061c4c0f253523aab803653b3f33007461..0d578410c37744e038ec81a04fd1a08eee4b8425 100644 --- a/manifests/cockroachdb/operator.yaml +++ b/manifests/cockroachdb/operator.yaml @@ -381,6 +381,8 @@ spec: spec: containers: - args: + # TESTING + - -feature-gates=TolerationRules=true,AffinityRules=true,TopologySpreadRules=true - -zap-log-level - info env: diff --git a/manifests/nats/cluster.yaml b/manifests/nats/cluster.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39e41958fa8fee02f9e49b08ba88e7e3025131cb --- /dev/null +++ b/manifests/nats/cluster.yaml @@ -0,0 +1,34 @@ +container: + image: + tags: 2.9-alpine + env: + # different from k8s units, suffix must be B, KiB, MiB, GiB, or TiB + # should be ~90% of memory limit + GOMEMLIMIT: 400MiB + merge: + # recommended limit is at least 2 CPU cores and 8Gi Memory for production JetStream clusters + resources: + requests: + cpu: 1 # 2 + memory: 500Mi # 4Gi + limits: + cpu: 1 # 4 + memory: 1Gi # 8Gi + +config: + cluster: + enabled: true + replicas: 3 + jetstream: + enabled: true + fileStore: + pvc: + size: 4Gi + +# Force one pod per node, if possible +podTemplate: + topologySpreadConstraints: + kubernetes.io/hostname: + maxSkew: 1 + whenUnsatisfiable: ScheduleAnyway + \ No newline at end of file diff --git a/my_deploy.sh b/my_deploy.sh index c358bc1afc5207a76cea9e4049c1fa223d49f9bf..92a1bfb63bbd20d2750261c87c9d46491031e12e 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -101,7 +101,7 @@ export CRDB_DATABASE="tfs" # Set CockroachDB installation mode to 'single'. This option is convenient for development and testing. # See ./deploy/all.sh or ./deploy/crdb.sh for additional details -export CRDB_DEPLOY_MODE="single" +export CRDB_DEPLOY_MODE="cluster" # Disable flag for dropping database, if it exists. export CRDB_DROP_DATABASE_IF_EXISTS="" @@ -121,6 +121,11 @@ export NATS_EXT_PORT_CLIENT="4222" # Set the external port NATS HTTP Mgmt GUI interface will be exposed to. export NATS_EXT_PORT_HTTP="8222" +# TESTING +# Set NATS installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/nats.sh for additional details +export NATS_DEPLOY_MODE="single" + # Disable flag for re-deploying NATS from scratch. export NATS_REDEPLOY=""