diff --git a/deploy/all.sh b/deploy/all.sh index c169bc92c0d9a6dea87de919ad20b4cf3afc1199..c5d423a2f5112704f529670cb8bf18b8be7d4642 100755 --- a/deploy/all.sh +++ b/deploy/all.sh @@ -18,6 +18,11 @@ # Read deployment settings ######################################################################################################################## +# ----- Redeploy All ------------------------------------------------------------ + +# If not already set, enables all components redeployment +export REDEPLOYALL=${REDEPLOYALL:-""} + # ----- TeraFlowSDN ------------------------------------------------------------ @@ -102,6 +107,14 @@ export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} # If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to. export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"} +# If not already set, set NATS installation mode. Accepted values are: 'single' and 'cluster'. +# - If NATS_DEPLOY_MODE is "single", NATS is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# - If NATS_DEPLOY_MODE is "cluster", NATS is deployed in cluster mode, and an entire NATS cluster +# with 3 replicas (set by default) will be deployed. It is convenient for production and +# provides scalability features. +export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"} + # If not already set, disable flag for re-deploying NATS from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! # If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS. diff --git a/deploy/crdb.sh b/deploy/crdb.sh index c979ad4f2c18861c6a93b6b04e5d8e3e71aae41e..6412d13164e1ef6b5fca2dc292b06f59f98cae54 100755 --- a/deploy/crdb.sh +++ b/deploy/crdb.sh @@ -18,6 +18,11 @@ # Read deployment settings ######################################################################################################################## +# ----- Redeploy All ------------------------------------------------------------ +# If not already set, enables all components redeployment +export REDEPLOYALL=${REDEPLOYALL:-""} + + # If not already set, set the namespace where CockroackDB will be deployed. export CRDB_NAMESPACE=${CRDB_NAMESPACE:-"crdb"} @@ -223,7 +228,7 @@ function crdb_deploy_cluster() { kubectl create namespace ${CRDB_NAMESPACE} echo - echo "CockroachDB" + echo "CockroachDB (cluster-mode)" echo ">>> Checking if CockroachDB is deployed..." if kubectl get --namespace ${CRDB_NAMESPACE} statefulset/cockroachdb &> /dev/null; then echo ">>> CockroachDB is present; skipping step." @@ -360,7 +365,7 @@ function crdb_drop_database_cluster() { } if [ "$CRDB_DEPLOY_MODE" == "single" ]; then - if [ "$CRDB_REDEPLOY" == "YES" ]; then + if [ "$CRDB_REDEPLOY" == "YES" ] || [ "$REDEPLOYALL" == "YES" ]; then crdb_undeploy_single fi @@ -370,7 +375,7 @@ if [ "$CRDB_DEPLOY_MODE" == "single" ]; then crdb_drop_database_single fi elif [ "$CRDB_DEPLOY_MODE" == "cluster" ]; then - if [ "$CRDB_REDEPLOY" == "YES" ]; then + if [ "$CRDB_REDEPLOY" == "YES" ] || [ "$REDEPLOYALL" == "YES" ]; then crdb_undeploy_cluster fi diff --git a/deploy/nats.sh b/deploy/nats.sh index 366270a6915a1eef969846446ecc9152c3fa9531..57fda629cf9a1cb62cf5100ba609147bb92168fc 100755 --- a/deploy/nats.sh +++ b/deploy/nats.sh @@ -18,6 +18,10 @@ # Read deployment settings ######################################################################################################################## +# ----- Redeploy All ------------------------------------------------------------ +# If not already set, enables all components redeployment +export REDEPLOYALL=${REDEPLOYALL:-""} + # If not already set, set the namespace where NATS will be deployed. export NATS_NAMESPACE=${NATS_NAMESPACE:-"nats"} @@ -27,16 +31,31 @@ export NATS_EXT_PORT_CLIENT=${NATS_EXT_PORT_CLIENT:-"4222"} # If not already set, set the external port NATS HTTP Mgmt GUI interface will be exposed to. export NATS_EXT_PORT_HTTP=${NATS_EXT_PORT_HTTP:-"8222"} +# If not already set, set NATS installation mode. Accepted values are: 'single' and 'cluster'. +# - If NATS_DEPLOY_MODE is "single", NATS is deployed in single node mode. It is convenient for +# development and testing purposes and should fit in a VM. IT SHOULD NOT BE USED IN PRODUCTION ENVIRONMENTS. +# - If NATS_DEPLOY_MODE is "cluster", NATS is deployed in cluster mode, and an entire NATS cluster +# with 3 replicas (set by default) will be deployed. It is convenient for production and +# provides scalability features. +export NATS_DEPLOY_MODE=${NATS_DEPLOY_MODE:-"single"} + # If not already set, disable flag for re-deploying NATS from scratch. # WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE MESSAGE BROKER INFORMATION! # If NATS_REDEPLOY is "YES", the message broker will be dropped while checking/deploying NATS. export NATS_REDEPLOY=${NATS_REDEPLOY:-""} - ######################################################################################################################## # Automated steps start here ######################################################################################################################## +# Constants +TMP_FOLDER="./tmp" +NATS_MANIFESTS_PATH="manifests/nats" + +# Create a tmp folder for files modified during the deployment +TMP_MANIFESTS_FOLDER="${TMP_FOLDER}/${NATS_NAMESPACE}/manifests" +mkdir -p $TMP_MANIFESTS_FOLDER + function nats_deploy_single() { echo "NATS Namespace" echo ">>> Create NATS Namespace (if missing)" @@ -47,18 +66,86 @@ function nats_deploy_single() { helm3 repo add nats https://nats-io.github.io/k8s/helm/charts/ echo + echo "Install NATS (single-node)" + echo ">>> Checking if NATS is deployed..." + if kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; then + echo ">>> NATS is present; skipping step." + else + echo ">>> Deploy NATS" + helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} --set nats.image=nats:2.9-alpine --set config.cluster.enabled=true --set config.cluster.tls.enabled=true + + + echo ">>> Waiting NATS statefulset to be created..." + while ! kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; do + printf "%c" "." + sleep 1 + done + + # Wait for statefulset condition "Available=True" does not work + # Wait for statefulset condition "jsonpath='{.status.readyReplicas}'=3" throws error: + # "error: readyReplicas is not found" + # Workaround: Check the pods are ready + #echo ">>> NATS statefulset created. Waiting for readiness condition..." + #kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Available=True --timeout=300s statefulset/nats + #kubectl wait --namespace ${NATS_NAMESPACE} --for=jsonpath='{.status.readyReplicas}'=3 --timeout=300s \ + # statefulset/nats + echo ">>> NATS statefulset created. Waiting NATS pods to be created..." + while ! kubectl get --namespace ${NATS_NAMESPACE} pod/${NATS_NAMESPACE}-0 &> /dev/null; do + printf "%c" "." + sleep 1 + done + kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-0 + fi + echo + + echo "NATS Port Mapping" + echo ">>> Expose NATS Client port (4222->${NATS_EXT_PORT_CLIENT})" + NATS_PORT_CLIENT=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="client")].port}') + PATCH='{"data": {"'${NATS_EXT_PORT_CLIENT}'": "'${NATS_NAMESPACE}'/'${NATS_NAMESPACE}':'${NATS_PORT_CLIENT}'"}}' + kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" + + PORT_MAP='{"containerPort": '${NATS_EXT_PORT_CLIENT}', "hostPort": '${NATS_EXT_PORT_CLIENT}'}' + CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' + PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' + kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" + echo + + echo ">>> Expose NATS HTTP Mgmt GUI port (8222->${NATS_EXT_PORT_HTTP})" + NATS_PORT_HTTP=$(kubectl --namespace ${NATS_NAMESPACE} get service ${NATS_NAMESPACE} -o 'jsonpath={.spec.ports[?(@.name=="monitor")].port}') + PATCH='{"data": {"'${NATS_EXT_PORT_HTTP}'": "'${NATS_NAMESPACE}'/'${NATS_NAMESPACE}':'${NATS_PORT_HTTP}'"}}' + kubectl patch configmap nginx-ingress-tcp-microk8s-conf --namespace ingress --patch "${PATCH}" + + PORT_MAP='{"containerPort": '${NATS_EXT_PORT_HTTP}', "hostPort": '${NATS_EXT_PORT_HTTP}'}' + CONTAINER='{"name": "nginx-ingress-microk8s", "ports": ['${PORT_MAP}']}' + PATCH='{"spec": {"template": {"spec": {"containers": ['${CONTAINER}']}}}}' + kubectl patch daemonset nginx-ingress-microk8s-controller --namespace ingress --patch "${PATCH}" + echo +} + + +function nats_deploy_cluster() { + echo "NATS Namespace" + echo ">>> Create NATS Namespace (if missing)" + kubectl create namespace ${NATS_NAMESPACE} + echo + + echo "Add NATS Helm Chart" + helm3 repo add nats https://nats-io.github.io/k8s/helm/charts/ + echo + echo "Upgrade NATS Helm Chart" helm3 repo update nats echo - echo "Install NATS (single-node)" + echo "Install NATS (cluster-mode)" echo ">>> Checking if NATS is deployed..." if kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; then echo ">>> NATS is present; skipping step." else echo ">>> Deploy NATS" - helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} --set nats.image=nats:2.9-alpine - + cp "${NATS_MANIFESTS_PATH}/cluster.yaml" "${TMP_MANIFESTS_FOLDER}/nats_cluster.yaml" + helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} -f "${TMP_MANIFESTS_FOLDER}/nats_cluster.yaml" + echo ">>> Waiting NATS statefulset to be created..." while ! kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; do printf "%c" "." @@ -78,7 +165,17 @@ function nats_deploy_single() { printf "%c" "." sleep 1 done + while ! kubectl get --namespace ${NATS_NAMESPACE} pod/${NATS_NAMESPACE}-1 &> /dev/null; do + printf "%c" "." + sleep 1 + done + while ! kubectl get --namespace ${NATS_NAMESPACE} pod/${NATS_NAMESPACE}-2 &> /dev/null; do + printf "%c" "." + sleep 1 + done kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-0 + kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-1 + kubectl wait --namespace ${NATS_NAMESPACE} --for=condition=Ready --timeout=300s pod/${NATS_NAMESPACE}-2 fi echo @@ -110,7 +207,7 @@ function nats_deploy_single() { echo } -function nats_undeploy_single() { +function nats_undeploy() { echo "NATS" echo ">>> Checking if NATS is deployed..." if kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; then @@ -127,8 +224,14 @@ function nats_undeploy_single() { echo } -if [ "$NATS_REDEPLOY" == "YES" ]; then - nats_undeploy_single +if [ "$NATS_REDEPLOY" == "YES" ] || [ "$REDEPLOYALL" == "YES" ]; then + nats_undeploy fi -nats_deploy_single +if [ "$NATS_DEPLOY_MODE" == "single" ]; then + nats_deploy_single +elif [ "$NATS_DEPLOY_MODE" == "cluster" ]; then + nats_deploy_cluster +else + echo "Unsupported value: NATS_DEPLOY_MODE=$NATS_DEPLOY_MODE" +fi \ No newline at end of file diff --git a/deploy/qdb.sh b/deploy/qdb.sh index acbcfd4f96ccbd2b09d5d82f66a1bf801a710780..513ef9ae01f63dc31b31ee076e6fc9537ffbc1b1 100755 --- a/deploy/qdb.sh +++ b/deploy/qdb.sh @@ -18,6 +18,10 @@ # Read deployment settings ######################################################################################################################## +# ----- Redeploy All ------------------------------------------------------------ +# If not already set, enables all components redeployment +export REDEPLOYALL=${REDEPLOYALL:-""} + # If not already set, set the namespace where QuestDB will be deployed. export QDB_NAMESPACE=${QDB_NAMESPACE:-"qdb"} @@ -177,7 +181,7 @@ function qdb_drop_tables() { echo } -if [ "$QDB_REDEPLOY" == "YES" ]; then +if [ "$QDB_REDEPLOY" == "YES" ] || [ "$REDEPLOYALL" == "YES" ]; then qdb_undeploy fi diff --git a/manifests/cockroachdb/cluster.yaml b/manifests/cockroachdb/cluster.yaml index 4d9ef0f844b5ffb02753b6cc7a7be7d03928896c..bcb0c704948ecdbd8b271b68e685c481e669594b 100644 --- a/manifests/cockroachdb/cluster.yaml +++ b/manifests/cockroachdb/cluster.yaml @@ -39,8 +39,8 @@ spec: cpu: 8 memory: 8Gi tlsEnabled: true -# You can set either a version of the db or a specific image name -# cockroachDBVersion: v22.2.8 + # You can set either a version of the db or a specific image name + # cockroachDBVersion: v22.2.8 image: name: cockroachdb/cockroach:v22.2.8 # nodes refers to the number of crdb pods that are created @@ -49,21 +49,16 @@ spec: additionalLabels: crdb: is-cool # affinity is a new API field that is behind a feature gate that is - # disabled by default. To enable please see the operator.yaml file. + # disabled by default. To enable please see the operator.yaml file. # The affinity field will accept any podSpec affinity rule. - # affinity: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - weight: 100 - # podAffinityTerm: - # labelSelector: - # matchExpressions: - # - key: app.kubernetes.io/instance - # operator: In - # values: - # - cockroachdb - # topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: cockroachdb # nodeSelectors used to match against # nodeSelector: diff --git a/manifests/cockroachdb/operator.yaml b/manifests/cockroachdb/operator.yaml index 59d515061c4c0f253523aab803653b3f33007461..d8e691308e4cc16af3f545d87244281ab0730696 100644 --- a/manifests/cockroachdb/operator.yaml +++ b/manifests/cockroachdb/operator.yaml @@ -381,6 +381,7 @@ spec: spec: containers: - args: + - -feature-gates=TolerationRules=true,AffinityRules=true,TopologySpreadRules=true - -zap-log-level - info env: diff --git a/manifests/deviceservice.yaml b/manifests/deviceservice.yaml index e49ba23995b07f4e7e956a69f8d5383c8e976e89..bf599d0b4d898588ea2850927c4daca50c359202 100644 --- a/manifests/deviceservice.yaml +++ b/manifests/deviceservice.yaml @@ -70,11 +70,30 @@ spec: selector: app: deviceservice ports: - - name: grpc - protocol: TCP - port: 2020 - targetPort: 2020 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 2020 + targetPort: 2020 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: deviceservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: deviceservice + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 diff --git a/manifests/monitoringservice.yaml b/manifests/monitoringservice.yaml index 1540db0a12ba65e9fdb48cd73ea4a0a63fddf5e7..4058436e5d8cede64866fd0fb029da8a70d340d4 100644 --- a/manifests/monitoringservice.yaml +++ b/manifests/monitoringservice.yaml @@ -65,11 +65,30 @@ spec: selector: app: monitoringservice ports: - - name: grpc - protocol: TCP - port: 7070 - targetPort: 7070 - - name: metrics - protocol: TCP - port: 9192 - targetPort: 9192 + - name: grpc + protocol: TCP + port: 7070 + targetPort: 7070 + - name: metrics + protocol: TCP + port: 9192 + targetPort: 9192 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: monitoringservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: monitoringservice + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 diff --git a/manifests/nats/cluster.yaml b/manifests/nats/cluster.yaml new file mode 100644 index 0000000000000000000000000000000000000000..491c86628fc7769c3dbde4cdd6b2a2e98e7ab904 --- /dev/null +++ b/manifests/nats/cluster.yaml @@ -0,0 +1,34 @@ +container: + image: + tags: 2.9-alpine + env: + # different from k8s units, suffix must be B, KiB, MiB, GiB, or TiB + # should be ~90% of memory limit + GOMEMLIMIT: 400MiB + merge: + # recommended limit is at least 2 CPU cores and 8Gi Memory for production JetStream clusters + resources: + requests: + cpu: 1 + memory: 500Mi + limits: + cpu: 1 + memory: 1Gi + +config: + cluster: + enabled: true + replicas: 3 + jetstream: + enabled: true + fileStore: + pvc: + size: 4Gi + +# Force one pod per node, if possible +podTemplate: + topologySpreadConstraints: + kubernetes.io/hostname: + maxSkew: 1 + whenUnsatisfiable: ScheduleAnyway + \ No newline at end of file diff --git a/manifests/nginx_ingress_http.yaml b/manifests/nginx_ingress_http.yaml index 0892f0c9b790b936df5540ac5fe1aed0270b91a5..cb400ee7d936214c0bfc08af8e7ae3f85a46cd76 100644 --- a/manifests/nginx_ingress_http.yaml +++ b/manifests/nginx_ingress_http.yaml @@ -18,6 +18,11 @@ metadata: name: tfs-ingress annotations: nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/limit-rps: '5' + nginx.ingress.kubernetes.io/limit-connections: '10' + nginx.ingress.kubernetes.io/proxy-connect-timeout: '10' + nginx.ingress.kubernetes.io/proxy-send-timeout: '10' + nginx.ingress.kubernetes.io/proxy-read-timeout: '10' spec: rules: - http: diff --git a/manifests/webuiservice.yaml b/manifests/webuiservice.yaml index a519aa4a2f8a1e81f1b7f2a1be1965ec0b8bb386..132839edd95dfb742a56034f9a34406292199033 100644 --- a/manifests/webuiservice.yaml +++ b/manifests/webuiservice.yaml @@ -111,9 +111,31 @@ spec: selector: app: webuiservice ports: - - name: webui - port: 8004 - targetPort: 8004 - - name: grafana - port: 3000 - targetPort: 3000 + - name: webui + port: 8004 + targetPort: 8004 + - name: grafana + port: 3000 + targetPort: 3000 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: webuiservice-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: webuiservice + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + #behavior: + # scaleDown: + # stabilizationWindowSeconds: 30 diff --git a/my_deploy.sh b/my_deploy.sh index 8417f6eae510391e65d5f91202e59cccf32e1f98..991c21e711618d0c33f891051a84deb4b8479a93 100755 --- a/my_deploy.sh +++ b/my_deploy.sh @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# ----- Redeploy All ------------------------------------------------------------ + +# If not already set, enables all components redeployment +export REDEPLOYALL="" + # ----- TeraFlowSDN ------------------------------------------------------------ @@ -123,6 +128,10 @@ export NATS_EXT_PORT_CLIENT="4222" # Set the external port NATS HTTP Mgmt GUI interface will be exposed to. export NATS_EXT_PORT_HTTP="8222" +# Set NATS installation mode to 'single'. This option is convenient for development and testing. +# See ./deploy/all.sh or ./deploy/nats.sh for additional details +export NATS_DEPLOY_MODE="single" + # Disable flag for re-deploying NATS from scratch. export NATS_REDEPLOY=""