CockroachDB deployment:

# Ref: https://www.cockroachlabs.com/docs/stable/configure-cockroachdb-kubernetes.html

# CockroachDB configuration preparation

These steps reproduce how to generate Cockroach manifest files used in TeraFlowSDN and apply them to MicroK8s.

For stability reasons, we fix the versions providing the manifest files.

In future releases of TeraFlowSDN, we might consider dynamically downloading and modifying the files.

- Ref: https://www.cockroachlabs.com/docs/stable/configure-cockroachdb-kubernetes.html

## Steps:

DEPLOY_PATH="manifests/cockroachdb"

OPERATOR_BASE_URL="https://raw.githubusercontent.com/cockroachdb/cockroach-operator/master"

# Deploy CockroachDB Operator

curl -o "${DEPLOY_PATH}/operator.yaml" "${OPERATOR_BASE_URL}/install/operator.yaml"

nano "${DEPLOY_PATH}/operator.yaml"

# - add env var: WATCH_NAMESPACE='crdb'

kubectl apply -f "${DEPLOY_PATH}/operator.yaml"

# - add env var: WATCH_NAMESPACE=%TFS_CRDB_NAMESPACE%

sed s/%TFS_CRDB_NAMESPACE%/crdb/g ${DEPLOY_PATH}/operator.yaml > ${DEPLOY_PATH}/tfs_crdb_operator.yaml

kubectl apply -f "${DEPLOY_PATH}/tfs_crdb_operator.yaml"

# Deploy CockroachDB

curl -o "${DEPLOY_PATH}/cluster.yaml" "${OPERATOR_BASE_URL}/examples/example.yaml"

# Generated file, DO NOT EDIT. Source: cloud/kubernetes/templates/cluster-init.yaml

apiVersion: batch/v1

kind: Job

metadata:

  name: cluster-init

  labels:

    app: cockroachdb

spec:

  template:

    spec:

      containers:

      - name: cluster-init

        image: cockroachdb/cockroach:v22.1.6

        imagePullPolicy: IfNotPresent

        command:

          - "/cockroach/cockroach"

          - "init"

          - "--insecure"

          - "--host=cockroachdb-0.cockroachdb"

      restartPolicy: OnFailure

# Generated file, DO NOT EDIT. Source: cloud/kubernetes/templates/cockroachdb-statefulset.yaml

apiVersion: v1

kind: Service

metadata:

  # This service is meant to be used by clients of the database. It exposes a ClusterIP that will

  # automatically load balance connections to the different database pods.

  name: cockroachdb-public

  labels:

    app: cockroachdb

spec:

  ports:

  # The main port, served by gRPC, serves Postgres-flavor SQL, internode

  # traffic and the cli.

  - port: 26257

    targetPort: 26257

    name: grpc

  # The secondary port serves the UI as well as health and debug endpoints.

  - port: 8080

    targetPort: 8080

    name: http

  selector:

    app: cockroachdb

---

apiVersion: v1

kind: Service

metadata:

  # This service only exists to create DNS entries for each pod in the stateful

  # set such that they can resolve each other's IP addresses. It does not

  # create a load-balanced ClusterIP and should not be used directly by clients

  # in most circumstances.

  name: cockroachdb

  labels:

    app: cockroachdb

  annotations:

    # Use this annotation in addition to the actual publishNotReadyAddresses

    # field below because the annotation will stop being respected soon but the

    # field is broken in some versions of Kubernetes:

    # https://github.com/kubernetes/kubernetes/issues/58662

    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"

    # Enable automatic monitoring of all instances when Prometheus is running in the cluster.

    prometheus.io/scrape: "true"

    prometheus.io/path: "_status/vars"

    prometheus.io/port: "8080"

spec:

  ports:

  - port: 26257

    targetPort: 26257

    name: grpc

  - port: 8080

    targetPort: 8080

    name: http

  # We want all pods in the StatefulSet to have their addresses published for

  # the sake of the other CockroachDB pods even before they're ready, since they

  # have to be able to talk to each other in order to become ready.

  publishNotReadyAddresses: true

  clusterIP: None

  selector:

    app: cockroachdb

---

apiVersion: policy/v1beta1

kind: PodDisruptionBudget

metadata:

  name: cockroachdb-budget

  labels:

    app: cockroachdb

spec:

  selector:

    matchLabels:

      app: cockroachdb

  maxUnavailable: 1

---

apiVersion: apps/v1

kind: StatefulSet

metadata:

  name: cockroachdb

spec:

  serviceName: "cockroachdb"

  replicas: 3

  selector:

    matchLabels:

      app: cockroachdb

  template:

    metadata:

      labels:

        app: cockroachdb

    spec:

      affinity:

        podAntiAffinity:

          preferredDuringSchedulingIgnoredDuringExecution:

          - weight: 100

            podAffinityTerm:

              labelSelector:

                matchExpressions:

                - key: app

                  operator: In

                  values:

                  - cockroachdb

              topologyKey: kubernetes.io/hostname

      containers:

      - name: cockroachdb

        image: cockroachdb/cockroach:v22.1.6

        imagePullPolicy: IfNotPresent

        # TODO: Change these to appropriate values for the hardware that you're running. You can see

        # the resources that can be allocated on each of your Kubernetes nodes by running:

        #   kubectl describe nodes

        # Note that requests and limits should have identical values.

        resources:

          requests:

            cpu: "250m"

            memory: "1Gi"

          limits:

            cpu: "1"

            memory: "1Gi" 

        ports:

        - containerPort: 26257

          name: grpc

        - containerPort: 8080

          name: http

# We recommend that you do not configure a liveness probe on a production environment, as this can impact the availability of production databases.

#       livenessProbe:

#         httpGet:

#           path: "/health"

#           port: http

#         initialDelaySeconds: 30

#         periodSeconds: 5

        readinessProbe:

          httpGet:

            path: "/health?ready=1"

            port: http

          initialDelaySeconds: 10

          periodSeconds: 5

          failureThreshold: 2

        volumeMounts:

        - name: datadir

          mountPath: /cockroach/cockroach-data

        env:

        - name: COCKROACH_CHANNEL

          value: kubernetes-insecure

        - name: GOMAXPROCS

          valueFrom:

            resourceFieldRef:

              resource: limits.cpu

              divisor: "1"

        - name: MEMORY_LIMIT_MIB

          valueFrom:

            resourceFieldRef:

              resource: limits.memory

              divisor: "1Mi"

        command:

          - "/bin/bash"

          - "-ecx"

          # The use of qualified `hostname -f` is crucial:

          # Other nodes aren't able to look up the unqualified hostname.

          - exec

            /cockroach/cockroach

            start

            --logtostderr

            --insecure

            --advertise-host $(hostname -f)

            --http-addr 0.0.0.0

            --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb

            --cache $(expr $MEMORY_LIMIT_MIB / 4)MiB

            --max-sql-memory $(expr $MEMORY_LIMIT_MIB / 4)MiB

      # No pre-stop hook is required, a SIGTERM plus some time is all that's

      # needed for graceful shutdown of a node.

      terminationGracePeriodSeconds: 60

      volumes:

      - name: datadir

        persistentVolumeClaim:

          claimName: datadir

  podManagementPolicy: Parallel

  updateStrategy:

    type: RollingUpdate

  volumeClaimTemplates:

  - metadata:

      name: datadir

    spec:

      accessModes:

        - "ReadWriteOnce"

      resources:

        requests:

          storage: 10Gi

        - name: OPERATOR_NAME

          value: cockroachdb

        - name: WATCH_NAMESPACE

          value: crdb

          value: %TFS_CRDB_NAMESPACE%

        - name: POD_NAME

          valueFrom:

            fieldRef: