Commit 0306b222 authored by Waleed Akbar's avatar Waleed Akbar
Browse files

Changes to Address Minor Issues in Monitoring Services Deployment and Build: 

- Improved Kafka scripts.
- Updated manifests to handle ports for backend and frontend.
- Changed the return type of a method in settings to `int`.
- Added DB deployment in the service's main file.
parent aec985e7

deploy/kafka.sh

+7 −7
Original line number Diff line number Diff line
@@ -47,10 +47,10 @@ function kafka_deploy() { 
    cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"



    # echo "Apache Kafka Namespace"

    echo ">>> Delete Apache Kafka Namespace"

    echo "Delete Apache Kafka Namespace"

    kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found



    echo ">>> Create Apache Kafka Namespace"

    echo "Create Apache Kafka Namespace"

    kubectl create namespace ${KFK_NAMESPACE}



    # echo ">>> Deplying Apache Kafka Zookeeper"
@@ -76,15 +76,15 @@ function kafka_deploy() { 
    # fi

}



echo "Apache Kafka"

echo ">>> Checking if Apache Kafka is deployed ... "

echo ">>> Apache Kafka"

echo "Checking if Apache Kafka is deployed ... "

if [ "$KFK_REDEPLOY" == "YES" ]; then

    echo ">>> Redeploying kafka namespace"

    echo "Redeploying kafka namespace"

    kafka_deploy

elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then

    echo ">>> Apache Kafka already present; skipping step." 

    echo "Apache Kafka already present; skipping step." 

else

    echo ">>> Kafka namespace doesn't exists. Deploying kafka namespace"

    echo "Kafka namespace doesn't exists. Deploying kafka namespace"

    kafka_deploy

fi

echo

manifests/analyticsservice.yaml

+10 −2
Original line number Diff line number Diff line
@@ -39,6 +39,8 @@ spec: 
              value: "INFO"

            - name: CRDB_DATABASE

              value: "tfs_analytics"

            - name: METRICS_PORT

              value: "9192"

          envFrom:

            - secretRef:

                name: crdb-analytics
@@ -62,10 +64,12 @@ spec: 
          imagePullPolicy: Always

          ports:

            - containerPort: 30090

            - containerPort: 9192

            - containerPort: 9193

          env:

            - name: LOG_LEVEL

              value: "INFO"

            - name: METRICS_PORT

              value: "9193"

          envFrom:

            - secretRef:

                name: kfk-kpi-data
@@ -102,10 +106,14 @@ spec: 
      protocol: TCP

      port: 30090

      targetPort: 30090

    - name: metrics

    - name: metrics-frontend

      protocol: TCP

      port: 9192

      targetPort: 9192

    - name: metrics-backend

      protocol: TCP

      port: 9193

      targetPort: 9193

---

apiVersion: autoscaling/v2

kind: HorizontalPodAutoscaler

manifests/servicemonitors.yaml

+153 −0
Original line number Diff line number Diff line
@@ -475,3 +475,156 @@ spec: 
    any: false

    matchNames:

      - tfs # namespace where the app is running

---

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

  namespace: monitoring # namespace where prometheus is running

  name: tfs-analyticsservice-metric

  labels:

    app: analyticsservice

    #release: prometheus

    #release: prom  # name of the release

    # ( VERY IMPORTANT: You need to know the correct release name by viewing

    #   the servicemonitor of Prometheus itself: Without the correct name,

    #   Prometheus cannot identify the metrics of the Flask app as the target.)

spec:

  selector:

    matchLabels:

      # Target app service

      #namespace: tfs

      app: analyticsservice # same as above

      #release: prometheus # same as above

  endpoints:

    - port: metrics-frontend # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

    - port: metrics-backend # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

  namespaceSelector:

    any: false

    matchNames:

      - tfs # namespace where the app is running

---

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

  namespace: monitoring # namespace where prometheus is running

  name: tfs-telemetryservice-metric

  labels:

    app: telemetryservice

    #release: prometheus

    #release: prom  # name of the release

    # ( VERY IMPORTANT: You need to know the correct release name by viewing

    #   the servicemonitor of Prometheus itself: Without the correct name,

    #   Prometheus cannot identify the metrics of the Flask app as the target.)

spec:

  selector:

    matchLabels:

      # Target app service

      #namespace: tfs

      app: telemetryservice # same as above

      #release: prometheus # same as above

  endpoints:

    - port: metrics-frontend # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

    - port: metrics-backend # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

  namespaceSelector:

    any: false

    matchNames:

      - tfs # namespace where the app is running

---

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

  namespace: monitoring # namespace where prometheus is running

  name: tfs-kpi-managerservice-metric

  labels:

    app: kpi-managerservice

    #release: prometheus

    #release: prom  # name of the release

    # ( VERY IMPORTANT: You need to know the correct release name by viewing

    #   the servicemonitor of Prometheus itself: Without the correct name,

    #   Prometheus cannot identify the metrics of the Flask app as the target.)

spec:

  selector:

    matchLabels:

      # Target app service

      #namespace: tfs

      app: kpi-managerservice # same as above

      #release: prometheus # same as above

  endpoints:

    - port: metrics # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

  namespaceSelector:

    any: false

    matchNames:

      - tfs # namespace where the app is running

---

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

  namespace: monitoring # namespace where prometheus is running

  name: tfs-kpi_value_apiservice-metric

  labels:

    app: kpi_value_apiservice

    #release: prometheus

    #release: prom  # name of the release

    # ( VERY IMPORTANT: You need to know the correct release name by viewing

    #   the servicemonitor of Prometheus itself: Without the correct name,

    #   Prometheus cannot identify the metrics of the Flask app as the target.)

spec:

  selector:

    matchLabels:

      # Target app service

      #namespace: tfs

      app: kpi_value_apiservice # same as above

      #release: prometheus # same as above

  endpoints:

    - port: metrics # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

  namespaceSelector:

    any: false

    matchNames:

      - tfs # namespace where the app is running

---

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

  namespace: monitoring # namespace where prometheus is running

  name: tfs-kpi_value_writerservice-metric

  labels:

    app: kpi_value_writerservice

    #release: prometheus

    #release: prom  # name of the release

    # ( VERY IMPORTANT: You need to know the correct release name by viewing

    #   the servicemonitor of Prometheus itself: Without the correct name,

    #   Prometheus cannot identify the metrics of the Flask app as the target.)

spec:

  selector:

    matchLabels:

      # Target app service

      #namespace: tfs

      app: kpi_value_writerservice # same as above

      #release: prometheus # same as above

  endpoints:

    - port: metrics # named port in target app

      scheme: http

      path: /metrics # path to scrape

      interval: 5s # scrape interval

  namespaceSelector:

    any: false

    matchNames:

      - tfs # namespace where the app is running

manifests/telemetryservice.yaml

+10 −2
Original line number Diff line number Diff line
@@ -39,6 +39,8 @@ spec: 
              value: "INFO"

            - name: CRDB_DATABASE

              value: "tfs_kpi"

            - name: METRICS_PORT

              value: "9192"

          envFrom:

            - secretRef:

                name: crdb-telemetry
@@ -62,10 +64,12 @@ spec: 
          imagePullPolicy: Always

          ports:

            - containerPort: 30060

            - containerPort: 9192

            - containerPort: 9193

          env:

            - name: LOG_LEVEL

              value: "INFO"

            - name: METRICS_PORT

              value: "9193"

          envFrom:

            - secretRef:

                name: kfk-kpi-data
@@ -102,10 +106,14 @@ spec: 
      protocol: TCP

      port: 30060

      targetPort: 30060

    - name: metrics

    - name: metrics-frontend

      protocol: TCP

      port: 9192

      targetPort: 9192

    - name: metrics-backend

      protocol: TCP

      port: 9193

      targetPort: 9193

---

apiVersion: autoscaling/v2

kind: HorizontalPodAutoscaler

src/analytics/backend/service/__main__.py

+1 −1
Original line number Diff line number Diff line
@@ -37,7 +37,7 @@ def main(): 
    LOGGER.info('Starting...')



    # Start metrics server

    metrics_port = get_metrics_port()

    metrics_port = int(get_metrics_port())

    start_http_server(metrics_port)



    grpc_service = AnalyticsBackendService()