Commit 8c9c335b authored by Konstantinos Poulakakis's avatar Konstantinos Poulakakis
Browse files

Merge branch 'develop' into feat/159-automation-component-skeleton 

# Conflicts:
#	scripts/run_tests_locally-kpi-DB.sh
#	scripts/run_tests_locally-telemetry-DB.sh
#	scripts/run_tests_locally-telemetry-backend.sh
#	scripts/run_tests_locally-telemetry-frontend.sh
#	src/analytics/.gitlab-ci.yml
#	src/analytics/frontend/service/AnalyticsFrontendServiceServicerImpl.py
#	src/analytics/frontend/tests/test_frontend.py
#	src/common/tools/kafka/Variables.py
parents fa2fbda4 dab05407

.gitlab-ci.yml

+2 −0
Original line number Diff line number Diff line
@@ -50,5 +50,7 @@ include: 
  - local: '/src/kpi_value_api/.gitlab-ci.yml'

  - local: '/src/kpi_value_writer/.gitlab-ci.yml'

  - local: '/src/telemetry/.gitlab-ci.yml'

  - local: '/src/analytics/.gitlab-ci.yml'



  # This should be last one: end-to-end integration tests

  - local: '/src/tests/.gitlab-ci.yml'

deploy/kafka.sh

+7 −7
Original line number Diff line number Diff line
@@ -47,10 +47,10 @@ function kafka_deploy() { 
    cp "${KFK_MANIFESTS_PATH}/${KFK_MANIFEST}" "${TMP_MANIFESTS_FOLDER}/${KFK_MANIFEST}"



    # echo "Apache Kafka Namespace"

    echo ">>> Delete Apache Kafka Namespace"

    echo "Delete Apache Kafka Namespace"

    kubectl delete namespace ${KFK_NAMESPACE} --ignore-not-found



    echo ">>> Create Apache Kafka Namespace"

    echo "Create Apache Kafka Namespace"

    kubectl create namespace ${KFK_NAMESPACE}



    # echo ">>> Deplying Apache Kafka Zookeeper"
@@ -76,15 +76,15 @@ function kafka_deploy() { 
    # fi

}



echo "Apache Kafka"

echo ">>> Checking if Apache Kafka is deployed ... "

echo ">>> Apache Kafka"

echo "Checking if Apache Kafka is deployed ... "

if [ "$KFK_REDEPLOY" == "YES" ]; then

    echo ">>> Redeploying kafka namespace"

    echo "Redeploying kafka namespace"

    kafka_deploy

elif kubectl get namespace "${KFK_NAMESPACE}" &> /dev/null; then

    echo ">>> Apache Kafka already present; skipping step." 

    echo "Apache Kafka already present; skipping step." 

else

    echo ">>> Kafka namespace doesn't exists. Deploying kafka namespace"

    echo "Kafka namespace doesn't exists. Deploying kafka namespace"

    kafka_deploy

fi

echo

deploy/tfs.sh

+6 −40
Original line number Diff line number Diff line
@@ -146,55 +146,17 @@ kubectl create namespace $TFS_K8S_NAMESPACE 
sleep 2

printf "\n"



echo "Create secret with CockroachDB data"

echo ">>> Create Secret with CockroachDB data..."

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_CONTEXT=${CRDB_DATABASE}  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_CONTEXT} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with CockroachDB data for KPI Management microservices"

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_KPI_MGMT="tfs_kpi_mgmt"  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_KPI_MGMT} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with CockroachDB data for Telemetry microservices"

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_TELEMETRY="tfs_telemetry"  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with CockroachDB data for Analytics microservices"

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_ANALYTICS="tfs_analytics"  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-analytics --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_ANALYTICS} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"



echo "Create secret with Apache Kafka data for KPI, Telemetry and Analytics microservices"

echo ">>> Create Secret with Apache Kakfa..."

KFK_SERVER_PORT=$(kubectl --namespace ${KFK_NAMESPACE} get service kafka-service -o 'jsonpath={.spec.ports[0].port}')

kubectl create secret generic kfk-kpi-data --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=KFK_NAMESPACE=${KFK_NAMESPACE} \
@@ -669,6 +631,10 @@ if [[ "$TFS_COMPONENTS" == *"monitoring"* ]] && [[ "$TFS_COMPONENTS" == *"webui" 
    printf "\n\n"

fi



echo "Pruning Docker Images..."

docker image prune --force

printf "\n\n"



if [ "$DOCKER_BUILD" == "docker buildx build" ]; then

    echo "Pruning Docker Buildx Cache..."

    docker buildx prune --force

manifests/analyticsservice.yaml

+13 −3
Original line number Diff line number Diff line
@@ -37,9 +37,13 @@ spec: 
          env:

            - name: LOG_LEVEL

              value: "INFO"

            - name: CRDB_DATABASE

              value: "tfs_analytics"

            - name: METRICS_PORT

              value: "9192"

          envFrom:

            - secretRef:

                name: crdb-analytics

                name: crdb-data

            - secretRef:

                name: kfk-kpi-data

          readinessProbe:
@@ -60,10 +64,12 @@ spec: 
          imagePullPolicy: Always

          ports:

            - containerPort: 30090

            - containerPort: 9192

            - containerPort: 9193

          env:

            - name: LOG_LEVEL

              value: "INFO"

            - name: METRICS_PORT

              value: "9193"

          envFrom:

            - secretRef:

                name: kfk-kpi-data
@@ -100,10 +106,14 @@ spec: 
      protocol: TCP

      port: 30090

      targetPort: 30090

    - name: metrics

    - name: metrics-frontend

      protocol: TCP

      port: 9192

      targetPort: 9192

    - name: metrics-backend

      protocol: TCP

      port: 9193

      targetPort: 9193

---

apiVersion: autoscaling/v2

kind: HorizontalPodAutoscaler

manifests/contextservice.yaml

+2 −0
Original line number Diff line number Diff line
@@ -45,6 +45,8 @@ spec: 
              value: "FALSE"

            - name: ALLOW_EXPLICIT_ADD_LINK_TO_TOPOLOGY

              value: "FALSE"

            - name: CRDB_DATABASE

              value: "tfs_context"

          envFrom:

            - secretRef:

                name: crdb-data