Commit 5266289b authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/slice-grouping' into 'develop' 

Slice Grouping

See merge request !64
parents 8dcee204 1b2785f2

deploy/all.sh

+6 −2
Original line number Original line Diff line number Diff line
@@ -110,10 +110,14 @@ export QDB_PASSWORD=${QDB_PASSWORD:-"quest"} 
# If not already set, set the table name to be used by Monitoring for KPIs.

# If not already set, set the table name to be used by Monitoring for KPIs.

export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}





# If not already set, set the table name to be used by Slice for plotting groups.

export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}



# If not already set, disable flag for dropping tables if they exist.

# If not already set, disable flag for dropping tables if they exist.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!

# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variable

# If QDB_DROP_TABLES_IF_EXIST is "YES", the tables pointed by variables

# QDB_TABLE_MONITORING_KPIS will be dropped while checking/deploying QuestDB.

# QDB_TABLE_MONITORING_KPIS and QDB_TABLE_SLICE_GROUPS will be dropped while 

# checking/deploying QuestDB.

export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}

export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}





# If not already set, disable flag for re-deploying QuestDB from scratch.

# If not already set, disable flag for re-deploying QuestDB from scratch.

deploy/qdb.sh

+8 −2
Original line number Original line Diff line number Diff line
@@ -30,10 +30,14 @@ export QDB_PASSWORD=${QDB_PASSWORD:-"quest"} 
# If not already set, set the table name to be used by Monitoring for KPIs.

# If not already set, set the table name to be used by Monitoring for KPIs.

export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}





# If not already set, set the table name to be used by Slice for plotting groups.

export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}



# If not already set, disable flag for dropping tables if they exist.

# If not already set, disable flag for dropping tables if they exist.

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!

# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!

# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variable

# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variables

# QDB_TABLE_MONITORING_KPIS will be dropped while checking/deploying QuestDB.

# QDB_TABLE_MONITORING_KPIS and QDB_TABLE_SLICE_GROUPS will be dropped

# while checking/deploying QuestDB.

export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}

export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}





# If not already set, disable flag for re-deploying QuestDB from scratch.

# If not already set, disable flag for re-deploying QuestDB from scratch.
@@ -151,6 +155,8 @@ function qdb_drop_tables() { 
    echo "Drop tables, if exist"

    echo "Drop tables, if exist"

    curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;"

    curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;"

    echo

    echo

    curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_SLICE_GROUPS}+;"

    echo

}

}





if [ "$QDB_REDEPLOY" == "YES" ]; then

if [ "$QDB_REDEPLOY" == "YES" ]; then

deploy/tfs.sh

+44 −0
Original line number Original line Diff line number Diff line
@@ -81,6 +81,9 @@ export QDB_PASSWORD=${QDB_PASSWORD:-"quest"} 
# If not already set, set the table name to be used by Monitoring for KPIs.

# If not already set, set the table name to be used by Monitoring for KPIs.

export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}





# If not already set, set the table name to be used by Slice for plotting groups.

export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}







########################################################################################################################

########################################################################################################################

# Automated steps start here

# Automated steps start here
@@ -131,6 +134,7 @@ kubectl create secret generic qdb-data --namespace ${TFS_K8S_NAMESPACE} --type=' 
    --from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \

    --from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \

    --from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \

    --from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \

    --from-literal=METRICSDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS} \

    --from-literal=METRICSDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS} \

    --from-literal=METRICSDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS} \

    --from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \

    --from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \

    --from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}

    --from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}

printf "\n"

printf "\n"
@@ -321,11 +325,17 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring" 
        "confirmNew" : "'${TFS_GRAFANA_PASSWORD}'"

        "confirmNew" : "'${TFS_GRAFANA_PASSWORD}'"

    }' ${GRAFANA_URL_DEFAULT}/api/user/password

    }' ${GRAFANA_URL_DEFAULT}/api/user/password

    echo

    echo

    echo





    # Updated Grafana API URL

    # Updated Grafana API URL

    GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_URL}"

    GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_URL}"

    echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT

    echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT





    echo ">> Installing Scatter Plot plugin..."

    curl -X POST -H "Content-Type: application/json" -H "Content-Length: 0" \

        ${GRAFANA_URL_UPDATED}/api/plugins/michaeldmoore-scatter-panel/install

    echo



    # Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/

    # Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/

    QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}"

    QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}"

    echo ">> Creating datasources..."

    echo ">> Creating datasources..."
@@ -354,17 +364,51 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring" 
    }' ${GRAFANA_URL_UPDATED}/api/datasources

    }' ${GRAFANA_URL_UPDATED}/api/datasources

    echo

    echo





    curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{

        "access"   : "proxy",

        "type"     : "postgres",

        "name"     : "questdb-slc-grp",

        "url"      : "'${QDB_HOST_PORT}'",

        "database" : "'${QDB_TABLE_SLICE_GROUPS}'",

        "user"     : "'${QDB_USERNAME}'",

        "basicAuth": false,

        "isDefault": false,

        "jsonData" : {

            "sslmode"               : "disable",

            "postgresVersion"       : 1100,

            "maxOpenConns"          : 0,

            "maxIdleConns"          : 2,

            "connMaxLifetime"       : 14400,

            "tlsAuth"               : false,

            "tlsAuthWithCACert"     : false,

            "timescaledb"           : false,

            "tlsConfigurationMethod": "file-path",

            "tlsSkipVerify"         : true

        },

        "secureJsonData": {"password": "'${QDB_PASSWORD}'"}

    }' ${GRAFANA_URL_UPDATED}/api/datasources

    printf "\n\n"



    echo ">> Creating dashboards..."

    echo ">> Creating dashboards..."

    # Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/

    # Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/

    curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_mon_kpis_psql.json' \

    curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_mon_kpis_psql.json' \

        ${GRAFANA_URL_UPDATED}/api/dashboards/db

        ${GRAFANA_URL_UPDATED}/api/dashboards/db

    echo

    echo





    curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_slc_grps_psql.json' \

        ${GRAFANA_URL_UPDATED}/api/dashboards/db

    printf "\n\n"



    echo ">> Staring dashboards..."

    echo ">> Staring dashboards..."

    DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-l3-monit"

    DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-l3-monit"

    DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')

    DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')

    curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}

    curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}

    echo

    echo





    DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-slice-grps"

    DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')

    curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}

    echo



    printf "\n\n"

    printf "\n\n"

fi
 
fi

manifests/sliceservice.yaml

+5 −0
Original line number Original line Diff line number Diff line
@@ -37,6 +37,11 @@ spec: 
        env:

        env:

        - name: LOG_LEVEL

        - name: LOG_LEVEL

          value: "INFO"

          value: "INFO"

        - name: SLICE_GROUPING

          value: "DISABLE"

        envFrom:

        - secretRef:

            name: qdb-data

        readinessProbe:

        readinessProbe:

          exec:

          exec:

            command: ["/bin/grpc_health_probe", "-addr=:4040"]

            command: ["/bin/grpc_health_probe", "-addr=:4040"]

my_deploy.sh

+3 −0
Original line number Original line Diff line number Diff line
@@ -86,6 +86,9 @@ export QDB_PASSWORD="quest" 
# Set the table name to be used by Monitoring for KPIs.

# Set the table name to be used by Monitoring for KPIs.

export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis"

export QDB_TABLE_MONITORING_KPIS="tfs_monitoring_kpis"





# Set the table name to be used by Slice for plotting groups.

export QDB_TABLE_SLICE_GROUPS="tfs_slice_groups"



# Disable flag for dropping tables if they exist.

# Disable flag for dropping tables if they exist.

export QDB_DROP_TABLES_IF_EXIST=""

export QDB_DROP_TABLES_IF_EXIST=""