Slice Grouping

Deploy scripts:

  • Added variable to specify QuestDB table to use to plot slice groups
  • Extended configuration of Grafana for Slice Grouping monitoring
  • Minor cosmetic changes

Proto:

  • Added "availability" field in SLA Availability
  • Extended load_generator proto to configure/start/stop/monitor Load Generator component

Common - Tools - gRPC Helpers:

  • Added "availability" field in SLA Availability

Common - Tools - Object Factory:

  • Added "availability" field in SLA Availability

Compute component:

  • Added "availability" field in SLA Availability

Context component:

  • Added indexes to enhance performance

Load Generator component:

  • Added configuration parameters for remote management from WebUI
  • Added method to retrieve status of the Load Generator
  • Added "availability" field in SLA Availability

PathComp component - Frontend:

  • Added "availability" field in SLA Availability

Slice component:

  • Added env var to activate/deactivate slice grouping functionality
  • Added missing methods in SliceClient
  • Implemented SliceGrouping feature
  • Implemented MetricsExporter to monitor slice groups in Grafana

WebUI component:

  • Added page to remotely control and monitor Load Generator component
  • Added basic Debug-API links
  • Added "availability" field in SLA Availability in Service/Slice details
  • Added Grafana dashboard to monitor Slice Groups
Edited by Lluis Gifre Renom

Merge request reports

Loading
+6 −2
Changes for deploy/all.sh: 6 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -110,10 +110,14 @@ export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}
# If not already set, set the table name to be used by Monitoring for KPIs.
export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

# If not already set, set the table name to be used by Slice for plotting groups.
export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}

# If not already set, disable flag for dropping tables if they exist.
# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!
# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variable
# QDB_TABLE_MONITORING_KPIS will be dropped while checking/deploying QuestDB.
# If QDB_DROP_TABLES_IF_EXIST is "YES", the tables pointed by variables
# QDB_TABLE_MONITORING_KPIS and QDB_TABLE_SLICE_GROUPS will be dropped while 
# checking/deploying QuestDB.
export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}

# If not already set, disable flag for re-deploying QuestDB from scratch.
+8 −2
Changes for deploy/qdb.sh: 8 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -30,10 +30,14 @@ export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}
# If not already set, set the table name to be used by Monitoring for KPIs.
export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

# If not already set, set the table name to be used by Slice for plotting groups.
export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}

# If not already set, disable flag for dropping tables if they exist.
# WARNING: ACTIVATING THIS FLAG IMPLIES LOOSING THE TABLE INFORMATION!
# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variable
# QDB_TABLE_MONITORING_KPIS will be dropped while checking/deploying QuestDB.
# If QDB_DROP_TABLES_IF_EXIST is "YES", the table pointed by variables
# QDB_TABLE_MONITORING_KPIS and QDB_TABLE_SLICE_GROUPS will be dropped
# while checking/deploying QuestDB.
export QDB_DROP_TABLES_IF_EXIST=${QDB_DROP_TABLES_IF_EXIST:-""}

# If not already set, disable flag for re-deploying QuestDB from scratch.
@@ -151,6 +155,8 @@ function qdb_drop_tables() {
    echo "Drop tables, if exist"
    curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_MONITORING_KPIS}+;"
    echo
    curl "http://${QDB_HOST}:${QDB_PORT}/exec?fmt=json&query=DROP+TABLE+IF+EXISTS+${QDB_TABLE_SLICE_GROUPS}+;"
    echo
}

if [ "$QDB_REDEPLOY" == "YES" ]; then
+44 −0
Changes for deploy/tfs.sh: 44 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -81,6 +81,9 @@ export QDB_PASSWORD=${QDB_PASSWORD:-"quest"}
# If not already set, set the table name to be used by Monitoring for KPIs.
export QDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS:-"tfs_monitoring_kpis"}

# If not already set, set the table name to be used by Slice for plotting groups.
export QDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS:-"tfs_slice_groups"}


########################################################################################################################
# Automated steps start here
@@ -131,6 +134,7 @@ kubectl create secret generic qdb-data --namespace ${TFS_K8S_NAMESPACE} --type='
    --from-literal=METRICSDB_ILP_PORT=${QDB_ILP_PORT} \
    --from-literal=METRICSDB_SQL_PORT=${QDB_SQL_PORT} \
    --from-literal=METRICSDB_TABLE_MONITORING_KPIS=${QDB_TABLE_MONITORING_KPIS} \
    --from-literal=METRICSDB_TABLE_SLICE_GROUPS=${QDB_TABLE_SLICE_GROUPS} \
    --from-literal=METRICSDB_USERNAME=${QDB_USERNAME} \
    --from-literal=METRICSDB_PASSWORD=${QDB_PASSWORD}
printf "\n"
@@ -321,11 +325,17 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
        "confirmNew" : "'${TFS_GRAFANA_PASSWORD}'"
    }' ${GRAFANA_URL_DEFAULT}/api/user/password
    echo
    echo

    # Updated Grafana API URL
    GRAFANA_URL_UPDATED="http://${GRAFANA_USERNAME}:${TFS_GRAFANA_PASSWORD}@${GRAFANA_URL}"
    echo "export GRAFANA_URL_UPDATED=${GRAFANA_URL_UPDATED}" >> $ENV_VARS_SCRIPT

    echo ">> Installing Scatter Plot plugin..."
    curl -X POST -H "Content-Type: application/json" -H "Content-Length: 0" \
        ${GRAFANA_URL_UPDATED}/api/plugins/michaeldmoore-scatter-panel/install
    echo

    # Ref: https://grafana.com/docs/grafana/latest/http_api/data_source/
    QDB_HOST_PORT="${METRICSDB_HOSTNAME}:${QDB_SQL_PORT}"
    echo ">> Creating datasources..."
@@ -354,17 +364,51 @@ if [[ "$TFS_COMPONENTS" == *"webui"* ]] && [[ "$TFS_COMPONENTS" == *"monitoring"
    }' ${GRAFANA_URL_UPDATED}/api/datasources
    echo

    curl -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{
        "access"   : "proxy",
        "type"     : "postgres",
        "name"     : "questdb-slc-grp",
        "url"      : "'${QDB_HOST_PORT}'",
        "database" : "'${QDB_TABLE_SLICE_GROUPS}'",
        "user"     : "'${QDB_USERNAME}'",
        "basicAuth": false,
        "isDefault": false,
        "jsonData" : {
            "sslmode"               : "disable",
            "postgresVersion"       : 1100,
            "maxOpenConns"          : 0,
            "maxIdleConns"          : 2,
            "connMaxLifetime"       : 14400,
            "tlsAuth"               : false,
            "tlsAuthWithCACert"     : false,
            "timescaledb"           : false,
            "tlsConfigurationMethod": "file-path",
            "tlsSkipVerify"         : true
        },
        "secureJsonData": {"password": "'${QDB_PASSWORD}'"}
    }' ${GRAFANA_URL_UPDATED}/api/datasources
    printf "\n\n"

    echo ">> Creating dashboards..."
    # Ref: https://grafana.com/docs/grafana/latest/http_api/dashboard/
    curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_mon_kpis_psql.json' \
        ${GRAFANA_URL_UPDATED}/api/dashboards/db
    echo

    curl -X POST -H "Content-Type: application/json" -d '@src/webui/grafana_db_slc_grps_psql.json' \
        ${GRAFANA_URL_UPDATED}/api/dashboards/db
    printf "\n\n"

    echo ">> Staring dashboards..."
    DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-l3-monit"
    DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')
    curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
    echo

    DASHBOARD_URL="${GRAFANA_URL_UPDATED}/api/dashboards/uid/tfs-slice-grps"
    DASHBOARD_ID=$(curl -s "${DASHBOARD_URL}" | jq '.dashboard.id')
    curl -X POST ${GRAFANA_URL_UPDATED}/api/user/stars/dashboard/${DASHBOARD_ID}
    echo

    printf "\n\n"
fi
+5 −0
Changes for manifests/sliceservice.yaml: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -37,6 +37,11 @@ spec:
        env:
        - name: LOG_LEVEL
          value: "INFO"
        - name: SLICE_GROUPING
          value: "DISABLE"
        envFrom:
        - secretRef:
            name: qdb-data
        readinessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr=:4040"]
+1 −0
Changes for proto/context.proto: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -509,6 +509,7 @@ message Constraint_SLA_Capacity {
message Constraint_SLA_Availability {
  uint32 num_disjoint_paths = 1;
  bool all_active = 2;
  float availability = 3; // 0.0 .. 100.0 percentage of availability
}

enum IsolationLevelEnum {
Loading
Loading