Refactor dump-logs.sh to enhance log

# See the License for the specific language governing permissions and

# limitations under the License.

# Collects logs from every running TFS service (Kubernetes) and/or every running

# Docker container, into a timestamped directory with timestamped file names.

#

# Previous collections are never deleted, so runs can be compared against each

# other. What gets collected is auto-detected from the host rather than being

# hardcoded per hostname, so new components are picked up without editing this

# script.

#

# Usage:

#   ./dump-logs.sh                 # collect everything it can find

#   ./dump-logs.sh --tail 5000     # only the last N lines of each log

#   ./dump-logs.sh --since 30m     # only entries from the last 30m (k8s only)

#   ./dump-logs.sh --out-dir DIR   # base directory (default: tmp/exec)

# Set working directory

cd "$(dirname "$0")" || exit 1

# Assuming the instances are named as: simap-datastore, tfs-e2e-ctrl, tfs-agg-ctrl, tfs-ip-ctrl

# Get the current hostname

HOSTNAME=$(hostname)

echo "Collecting logs for ${HOSTNAME}..."

rm logs -rf tmp/exec

mkdir -p tmp/exec

case "$HOSTNAME" in

    simap-datastore)

        echo "Collecting Docker container logs..."

        docker logs simap-datastore > tmp/exec/simap-datastore.log 2>&1

        docker logs nce-fan-ctrl    > tmp/exec/nce-fan-ctrl.log 2>&1

        docker logs nce-t-ctrl      > tmp/exec/nce-t-ctrl.log 2>&1

        docker logs ai-engine       > tmp/exec/ai-engine.log 2>&1

        ;;

    tfs-e2e-ctrl)

        echo "Collecting TFS E2E Controller logs..."

        kubectl logs --namespace tfs service/contextservice          -c server   > tmp/exec/e2e-context.log

        kubectl logs --namespace tfs service/deviceservice           -c server   > tmp/exec/e2e-device.log

        kubectl logs --namespace tfs service/serviceservice          -c server   > tmp/exec/e2e-service.log

        kubectl logs --namespace tfs service/pathcompservice         -c frontend > tmp/exec/e2e-pathcomp-frontend.log

        kubectl logs --namespace tfs service/pathcompservice         -c backend  > tmp/exec/e2e-pathcomp-backend.log

        kubectl logs --namespace tfs service/webuiservice            -c server   > tmp/exec/e2e-webui.log

        kubectl logs --namespace tfs service/nbiservice              -c server   > tmp/exec/e2e-nbi.log

        kubectl logs --namespace tfs service/simap-connectorservice  -c server   > tmp/exec/e2e-simap-connector.log

        ;;

    tfs-agg-ctrl)

        echo "Collecting TFS Aggregation Controller logs..."

        kubectl logs --namespace tfs service/contextservice          -c server   > tmp/exec/agg-context.log

        kubectl logs --namespace tfs service/deviceservice           -c server   > tmp/exec/agg-device.log

        kubectl logs --namespace tfs service/serviceservice          -c server   > tmp/exec/agg-service.log

        kubectl logs --namespace tfs service/pathcompservice         -c frontend > tmp/exec/agg-pathcomp-frontend.log

        kubectl logs --namespace tfs service/pathcompservice         -c backend  > tmp/exec/agg-pathcomp-backend.log

        kubectl logs --namespace tfs service/webuiservice            -c server   > tmp/exec/agg-webui.log

        kubectl logs --namespace tfs service/nbiservice              -c server   > tmp/exec/agg-nbi.log

        kubectl logs --namespace tfs service/simap-connectorservice  -c server   > tmp/exec/agg-simap-connector.log

        ;;

    tfs-ip-ctrl)

        echo "Collecting TFS IP Controller logs..."

        kubectl logs --namespace tfs service/contextservice          -c server   > tmp/exec/ip-context.log

        kubectl logs --namespace tfs service/deviceservice           -c server   > tmp/exec/ip-device.log

        kubectl logs --namespace tfs service/serviceservice          -c server   > tmp/exec/ip-service.log

        kubectl logs --namespace tfs service/pathcompservice         -c frontend > tmp/exec/ip-pathcomp-frontend.log

        kubectl logs --namespace tfs service/pathcompservice         -c backend  > tmp/exec/ip-pathcomp-backend.log

        kubectl logs --namespace tfs service/webuiservice            -c server   > tmp/exec/ip-webui.log

        kubectl logs --namespace tfs service/nbiservice              -c server   > tmp/exec/ip-nbi.log

        kubectl logs --namespace tfs service/simap-connectorservice  -c server   > tmp/exec/ip-simap-connector.log

        ;;

set -o pipefail

#-------------------------------------------------------------------------------

# Settings

#-------------------------------------------------------------------------------

NAMESPACE="tfs"

OUT_BASE="tmp/exec"

TAIL_LINES=""

SINCE=""

GZIP_THRESHOLD_MB=20      # logs bigger than this are gzipped so they stay copyable

while [ $# -gt 0 ]; do

    case "$1" in

        --tail)     TAIL_LINES="$2"; shift 2 ;;

        --since)    SINCE="$2";      shift 2 ;;

        --out-dir)  OUT_BASE="$2";   shift 2 ;;

        --namespace) NAMESPACE="$2"; shift 2 ;;

        -h|--help)

            sed -n '17,30p' "$0"

            exit 0 ;;

        *)

        echo "Unknown host: $HOSTNAME"

        echo "No logs to collect."

        ;;

            echo "Unknown argument: $1" >&2

            exit 1 ;;

    esac

done

HOSTNAME_FULL="$(hostname)"

TIMESTAMP="$(date +%Y%m%d-%H%M%S)"

# Short prefix per known host; falls back to a sanitized hostname.

case "$HOSTNAME_FULL" in

    tfs-e2e-ctrl*)                  PREFIX="e2e"   ;;

    tfs-agg-ctrl*)                  PREFIX="agg"   ;;

    tfs-ip-ctrl*)                   PREFIX="ip"    ;;

    simap-server*|simap-datastore*) PREFIX="simap" ;;

    *) PREFIX="$(echo "$HOSTNAME_FULL" | tr -cd '[:alnum:]._-')" ;;

esac

OUT_DIR="${OUT_BASE}/${PREFIX}_${TIMESTAMP}"

mkdir -p "$OUT_DIR" || exit 1

SUMMARY="${OUT_DIR}/${PREFIX}-collection-summary_${TIMESTAMP}.txt"

echo "Collecting logs for ${HOSTNAME_FULL} (prefix '${PREFIX}')..."

echo "Output directory: ${OUT_DIR}"

echo

{

    echo "TFS log collection"

    echo "  host       : ${HOSTNAME_FULL}"

    echo "  collected  : $(date -Is)"

    echo "  uptime     : $(uptime -p 2>/dev/null)"

    echo "  namespace  : ${NAMESPACE}"

    echo "  tail lines : ${TAIL_LINES:-all}"

    echo "  since      : ${SINCE:-all}"

    echo

} > "$SUMMARY"

#-------------------------------------------------------------------------------

# Helpers

#-------------------------------------------------------------------------------

# Gzip a file if it exceeds GZIP_THRESHOLD_MB, then report it.

# $1 = path to log file, $2 = human label

report_file() {

    local file="$1" label="$2"

    if [ ! -s "$file" ]; then

        echo "  [empty] ${label}" | tee -a "$SUMMARY"

        rm -f "$file"

        return

    fi

    local lines errors warnings size_mb

    lines=$(wc -l < "$file" | tr -d ' ')

    errors=$(grep -c -E 'ERROR|CRITICAL|Traceback' "$file" 2>/dev/null | head -1)

    warnings=$(grep -c -E 'WARNING' "$file" 2>/dev/null | head -1)

    errors=${errors:-0}

    warnings=${warnings:-0}

    size_mb=$(( $(stat -c %s "$file") / 1048576 ))

    if [ "$size_mb" -ge "$GZIP_THRESHOLD_MB" ]; then

        gzip -f "$file" && file="${file}.gz"

    fi

    printf '  %-46s %8s lines  %4s err  %4s warn  %s\n' \

        "$(basename "$file")" "$lines" "$errors" "$warnings" \

        "$(du -h "$file" | cut -f1)" | tee -a "$SUMMARY"

}

#-------------------------------------------------------------------------------

# Kubernetes: every container of every pod in the namespace

#-------------------------------------------------------------------------------

collect_kubernetes() {

    echo "Collecting Kubernetes logs from namespace '${NAMESPACE}'..." | tee -a "$SUMMARY"

    # Cluster state first - invaluable when a pod crashed or never started.

    {

        echo "===== kubectl get pods -o wide ====="

        kubectl get pods -n "$NAMESPACE" -o wide 2>&1

        echo

        echo "===== kubectl get services ====="

        kubectl get svc -n "$NAMESPACE" 2>&1

        echo

        echo "===== pods not Running/Ready ====="

        kubectl get pods -n "$NAMESPACE" \

            -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].ready}{"\n"}{end}' 2>&1

        echo

        echo "===== recent events ====="

        kubectl get events -n "$NAMESPACE" --sort-by=.lastTimestamp 2>&1 | tail -50

    } > "${OUT_DIR}/${PREFIX}-cluster-state_${TIMESTAMP}.txt" 2>&1

    report_file "${OUT_DIR}/${PREFIX}-cluster-state_${TIMESTAMP}.txt" "cluster state"

    local pods

    pods=$(kubectl get pods -n "$NAMESPACE" \

        -o 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null)

    if [ -z "$pods" ]; then

        echo "  No pods found in namespace '${NAMESPACE}'." | tee -a "$SUMMARY"

        return

    fi

    local pod containers container component name restarts

    for pod in $pods; do

        containers=$(kubectl get pod "$pod" -n "$NAMESPACE" \

            -o 'jsonpath={range .spec.containers[*]}{.name}{"\n"}{end}' 2>/dev/null)

        # contextservice-57bd978fcb-ppk8x -> context

        component=$(echo "$pod" | sed -E 's/-[a-z0-9]+-[a-z0-9]+$//; s/service$//')

        for container in $containers; do

            # Only disambiguate by container when the pod has more than one.

            if [ "$(echo "$containers" | wc -l)" -gt 1 ]; then

                name="${PREFIX}-${component}-${container}"

            else

                name="${PREFIX}-${component}"

            fi

            kubectl logs "$pod" -n "$NAMESPACE" -c "$container" \

                ${TAIL_LINES:+--tail="$TAIL_LINES"} ${SINCE:+--since="$SINCE"} \

                > "${OUT_DIR}/${name}_${TIMESTAMP}.log" 2>&1

            report_file "${OUT_DIR}/${name}_${TIMESTAMP}.log" "$name"

            # If the container has restarted, the pre-restart log holds the crash.

            restarts=$(kubectl get pod "$pod" -n "$NAMESPACE" \

                -o "jsonpath={.status.containerStatuses[?(@.name=='${container}')].restartCount}" 2>/dev/null)

            if [ -n "$restarts" ] && [ "$restarts" -gt 0 ] 2>/dev/null; then

                kubectl logs "$pod" -n "$NAMESPACE" -c "$container" --previous \

                    > "${OUT_DIR}/${name}-previous_${TIMESTAMP}.log" 2>&1

                report_file "${OUT_DIR}/${name}-previous_${TIMESTAMP}.log" "${name} (previous)"

            fi

        done

    done

}

#-------------------------------------------------------------------------------

# Docker: every running container

#-------------------------------------------------------------------------------

collect_docker() {

    echo "Collecting Docker container logs..." | tee -a "$SUMMARY"

    {

        echo "===== docker ps ====="

        docker ps 2>&1

    } > "${OUT_DIR}/${PREFIX}-docker-state_${TIMESTAMP}.txt" 2>&1

    report_file "${OUT_DIR}/${PREFIX}-docker-state_${TIMESTAMP}.txt" "docker state"

    local names name

    names=$(docker ps --format '{{.Names}}' 2>/dev/null)

    for name in $names; do

        docker logs ${TAIL_LINES:+--tail "$TAIL_LINES"} ${SINCE:+--since "$SINCE"} \

            "$name" > "${OUT_DIR}/${PREFIX}-${name}_${TIMESTAMP}.log" 2>&1

        report_file "${OUT_DIR}/${PREFIX}-${name}_${TIMESTAMP}.log" "$name"

    done

}

#-------------------------------------------------------------------------------

# Collect whatever this host actually runs

#-------------------------------------------------------------------------------

COLLECTED_ANY=0

if command -v kubectl >/dev/null 2>&1 && \

   kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then

    collect_kubernetes

    COLLECTED_ANY=1

    echo

fi

if command -v docker >/dev/null 2>&1 && [ -n "$(docker ps -q 2>/dev/null)" ]; then

    collect_docker

    COLLECTED_ANY=1

    echo

fi

if [ "$COLLECTED_ANY" -eq 0 ]; then

    echo "Nothing to collect on ${HOSTNAME_FULL}:" | tee -a "$SUMMARY"

    echo "  no reachable '${NAMESPACE}' Kubernetes namespace and no running Docker containers." \

        | tee -a "$SUMMARY"

    exit 1

fi

printf "\n"

# Convenience pointer to the most recent collection.

rm -f "${OUT_BASE}/latest"

ln -sfn "${PREFIX}_${TIMESTAMP}" "${OUT_BASE}/latest" 2>/dev/null

echo

echo "Summary written to ${SUMMARY}"

echo "Logs collected in ${OUT_DIR} (also reachable as ${OUT_BASE}/latest)"

echo "Done!"