Setting the file. One moment.
Pod Evidence · Azure Diagnostics · microsoft/azure-skills · Skills Docs
Back to the top of the page 30.9
AKS Baseline · scripts
scripts/pod-evidence.sh
scripts/ pod-evidence.sh
Shell · 170 lines · 7 KB
14 #
15 # This script only GATHERS and DIGESTS evidence. It never mutates cluster state.
16 # Interpreting the digest to pick a fix (exit-code / event / probe decision tables)
17 # stays with the caller.
18 #
19 # Usage:
20 # ./pod-evidence.sh <pod> -n <namespace> [--tail <n>]
21 # ./pod-evidence.sh --all-failing [-n <namespace>] [--tail <n>]
22 #
23 # Options:
24 # -n, --namespace <ns> Namespace of the pod. Required in single-pod mode.
25 # In --all-failing mode, limits the scan to this namespace.
26 # --all-failing Auto-select every pod not in Running/Succeeded phase
27 # (across all namespaces unless -n is given) and digest each.
28 # --tail <n> Number of log lines to show per stream (default 50).
29 # -h, --help Show this help.
30 #
31 # Examples:
32 # ./pod-evidence.sh my-api-7d9f-abcde -n prod
33 # ./pod-evidence.sh --all-failing
34 # ./pod-evidence.sh --all-failing -n prod --tail 100
35
36 set -euo pipefail
37
38 POD = ""
39 NAMESPACE = ""
40 ALL_FAILING = false
41 TAIL = 50
42
43 usage () {
44 sed -n '2,40p' " $0 " | sed 's/^# \{0,1\}//'
45 }
46
47 while [ $# -gt 0 ]; do
48 case " $1 " in
49 -n | --namespace ) NAMESPACE = " ${2 :? --namespace requires a value } " ; shift 2 ;;
50 --all-failing ) ALL_FAILING = true ; shift ;;
51 --tail ) TAIL = " ${2 :? --tail requires a value } " ; shift 2 ;;
52 -h | --help ) usage ; exit 0 ;;
53 - * ) echo "Unknown option: $1 " >&2 ; usage ; exit 2 ;;
54 *) POD = " $1 " ; shift ;;
55 esac
56 done
57
58 if ! command -v kubectl > /dev/null 2>&1 ; then
59 echo "ERROR: kubectl not found on PATH." >&2
60 exit 1
61 fi
62
63 case " $TAIL " in
64 '' |* [!0-9] * ) echo "ERROR: --tail must be a positive integer (got ' $TAIL ')." >&2 ; exit 2 ;;
65 0 ) echo "ERROR: --tail must be a positive integer (got ' $TAIL ')." >&2 ; exit 2 ;;
66 esac
67
68 # Digest a single pod. Args: <namespace> <pod>
69 digest_pod () {
70 local ns = " $1 " pod = " $2 "
71
72 echo "=================================================================="
73 echo "POD: $pod NAMESPACE: $ns "
74 echo "=================================================================="
75
76 echo "--- STATUS (ready / phase / restarts) ---"
77 kubectl get pod " $pod " -n " $ns " -o wide 2>&1 || echo "(unable to get pod)"
78 echo ""
79
80 echo "--- STATE (exit code / reason / last state) ---"
81 kubectl get pod " $pod " -n " $ns " -o jsonpath='{range .status.containerStatuses[*]}container={.name}{"\n"} ready={.ready} restarts={.restartCount}{"\n"} current: waiting={.state.waiting.reason} running={.state.running.startedAt} terminated={.state.terminated.reason}(exit={.state.terminated.exitCode}){"\n"} lastState: terminated={.lastState.terminated.reason}(exit={.lastState.terminated.exitCode}) at {.lastState.terminated.finishedAt}{"\n"}{end}' 2> /dev/null \
82 || echo "(no container status available)"
83 echo ""
84
85 echo "--- EVENTS ---"
86 if kubectl describe pod " $pod " -n " $ns " > /dev/null 2>&1 ; then
87 # Read the whole describe output in a single awk pass (no `head` in the pipe:
88 # an early-exiting `head` would SIGPIPE the upstream kubectl and, under
89 # `set -o pipefail`, abort the script). awk consumes all input and prints only
90 # the first 25 lines of the Events section.
91 kubectl describe pod " $pod " -n " $ns " 2> /dev/null | awk '/^Events:/{f=1} f && n<25 {print; n++}'
92 else
93 echo "(unable to describe pod)"
94 fi
95 echo ""
96
97 echo "--- LOGS (current, last $TAIL lines) ---"
98 kubectl logs " $pod " -n " $ns " --tail= " $TAIL " 2>&1 || echo "(no current logs)"
99 echo ""
100
101 echo "--- PREV LOGS (previous instance, last $TAIL lines) ---"
102 if kubectl logs " $pod " -n " $ns " --previous --tail= " $TAIL " 2> /dev/null ; then
103 :
104 else
105 echo "(no previous-instance logs - pod has not restarted or they were rotated)"
106 fi
107 echo ""
108
109 echo "--- RESOURCES (requests/limits vs live usage) ---"
110 echo "requests/limits:"
111 kubectl get pod " $pod " -n " $ns " -o jsonpath='{range .spec.containers[*]} {.name}: requests={.resources.requests} limits={.resources.limits}{"\n"}{end}' 2> /dev/null \
112 || echo " (unable to read resources)"
113 echo "live usage:"
114 kubectl top pod " $pod " -n " $ns " 2>&1 | sed 's/^/ /' || echo " (metrics-server unavailable)"
115 echo ""
116 }
117
118 if [ " $ALL_FAILING " = true ]; then
119 echo "pod-evidence: scanning for pods not in Running/Succeeded${ NAMESPACE : + in namespace '$NAMESPACE'}..."
120 # Portable, set -e-safe row collection: capture output + exit status via command
121 # substitution (not process substitution, whose failures don't propagate under set -e)
122 # so a failed scan is reported as an error instead of a misleading "no unhealthy pods".
123 # Avoids `mapfile`, which is unavailable in Bash 3.2 / macOS.
124 if [ -n " $NAMESPACE " ]; then
125 SCAN_ARGS = ( -n " $NAMESPACE " )
126 else
127 SCAN_ARGS = ( -A )
128 fi
129 set +e
130 SCAN_OUT = $( kubectl get pods "${ SCAN_ARGS [ @ ]}" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2> /dev/null )
131 SCAN_RC = $?
132 set -e
133 if [ " $SCAN_RC " -ne 0 ]; then
134 echo "ERROR: unable to list pods (kubectl exited $SCAN_RC ). Check your cluster context and credentials." >&2
135 exit 1
136 fi
137 ROWS = ()
138 while IFS = read -r line ; do
139 [ -n " $line " ] && ROWS += ( " $line " )
140 done <<< " $SCAN_OUT "
141
142 if [ "${ # ROWS [ @ ]}" -eq 0 ]; then
143 echo "No unhealthy pods found (all pods are Running or Succeeded)."
144 exit 0
145 fi
146
147 echo "Found ${ # ROWS [ @ ]} unhealthy pod(s). Collecting evidence for each below."
148 echo ""
149 for row in "${ ROWS [ @ ]}" ; do
150 [ -z " $row " ] && continue
151 ns = "${ row %% * }"
152 name = "${ row ##* }"
153 digest_pod " $ns " " $name "
154 done
155 echo "pod-evidence: done. Reviewed ${ # ROWS [ @ ]} failing pod(s) - use the STATE/EVENTS/LOGS above to pick a fix."
156 else
157 if [ -z " $POD " ]; then
158 echo "ERROR: a pod name is required (or use --all-failing)." >&2
159 usage
160 exit 2
161 fi
162 if [ -z " $NAMESPACE " ]; then
163 echo "ERROR: -n/--namespace is required in single-pod mode." >&2
164 exit 2
165 fi
166 echo "pod-evidence: collecting the read-only evidence bundle for pod ' $POD ' in namespace ' $NAMESPACE '."
167 echo ""
168 digest_pod " $NAMESPACE " " $POD "
169 echo "pod-evidence: done. Use the STATE/EVENTS/LOGS/RESOURCES digest above to pick a fix."
170 fi