Thermal Sample

1#!/usr/bin/env bash
2# Background thermal/power sampler for a long training run.
3# Output contract: CSV, one line per sample, matching
4# `nvidia-smi --query-gpu` field order — pipe-compatible with
5# the training log for later correlation by timestamp.
6# Usage: bash thermal-sample.sh [interval_seconds] [logfile]
7set -uo pipefail
8
9command -v nvidia-smi >/dev/null || { echo "nvidia-smi not found" >&2; exit 1; }
10
11INTERVAL="${1:-30}"
12LOGFILE="${2:-thermal.log}"
13PIDFILE="${LOGFILE}.pid"
14
15if [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
16  echo "Stopping existing sampler (pid $(cat "$PIDFILE"))"
17  kill "$(cat "$PIDFILE")"
18fi
19
20echo "timestamp,temperature.gpu,power.draw" > "$LOGFILE"
21nvidia-smi --query-gpu=timestamp,temperature.gpu,power.draw \
22  --format=csv,noheader -l "$INTERVAL" >> "$LOGFILE" &
23PID=$!
24echo "$PID" > "$PIDFILE"
25echo "Sampling every ${INTERVAL}s into ${LOGFILE} (pid $PID)"
26echo "A sustained ~100W reading is the platform cap, not a bug — see SKILL.md Thermal Monitoring."