102 lines
5.0 KiB
Bash
102 lines
5.0 KiB
Bash
#!/usr/bin/env bash
|
|||
|
|
# monitor_pack.sh — 1 Hz sampler for I/O- and memory-bound stages (pack, merge_partitions).
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# scripts/monitor_pack.sh --pid PID OUT_DIR [OUTPUT_INDEX_DIR] # attach to a running process
|
||
|
|
# scripts/monitor_pack.sh --cmd OUT_DIR [OUTPUT_INDEX_DIR] -- CMD... # start CMD and sample it
|
||
|
|
# env: DT=1 (seconds between samples)
|
||
|
|
#
|
||
|
|
# Writes OUT_DIR/samples.csv (one row per sample) and OUT_DIR/env.txt (context
|
||
|
|
# needed to read it). Linux only. Never reads smaps (it takes mmap_lock and
|
||
|
|
# slows a process with many mappings); only /proc/PID/{status,io,task/*/stat}
|
||
|
|
# and /proc/meminfo.
|
||
|
|
#
|
||
|
|
# Columns of samples.csv
|
||
|
|
# t epoch seconds
|
||
|
|
# rss_anon_kb, rss_file_kb, rss_shmem_kb from /proc/PID/status
|
||
|
|
# threads thread count
|
||
|
|
# n_R n_D n_S threads running / in uninterruptible sleep (I/O wait) / sleeping
|
||
|
|
# (futex, channel, sleep: NOT disk). n_D is the direct count of
|
||
|
|
# workers blocked on I/O.
|
||
|
|
# maps number of mappings (wc -l /proc/PID/maps)
|
||
|
|
# read_bytes write_bytes /proc/PID/io (cumulative, bytes submitted to the
|
||
|
|
# block layer; buffered writes counted at page dirtying)
|
||
|
|
# rchar wchar /proc/PID/io (cumulative, bytes through read()/write() syscalls)
|
||
|
|
# dirty_kb writeback_kb cached_kb mem_avail_kb /proc/meminfo
|
||
|
|
#
|
||
|
|
# Pair it with the process's own log (RUST_LOG=info, plus obisys=debug for the
|
||
|
|
# "activated N worker(s)" lines and obikindex=debug for per-partition timings)
|
||
|
|
# to align stages on wall-clock time: the log timestamps are UTC ISO-8601, `t`
|
||
|
|
# is epoch seconds.
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
MODE=${1:-}; shift || true
|
||
|
|
case "$MODE" in
|
||
|
|
--pid) PID=${1:?PID}; shift ;;
|
||
|
|
--cmd) ;;
|
||
|
|
*) sed -n 2,12p "$0"; exit 2 ;;
|
||
|
|
esac
|
||
|
|
OUT=${1:?OUT_DIR}; shift
|
||
|
|
IDX=""
|
||
|
|
if [ "${1:-}" != "" ] && [ "${1:-}" != "--" ]; then IDX=$1; shift; fi
|
||
|
|
mkdir -p "$OUT"
|
||
|
|
|
||
|
|
if [ "$MODE" = "--cmd" ]; then
|
||
|
|
[ "${1:-}" = "--" ] || { echo "--cmd needs: -- CMD..." >&2; exit 2; }
|
||
|
|
shift
|
||
|
|
"$@" 2>"$OUT/cmd.stderr" &
|
||
|
|
PID=$!
|
||
|
|
fi
|
||
|
|
|
||
|
|
{
|
||
|
|
echo "date_utc: $(date -u +%FT%TZ)"
|
||
|
|
echo "pid: $PID"
|
||
|
|
echo "nproc: $(nproc)"
|
||
|
|
echo "kernel: $(uname -r)"
|
||
|
|
grep -E 'MemTotal|SwapTotal' /proc/meminfo
|
||
|
|
for k in vm.dirty_ratio vm.dirty_background_ratio vm.dirty_bytes vm.dirty_background_bytes \
|
||
|
|
vm.dirty_expire_centisecs vm.dirty_writeback_centisecs vm.max_map_count; do
|
||
|
|
echo "$k: $(sysctl -n $k 2>/dev/null || echo '?')"
|
||
|
|
done
|
||
|
|
echo "ulimit -n: $(ulimit -n)"
|
||
|
|
echo "cmdline: $(tr '\0' ' ' </proc/$PID/cmdline 2>/dev/null)"
|
||
|
|
if [ -n "$IDX" ]; then
|
||
|
|
echo "output_dir: $IDX"
|
||
|
|
echo "fs_type: $(stat -f -c %T "$IDX" 2>/dev/null || echo '?')"
|
||
|
|
echo "mount: $(df -P "$IDX" 2>/dev/null | tail -1)"
|
||
|
|
command -v beegfs-ctl >/dev/null && echo "beegfs: $(beegfs-ctl --getentryinfo "$IDX" 2>&1 | head -8 | tr '\n' ' ')"
|
||
|
|
fi
|
||
|
|
env | grep -E '^(MALLOC_|RUST_LOG|RAYON_)' || true
|
||
|
|
} >"$OUT/env.txt"
|
||
|
|
|
||
|
|
DT=${DT:-1}
|
||
|
|
printf 't,rss_anon_kb,rss_file_kb,rss_shmem_kb,threads,n_R,n_D,n_S,maps,read_bytes,write_bytes,rchar,wchar,dirty_kb,writeback_kb,cached_kb,mem_avail_kb\n' >"$OUT/samples.csv"
|
||
|
|
|
||
|
|
while [ -d "/proc/$PID" ]; do
|
||
|
|
t=$(date +%s)
|
||
|
|
st=$(awk '/^RssAnon:/{a=$2} /^RssFile:/{f=$2} /^RssShmem:/{s=$2} /^Threads:/{th=$2} END{printf "%s,%s,%s,%s",a,f,s,th}' "/proc/$PID/status" 2>/dev/null) || break
|
||
|
|
# thread states: field 3 of /proc/PID/task/TID/stat, after the ")" closing comm
|
||
|
|
states=$(cat /proc/$PID/task/*/stat 2>/dev/null | sed -E 's/^.*\) (.).*/\1/' | sort | uniq -c \
|
||
|
|
| awk '{c[$2]=$1} END{printf "%d,%d,%d", c["R"]+0, c["D"]+0, c["S"]+0}')
|
||
|
|
maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null || echo 0)
|
||
|
|
io=$(awk '/^rchar:/{rc=$2} /^wchar:/{wc=$2} /^read_bytes:/{r=$2} /^write_bytes:/{w=$2} END{printf "%s,%s,%s,%s",r,w,rc,wc}' "/proc/$PID/io" 2>/dev/null || echo ",,,")
|
||
|
|
mi=$(awk '/^Dirty:/{d=$2} /^Writeback:/{w=$2} /^Cached:/{c=$2} /^MemAvailable:/{m=$2} END{printf "%s,%s,%s,%s",d,w,c,m}' /proc/meminfo)
|
||
|
|
echo "$t,$st,$states,$maps,$io,$mi" >>"$OUT/samples.csv"
|
||
|
|
sleep "$DT"
|
||
|
|
done
|
||
|
|
|
||
|
|
if [ "$MODE" = "--cmd" ]; then wait "$PID" || echo "command exit code: $?" >>"$OUT/env.txt"; fi
|
||
|
|
|
||
|
|
# ── summary ──────────────────────────────────────────────────────────────────
|
||
|
|
awk -F, 'NR==1{next}
|
||
|
|
{ n++; if(NR==2){t0=$1; r0=$10; w0=$11}
|
||
|
|
if($2>ma)ma=$2; if($3>mf)mf=$3; if($4>ms)ms=$4; if($5>mt)mt=$5
|
||
|
|
if($6>mR)mR=$6; if($7>mD)mD=$7; if($8>mS)mS=$8; sR+=$6; sD+=$7; sS+=$8
|
||
|
|
if($14>md)md=$14; if($15>mw)mw=$15; if($9>mm)mm=$9
|
||
|
|
t1=$1; r1=$10; w1=$11 }
|
||
|
|
END{ if(!n)exit
|
||
|
|
printf "samples: %d over %ds\n", n, t1-t0
|
||
|
|
printf "peak RssAnon %.1f GB | RssFile %.1f GB | RssShmem %.1f GB | threads %d | maps %d\n", ma/1e6, mf/1e6, ms/1e6, mt, mm
|
||
|
|
printf "threads by state (mean / max): R %.1f / %d | D(I/O wait) %.1f / %d | S %.1f / %d\n", sR/n, mR, sD/n, mD, sS/n, mS
|
||
|
|
printf "peak Dirty %.1f GB | Writeback %.1f GB\n", md/1e6, mw/1e6
|
||
|
|
printf "block I/O over the run: read %.1f GB, write %.1f GB\n", (r1-r0)/1e9, (w1-w0)/1e9 }' "$OUT/samples.csv" | tee "$OUT/summary.txt"
|