#!/usr/bin/env bash # monitor_pack.sh — 1 Hz sampler for I/O- and memory-bound stages (pack, merge_partitions). # # Usage: # scripts/monitor_pack.sh --pid PID OUT_DIR [OUTPUT_INDEX_DIR] # attach to a running process # scripts/monitor_pack.sh --cmd OUT_DIR [OUTPUT_INDEX_DIR] -- CMD... # start CMD and sample it # env: DT=1 (seconds between samples) # # Writes OUT_DIR/samples.csv (one row per sample) and OUT_DIR/env.txt (context # needed to read it). Linux only. Never reads smaps (it takes mmap_lock and # slows a process with many mappings); only /proc/PID/{status,io,task/*/stat} # and /proc/meminfo. # # Columns of samples.csv # t epoch seconds # rss_anon_kb, rss_file_kb, rss_shmem_kb from /proc/PID/status # threads thread count # n_R n_D n_S threads running / in uninterruptible sleep (I/O wait) / sleeping # (futex, channel, sleep: NOT disk). n_D is the direct count of # workers blocked on I/O. # maps number of mappings (wc -l /proc/PID/maps) # read_bytes write_bytes /proc/PID/io (cumulative, bytes submitted to the # block layer; buffered writes counted at page dirtying) # rchar wchar /proc/PID/io (cumulative, bytes through read()/write() syscalls) # dirty_kb writeback_kb cached_kb mem_avail_kb /proc/meminfo # # Pair it with the process's own log (RUST_LOG=info, plus obisys=debug for the # "activated N worker(s)" lines and obikindex=debug for per-partition timings) # to align stages on wall-clock time: the log timestamps are UTC ISO-8601, `t` # is epoch seconds. set -euo pipefail MODE=${1:-}; shift || true case "$MODE" in --pid) PID=${1:?PID}; shift ;; --cmd) ;; *) sed -n 2,12p "$0"; exit 2 ;; esac OUT=${1:?OUT_DIR}; shift IDX="" if [ "${1:-}" != "" ] && [ "${1:-}" != "--" ]; then IDX=$1; shift; fi mkdir -p "$OUT" if [ "$MODE" = "--cmd" ]; then [ "${1:-}" = "--" ] || { echo "--cmd needs: -- CMD..." >&2; exit 2; } shift "$@" 2>"$OUT/cmd.stderr" & PID=$! fi { echo "date_utc: $(date -u +%FT%TZ)" echo "pid: $PID" echo "nproc: $(nproc)" echo "kernel: $(uname -r)" grep -E 'MemTotal|SwapTotal' /proc/meminfo for k in vm.dirty_ratio vm.dirty_background_ratio vm.dirty_bytes vm.dirty_background_bytes \ vm.dirty_expire_centisecs vm.dirty_writeback_centisecs vm.max_map_count; do echo "$k: $(sysctl -n $k 2>/dev/null || echo '?')" done echo "ulimit -n: $(ulimit -n)" echo "cmdline: $(tr '\0' ' ' /dev/null)" if [ -n "$IDX" ]; then echo "output_dir: $IDX" echo "fs_type: $(stat -f -c %T "$IDX" 2>/dev/null || echo '?')" echo "mount: $(df -P "$IDX" 2>/dev/null | tail -1)" command -v beegfs-ctl >/dev/null && echo "beegfs: $(beegfs-ctl --getentryinfo "$IDX" 2>&1 | head -8 | tr '\n' ' ')" fi env | grep -E '^(MALLOC_|RUST_LOG|RAYON_)' || true } >"$OUT/env.txt" DT=${DT:-1} printf 't,rss_anon_kb,rss_file_kb,rss_shmem_kb,threads,n_R,n_D,n_S,maps,read_bytes,write_bytes,rchar,wchar,dirty_kb,writeback_kb,cached_kb,mem_avail_kb\n' >"$OUT/samples.csv" while [ -d "/proc/$PID" ]; do t=$(date +%s) st=$(awk '/^RssAnon:/{a=$2} /^RssFile:/{f=$2} /^RssShmem:/{s=$2} /^Threads:/{th=$2} END{printf "%s,%s,%s,%s",a,f,s,th}' "/proc/$PID/status" 2>/dev/null) || break # thread states: field 3 of /proc/PID/task/TID/stat, after the ")" closing comm states=$(cat /proc/$PID/task/*/stat 2>/dev/null | sed -E 's/^.*\) (.).*/\1/' | sort | uniq -c \ | awk '{c[$2]=$1} END{printf "%d,%d,%d", c["R"]+0, c["D"]+0, c["S"]+0}') maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null || echo 0) io=$(awk '/^rchar:/{rc=$2} /^wchar:/{wc=$2} /^read_bytes:/{r=$2} /^write_bytes:/{w=$2} END{printf "%s,%s,%s,%s",r,w,rc,wc}' "/proc/$PID/io" 2>/dev/null || echo ",,,") mi=$(awk '/^Dirty:/{d=$2} /^Writeback:/{w=$2} /^Cached:/{c=$2} /^MemAvailable:/{m=$2} END{printf "%s,%s,%s,%s",d,w,c,m}' /proc/meminfo) echo "$t,$st,$states,$maps,$io,$mi" >>"$OUT/samples.csv" sleep "$DT" done if [ "$MODE" = "--cmd" ]; then wait "$PID" || echo "command exit code: $?" >>"$OUT/env.txt"; fi # ── summary ────────────────────────────────────────────────────────────────── awk -F, 'NR==1{next} { n++; if(NR==2){t0=$1; r0=$10; w0=$11} if($2>ma)ma=$2; if($3>mf)mf=$3; if($4>ms)ms=$4; if($5>mt)mt=$5 if($6>mR)mR=$6; if($7>mD)mD=$7; if($8>mS)mS=$8; sR+=$6; sD+=$7; sS+=$8 if($14>md)md=$14; if($15>mw)mw=$15; if($9>mm)mm=$9 t1=$1; r1=$10; w1=$11 } END{ if(!n)exit printf "samples: %d over %ds\n", n, t1-t0 printf "peak RssAnon %.1f GB | RssFile %.1f GB | RssShmem %.1f GB | threads %d | maps %d\n", ma/1e6, mf/1e6, ms/1e6, mt, mm printf "threads by state (mean / max): R %.1f / %d | D(I/O wait) %.1f / %d | S %.1f / %d\n", sR/n, mR, sD/n, mD, sS/n, mS printf "peak Dirty %.1f GB | Writeback %.1f GB\n", md/1e6, mw/1e6 printf "block I/O over the run: read %.1f GB, write %.1f GB\n", (r1-r0)/1e9, (w1-w0)/1e9 }' "$OUT/samples.csv" | tee "$OUT/summary.txt"