#!/usr/bin/env bash # watch_merge.sh PID OUT.csv — samples a running process and kills it past a threshold. # # Every INTERVAL seconds appends one CSV row: number of memory mappings # (lines of /proc/PID/maps), VmSize, VmData, VmRSS, threads, and the # system-wide Committed_AS / CommitLimit. If the mapping count exceeds # MAPS_MAX, VmRSS exceeds RSS_MAX_GB, or VmSize exceeds VMSIZE_MAX_GB (stay # below `ulimit -v`: past it, allocations fail instead of the guard firing), the process receives SIGTERM (then # SIGKILL after 30 s) and the reason is written to OUT.csv.reason — a guard # so a runaway merge ends with a clean message instead of a failed # allocation (or the OOM killer on a shared node). # # Reading /proc/PID/maps is itself slow when the process has hundreds of # thousands of mappings (it held the kernel's mmap lock in earlier # measurements): keep INTERVAL >= 5. It never reads /proc/PID/smaps. # # Usage: scripts/watch_merge.sh PID OUT.csv # env: INTERVAL=5 MAPS_MAX=500000 RSS_MAX_GB=400 VMSIZE_MAX_GB=440 # # Typical use (Linux only): # nohup scripts/watch_merge.sh "$(pgrep -f 'obikmer merge' | head -1)" /tmp/watch.csv & set -u PID=${1:?usage: watch_merge.sh PID OUT.csv} OUT=${2:?usage: watch_merge.sh PID OUT.csv} INTERVAL=${INTERVAL:-5} MAPS_MAX=${MAPS_MAX:-500000} RSS_MAX_KB=$(( ${RSS_MAX_GB:-400} * 1024 * 1024 )) VMSIZE_MAX_KB=$(( ${VMSIZE_MAX_GB:-440} * 1024 * 1024 )) [ -r "/proc/$PID/maps" ] || { echo "cannot read /proc/$PID/maps" >&2; exit 1; } printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT" while kill -0 "$PID" 2>/dev/null; do maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" 2>/dev/null | awk '{printf "%s,", $2}') rss=$(grep '^VmRSS:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}') vsz=$(grep '^VmSize:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}') mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}') echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT" reason="" [ "$maps" -gt "$MAPS_MAX" ] && reason="mappings $maps > MAPS_MAX $MAPS_MAX" [ -n "${rss:-}" ] && [ "$rss" -gt "$RSS_MAX_KB" ] && reason="VmRSS ${rss} kB > RSS_MAX_GB" [ -n "${vsz:-}" ] && [ "$vsz" -gt "$VMSIZE_MAX_KB" ] && reason="VmSize ${vsz} kB > VMSIZE_MAX_GB" if [ -n "$reason" ]; then echo "$(date +%T) GUARD: $reason — sending SIGTERM to $PID" | tee "$OUT.reason" >&2 kill -TERM "$PID" 2>/dev/null for _ in $(seq 30); do kill -0 "$PID" 2>/dev/null || exit 0; sleep 1; done kill -KILL "$PID" 2>/dev/null exit 0 fi sleep "$INTERVAL" done