feat: add pack optimization prompt, debug infrastructure, and monitoring scripts
Add a prompt template for optimizing the `pack` step and a suite of Docker-based debug tools including build, run, and monitoring scripts. Enhance `pack_matrices` with LFD partition sizing and detailed timing reports, and update `stop` with additional system statistics. - Add `prompts/pack_optimization.md` to guide optimization of the `pack` step, specifying context, performance measurements, and deliverable structures. - Add `scripts/docker_debug/` infrastructure: Dockerfile for musl targets, `build.sh` for image creation, `run.sh` for parallel indexing and merging, `monitor.sh` for PID tracking, `gen_fasta.py` for synthetic genomes, and `gen_fasta_real.py` for real genome generation with `read_fna`. - Add `scripts/monitor_pack.sh` as a 1 Hz sampler for I/O and memory metrics, writing samples to CSV and summary reports. - Add `scripts/watch_merge.sh` to monitor process resource usage, enforcing memory limits via SIGTERM/SIGKILL guards based on VmRSS and VmSize. - Modify `src/obikindex/src/index/kmer_index.rs`: add `pending_matrix_bytes` and `report_pack_timings`; update `pack_matrices` to use LFD strategy for partition sizing and clarify documentation regarding `stat()` proxy bytes. - Modify `src/obisys/src/stage.rs`: update `stop` definition to include new variables (`user_secs`, `sys_secs`, faults, context switches, I/O blocks) and logic checking for swap usage.
This commit is contained in:
1 parent
24291a4484
commit
03eaa4544a
11 files changed
+497
-2
No files matched your search
@@ -0,0 +1,16 @@
|
||||
# Linux debug environment for the merge mapping/memory-growth hunt.
|
||||
# Same toolchain as .gitea/workflows/release.yml for the musl target
|
||||
# (cargo-zigbuild + zig cc), plus a glibc toolchain and diagnostic tools.
|
||||
FROM rust:1-bookworm
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential cmake pkg-config autoconf automake libtool \
|
||||
heaptrack valgrind gdb strace procps python3 python3-pip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN pip install ziglang --quiet --break-system-packages \
|
||||
&& cargo install cargo-zigbuild \
|
||||
&& rustup target add x86_64-unknown-linux-musl
|
||||
RUN ZIG=$(python3 -c "import ziglang, os; print(os.path.join(os.path.dirname(ziglang.__file__), 'zig'))") \
|
||||
&& printf '#!/bin/sh\nexec "%s" cc -target x86_64-linux-musl "$@"\n' "$ZIG" > /usr/local/bin/x86_64-linux-musl-gcc \
|
||||
&& printf '#!/bin/sh\nexec "%s" c++ -target x86_64-linux-musl "$@"\n' "$ZIG" > /usr/local/bin/x86_64-linux-musl-g++ \
|
||||
&& chmod +x /usr/local/bin/x86_64-linux-musl-gcc /usr/local/bin/x86_64-linux-musl-g++
|
||||
WORKDIR /work
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Builds the debug image. PLATFORM defaults to linux/amd64 (Rosetta on Apple Silicon).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
PLATFORM="${PLATFORM:-linux/amd64}"
|
||||
docker build --platform "$PLATFORM" -t "obikmer-debug:${PLATFORM//\//-}" .
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gen_fasta.py N OUTDIR [GENOME_LEN] — N deterministic synthetic genomes.
|
||||
|
||||
Each genome = unique random sequence + a random subset of a shared fragment
|
||||
pool (some fragments lightly mutated), so a merge both hits k-mers already
|
||||
in the destination and creates new ones, like related genomes would.
|
||||
"""
|
||||
import random, sys
|
||||
|
||||
n, out = int(sys.argv[1]), sys.argv[2]
|
||||
glen = int(sys.argv[3]) if len(sys.argv) > 3 else 150_000
|
||||
seed = int(sys.argv[4]) if len(sys.argv) > 4 else 20261118
|
||||
rng = random.Random(seed)
|
||||
|
||||
def rnd(m):
|
||||
return "".join(rng.choices("ACGT", k=m))
|
||||
|
||||
def mutate(s, rate=0.01):
|
||||
s = list(s)
|
||||
for i in range(len(s)):
|
||||
if rng.random() < rate:
|
||||
s[i] = rng.choice("ACGT")
|
||||
return "".join(s)
|
||||
|
||||
pool = [rnd(2000) for _ in range(300)] # 600 kb shared pool
|
||||
for g in range(n):
|
||||
parts = [rnd(glen)]
|
||||
for f in rng.sample(range(len(pool)), 60):
|
||||
parts.append(mutate(pool[f]) if rng.random() < 0.3 else pool[f])
|
||||
rng.shuffle(parts)
|
||||
with open(f"{out}/G{g:03d}.fasta", "w") as fh:
|
||||
for j, p in enumerate(parts):
|
||||
fh.write(f">G{g:03d}_{j}\n{p}\n")
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gen_fasta_real.py N OUTDIR GENOMES_DIR [WINDOW] [SEED] [MUT]
|
||||
|
||||
N derived genomes built from *real* genomes (GENOMES_DIR/*.fna.gz): each one is
|
||||
a random window of WINDOW bp of a randomly chosen real genome, point-mutated at
|
||||
rate MUT (default 3 %, which alters ~47 % of 21-mers), so a merge both finds
|
||||
k-mers already present in the destination and creates new ones, with realistic
|
||||
sequence statistics (no dedicated degenerate structure).
|
||||
"""
|
||||
import glob, gzip, random, sys
|
||||
|
||||
n, out, gdir = int(sys.argv[1]), sys.argv[2], sys.argv[3]
|
||||
window = int(sys.argv[4]) if len(sys.argv) > 4 else 300_000
|
||||
seed = int(sys.argv[5]) if len(sys.argv) > 5 else 20261118
|
||||
mut = float(sys.argv[6]) if len(sys.argv) > 6 else 0.03
|
||||
rng = random.Random(seed)
|
||||
|
||||
def read_fna(path):
|
||||
seq = []
|
||||
with gzip.open(path, "rt") as fh:
|
||||
for line in fh:
|
||||
if not line.startswith(">"):
|
||||
seq.append(line.strip().upper())
|
||||
return "".join(seq)
|
||||
|
||||
genomes = [read_fna(p) for p in sorted(glob.glob(f"{gdir}/*.fna.gz"))]
|
||||
genomes = [g for g in genomes if len(g) > window]
|
||||
for i in range(n):
|
||||
g = rng.choice(genomes)
|
||||
start = rng.randrange(0, len(g) - window)
|
||||
s = list(g[start:start + window])
|
||||
for j in range(len(s)):
|
||||
if s[j] in "ACGT" and rng.random() < mut:
|
||||
s[j] = rng.choice("ACGT")
|
||||
seq = "".join(c if c in "ACGT" else "A" for c in s)
|
||||
with open(f"{out}/R{i:03d}.fasta", "w") as fh:
|
||||
fh.write(f">R{i:03d}\n")
|
||||
for k in range(0, len(seq), 80):
|
||||
fh.write(seq[k:k + 80] + "\n")
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# monitor.sh PID OUT [INTERVAL_S] — samples mappings / VM / commit figures until PID exits.
|
||||
PID=$1; OUT=$2; DT=${3:-5}
|
||||
printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT"
|
||||
while kill -0 "$PID" 2>/dev/null; do
|
||||
maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break
|
||||
st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" | awk '{printf "%s,", $2}')
|
||||
mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}')
|
||||
echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT"
|
||||
sleep "$DT"
|
||||
done
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# run.sh — builds obikmer (release, debug symbols) and merges the benchmark presence
|
||||
# indexes in the container while sampling mappings/RSS/commit in parallel.
|
||||
#
|
||||
# Env:
|
||||
# PLATFORM linux/amd64 (default, Rosetta) | linux/arm64
|
||||
# GEN if set: build GEN synthetic indexes (gen_fasta.py) instead of using
|
||||
# the benchmark ones; GENOME_LEN (default 150000), K (21), M (9), PART (256)
|
||||
# REAL if set (with GEN): derive genomes from benchmark/run/genomes (gen_fasta_real.py)
|
||||
# SRC_DIR alternative source tree (default: <repo>/src); VARIANT: volume suffix;
|
||||
# LOCKED=0 to allow Cargo.lock changes (needed when a dep is added)
|
||||
# NSRC number of source indexes to merge (default: all 20)
|
||||
# TARGET gnu (default, glibc) | musl (static, same chain as the release CI)
|
||||
# TOOL none (default) | heaptrack (heaptrack output goes to OUT/)
|
||||
# MAXMAP if set, vm.max_map_count is lowered to this inside the VM for the
|
||||
# run and restored afterwards (needs --privileged; affects the whole
|
||||
# OrbStack VM while it runs)
|
||||
# MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_, MALLOC_TRIM_THRESHOLD_ passed through
|
||||
# OUT host output directory (default: ./out/<timestamp>)
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO="$(cd "$HERE/../.." && pwd)"
|
||||
PLATFORM="${PLATFORM:-linux/amd64}"
|
||||
NSRC="${NSRC:-20}"
|
||||
TOOL="${TOOL:-none}"
|
||||
TARGET="${TARGET:-gnu}"
|
||||
OUT="${OUT:-$HERE/out/$(date +%Y%m%d-%H%M%S)}"
|
||||
mkdir -p "$OUT"
|
||||
IMG="obikmer-debug:${PLATFORM//\//-}"
|
||||
VOL="obikmer-debug-target-${PLATFORM//\//-}-${TARGET}${VARIANT:+-$VARIANT}"
|
||||
|
||||
docker run --rm --privileged --platform "$PLATFORM" \
|
||||
-v "${SRC_DIR:-$REPO/src}:/work/src:$([ -n "${SRC_DIR:-}" ] && echo rw || echo ro)" \
|
||||
-v "$REPO/benchmark/run/specimen_index_presence:/idx_ro:ro" \
|
||||
-v "$VOL:/target" \
|
||||
-v "$OUT:/out" \
|
||||
-v "$HERE/monitor.sh:/monitor.sh:ro" \
|
||||
-v "$HERE/gen_fasta.py:/gen_fasta.py:ro" \
|
||||
-v "$HERE/gen_fasta_real.py:/gen_fasta_real.py:ro" \
|
||||
-v "$REPO/benchmark/run/genomes:/genomes_ro:ro" \
|
||||
-e CARGO_TARGET_DIR=/target -e CARGO_PROFILE_RELEASE_DEBUG=1 \
|
||||
-e GEN="${GEN:-}" -e LOCKED_FLAG="$([ "${LOCKED:-1}" = 0 ] || echo --locked)" -e SEED="${SEED:-20261118}" -e REAL="${REAL:-}" -e GENOME_LEN="${GENOME_LEN:-150000}" -e K="${K:-21}" -e M="${M:-9}" -e PART="${PART:-256}" -e RUST_LOG="${RUST_LOG:-}" -e NSRC="$NSRC" -e TARGET="$TARGET" -e TOOL="$TOOL" -e MAXMAP="${MAXMAP:-}" \
|
||||
-e MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-}" \
|
||||
-e CXXFLAGS="${CXXFLAGS:-}" \
|
||||
-e MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-}" \
|
||||
-e MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:-}" \
|
||||
"$IMG" bash -c '
|
||||
set -euo pipefail
|
||||
for v in MALLOC_ARENA_MAX MALLOC_MMAP_THRESHOLD_ MALLOC_TRIM_THRESHOLD_ RUST_LOG; do
|
||||
[ -n "${!v}" ] || unset $v
|
||||
done
|
||||
if [ "$TARGET" = musl ]; then
|
||||
PKG_CONFIG_ALLOW_CROSS=1 cargo zigbuild --release $LOCKED_FLAG --target x86_64-unknown-linux-musl \
|
||||
--manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
|
||||
BIN=/target/x86_64-unknown-linux-musl/release/obikmer
|
||||
else
|
||||
cargo build --release $LOCKED_FLAG --manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
|
||||
BIN=/target/release/obikmer
|
||||
fi
|
||||
file "$BIN" 2>/dev/null | cut -c1-150 || true
|
||||
# merge writes into its sources (lock files…): work on a copy, never on benchmark/
|
||||
rm -rf /tmp/idx && mkdir -p /tmp/idx
|
||||
if [ -n "$GEN" ]; then
|
||||
mkdir -p /tmp/fa
|
||||
if [ -n "$REAL" ]; then python3 /gen_fasta_real.py "$GEN" /tmp/fa /genomes_ro "$GENOME_LEN" "$SEED"
|
||||
else python3 /gen_fasta.py "$GEN" /tmp/fa "$GENOME_LEN" "$SEED"; fi
|
||||
NSRC=$GEN
|
||||
echo "indexing $GEN synthetic genomes (k=$K m=$M p=$PART)…"
|
||||
ls /tmp/fa/*.fasta | xargs -P 8 -I{} sh -c \
|
||||
"b=\$(basename {} .fasta); $BIN index -k $K -m $M --theta 0 -p $PART --label \$b -o /tmp/idx/\$b.idx {} >/dev/null 2>&1 || echo index failed: \$b"
|
||||
else
|
||||
cp -r /idx_ro/. /tmp/idx/
|
||||
fi
|
||||
mapfile -t SRC < <(find /tmp/idx -mindepth 1 -maxdepth 1 -type d | sort | head -n "$NSRC")
|
||||
echo "sources: ${#SRC[@]} target: $TARGET tool: $TOOL max_map_count: $(cat /proc/sys/vm/max_map_count)"
|
||||
if [ -n "$MAXMAP" ]; then
|
||||
OLD=$(cat /proc/sys/vm/max_map_count); sysctl -w vm.max_map_count="$MAXMAP" >/dev/null
|
||||
trap "sysctl -w vm.max_map_count=$OLD >/dev/null" EXIT
|
||||
fi
|
||||
rm -rf /tmp/merged
|
||||
CMD=("$BIN" merge --output /tmp/merged --force --force-presence "${SRC[@]}")
|
||||
if [ "$TOOL" = heaptrack ]; then CMD=(heaptrack -o /out/heaptrack "${CMD[@]}"); fi
|
||||
"${CMD[@]}" 2>/out/merge.stderr &
|
||||
PID=$!
|
||||
/monitor.sh "$PID" /out/monitor.csv 1 &
|
||||
MON=$!
|
||||
set +e; wait "$PID"; RC=$?; set -e
|
||||
wait "$MON" || true
|
||||
echo "merge exit code: $RC" | tee /out/exit_code.txt
|
||||
tail -5 /out/merge.stderr
|
||||
'
|
||||
echo "outputs: $OUT"
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env bash
|
||||
# monitor_pack.sh — 1 Hz sampler for I/O- and memory-bound stages (pack, merge_partitions).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/monitor_pack.sh --pid PID OUT_DIR [OUTPUT_INDEX_DIR] # attach to a running process
|
||||
# scripts/monitor_pack.sh --cmd OUT_DIR [OUTPUT_INDEX_DIR] -- CMD... # start CMD and sample it
|
||||
# env: DT=1 (seconds between samples)
|
||||
#
|
||||
# Writes OUT_DIR/samples.csv (one row per sample) and OUT_DIR/env.txt (context
|
||||
# needed to read it). Linux only. Never reads smaps (it takes mmap_lock and
|
||||
# slows a process with many mappings); only /proc/PID/{status,io,task/*/stat}
|
||||
# and /proc/meminfo.
|
||||
#
|
||||
# Columns of samples.csv
|
||||
# t epoch seconds
|
||||
# rss_anon_kb, rss_file_kb, rss_shmem_kb from /proc/PID/status
|
||||
# threads thread count
|
||||
# n_R n_D n_S threads running / in uninterruptible sleep (I/O wait) / sleeping
|
||||
# (futex, channel, sleep: NOT disk). n_D is the direct count of
|
||||
# workers blocked on I/O.
|
||||
# maps number of mappings (wc -l /proc/PID/maps)
|
||||
# read_bytes write_bytes /proc/PID/io (cumulative, bytes submitted to the
|
||||
# block layer; buffered writes counted at page dirtying)
|
||||
# rchar wchar /proc/PID/io (cumulative, bytes through read()/write() syscalls)
|
||||
# dirty_kb writeback_kb cached_kb mem_avail_kb /proc/meminfo
|
||||
#
|
||||
# Pair it with the process's own log (RUST_LOG=info, plus obisys=debug for the
|
||||
# "activated N worker(s)" lines and obikindex=debug for per-partition timings)
|
||||
# to align stages on wall-clock time: the log timestamps are UTC ISO-8601, `t`
|
||||
# is epoch seconds.
|
||||
set -euo pipefail
|
||||
|
||||
MODE=${1:-}; shift || true
|
||||
case "$MODE" in
|
||||
--pid) PID=${1:?PID}; shift ;;
|
||||
--cmd) ;;
|
||||
*) sed -n 2,12p "$0"; exit 2 ;;
|
||||
esac
|
||||
OUT=${1:?OUT_DIR}; shift
|
||||
IDX=""
|
||||
if [ "${1:-}" != "" ] && [ "${1:-}" != "--" ]; then IDX=$1; shift; fi
|
||||
mkdir -p "$OUT"
|
||||
|
||||
if [ "$MODE" = "--cmd" ]; then
|
||||
[ "${1:-}" = "--" ] || { echo "--cmd needs: -- CMD..." >&2; exit 2; }
|
||||
shift
|
||||
"$@" 2>"$OUT/cmd.stderr" &
|
||||
PID=$!
|
||||
fi
|
||||
|
||||
{
|
||||
echo "date_utc: $(date -u +%FT%TZ)"
|
||||
echo "pid: $PID"
|
||||
echo "nproc: $(nproc)"
|
||||
echo "kernel: $(uname -r)"
|
||||
grep -E 'MemTotal|SwapTotal' /proc/meminfo
|
||||
for k in vm.dirty_ratio vm.dirty_background_ratio vm.dirty_bytes vm.dirty_background_bytes \
|
||||
vm.dirty_expire_centisecs vm.dirty_writeback_centisecs vm.max_map_count; do
|
||||
echo "$k: $(sysctl -n $k 2>/dev/null || echo '?')"
|
||||
done
|
||||
echo "ulimit -n: $(ulimit -n)"
|
||||
echo "cmdline: $(tr '\0' ' ' </proc/$PID/cmdline 2>/dev/null)"
|
||||
if [ -n "$IDX" ]; then
|
||||
echo "output_dir: $IDX"
|
||||
echo "fs_type: $(stat -f -c %T "$IDX" 2>/dev/null || echo '?')"
|
||||
echo "mount: $(df -P "$IDX" 2>/dev/null | tail -1)"
|
||||
command -v beegfs-ctl >/dev/null && echo "beegfs: $(beegfs-ctl --getentryinfo "$IDX" 2>&1 | head -8 | tr '\n' ' ')"
|
||||
fi
|
||||
env | grep -E '^(MALLOC_|RUST_LOG|RAYON_)' || true
|
||||
} >"$OUT/env.txt"
|
||||
|
||||
DT=${DT:-1}
|
||||
printf 't,rss_anon_kb,rss_file_kb,rss_shmem_kb,threads,n_R,n_D,n_S,maps,read_bytes,write_bytes,rchar,wchar,dirty_kb,writeback_kb,cached_kb,mem_avail_kb\n' >"$OUT/samples.csv"
|
||||
|
||||
while [ -d "/proc/$PID" ]; do
|
||||
t=$(date +%s)
|
||||
st=$(awk '/^RssAnon:/{a=$2} /^RssFile:/{f=$2} /^RssShmem:/{s=$2} /^Threads:/{th=$2} END{printf "%s,%s,%s,%s",a,f,s,th}' "/proc/$PID/status" 2>/dev/null) || break
|
||||
# thread states: field 3 of /proc/PID/task/TID/stat, after the ")" closing comm
|
||||
states=$(cat /proc/$PID/task/*/stat 2>/dev/null | sed -E 's/^.*\) (.).*/\1/' | sort | uniq -c \
|
||||
| awk '{c[$2]=$1} END{printf "%d,%d,%d", c["R"]+0, c["D"]+0, c["S"]+0}')
|
||||
maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null || echo 0)
|
||||
io=$(awk '/^rchar:/{rc=$2} /^wchar:/{wc=$2} /^read_bytes:/{r=$2} /^write_bytes:/{w=$2} END{printf "%s,%s,%s,%s",r,w,rc,wc}' "/proc/$PID/io" 2>/dev/null || echo ",,,")
|
||||
mi=$(awk '/^Dirty:/{d=$2} /^Writeback:/{w=$2} /^Cached:/{c=$2} /^MemAvailable:/{m=$2} END{printf "%s,%s,%s,%s",d,w,c,m}' /proc/meminfo)
|
||||
echo "$t,$st,$states,$maps,$io,$mi" >>"$OUT/samples.csv"
|
||||
sleep "$DT"
|
||||
done
|
||||
|
||||
if [ "$MODE" = "--cmd" ]; then wait "$PID" || echo "command exit code: $?" >>"$OUT/env.txt"; fi
|
||||
|
||||
# ── summary ──────────────────────────────────────────────────────────────────
|
||||
awk -F, 'NR==1{next}
|
||||
{ n++; if(NR==2){t0=$1; r0=$10; w0=$11}
|
||||
if($2>ma)ma=$2; if($3>mf)mf=$3; if($4>ms)ms=$4; if($5>mt)mt=$5
|
||||
if($6>mR)mR=$6; if($7>mD)mD=$7; if($8>mS)mS=$8; sR+=$6; sD+=$7; sS+=$8
|
||||
if($14>md)md=$14; if($15>mw)mw=$15; if($9>mm)mm=$9
|
||||
t1=$1; r1=$10; w1=$11 }
|
||||
END{ if(!n)exit
|
||||
printf "samples: %d over %ds\n", n, t1-t0
|
||||
printf "peak RssAnon %.1f GB | RssFile %.1f GB | RssShmem %.1f GB | threads %d | maps %d\n", ma/1e6, mf/1e6, ms/1e6, mt, mm
|
||||
printf "threads by state (mean / max): R %.1f / %d | D(I/O wait) %.1f / %d | S %.1f / %d\n", sR/n, mR, sD/n, mD, sS/n, mS
|
||||
printf "peak Dirty %.1f GB | Writeback %.1f GB\n", md/1e6, mw/1e6
|
||||
printf "block I/O over the run: read %.1f GB, write %.1f GB\n", (r1-r0)/1e9, (w1-w0)/1e9 }' "$OUT/samples.csv" | tee "$OUT/summary.txt"
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# watch_merge.sh PID OUT.csv — samples a running process and kills it past a threshold.
|
||||
#
|
||||
# Every INTERVAL seconds appends one CSV row: number of memory mappings
|
||||
# (lines of /proc/PID/maps), VmSize, VmData, VmRSS, threads, and the
|
||||
# system-wide Committed_AS / CommitLimit. If the mapping count exceeds
|
||||
# MAPS_MAX, VmRSS exceeds RSS_MAX_GB, or VmSize exceeds VMSIZE_MAX_GB (stay
|
||||
# below `ulimit -v`: past it, allocations fail instead of the guard firing), the process receives SIGTERM (then
|
||||
# SIGKILL after 30 s) and the reason is written to OUT.csv.reason — a guard
|
||||
# so a runaway merge ends with a clean message instead of a failed
|
||||
# allocation (or the OOM killer on a shared node).
|
||||
#
|
||||
# Reading /proc/PID/maps is itself slow when the process has hundreds of
|
||||
# thousands of mappings (it held the kernel's mmap lock in earlier
|
||||
# measurements): keep INTERVAL >= 5. It never reads /proc/PID/smaps.
|
||||
#
|
||||
# Usage: scripts/watch_merge.sh PID OUT.csv
|
||||
# env: INTERVAL=5 MAPS_MAX=500000 RSS_MAX_GB=400 VMSIZE_MAX_GB=440
|
||||
#
|
||||
# Typical use (Linux only):
|
||||
# nohup scripts/watch_merge.sh "$(pgrep -f 'obikmer merge' | head -1)" /tmp/watch.csv &
|
||||
set -u
|
||||
PID=${1:?usage: watch_merge.sh PID OUT.csv}
|
||||
OUT=${2:?usage: watch_merge.sh PID OUT.csv}
|
||||
INTERVAL=${INTERVAL:-5}
|
||||
MAPS_MAX=${MAPS_MAX:-500000}
|
||||
RSS_MAX_KB=$(( ${RSS_MAX_GB:-400} * 1024 * 1024 ))
|
||||
VMSIZE_MAX_KB=$(( ${VMSIZE_MAX_GB:-440} * 1024 * 1024 ))
|
||||
|
||||
[ -r "/proc/$PID/maps" ] || { echo "cannot read /proc/$PID/maps" >&2; exit 1; }
|
||||
printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT"
|
||||
|
||||
while kill -0 "$PID" 2>/dev/null; do
|
||||
maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break
|
||||
st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" 2>/dev/null | awk '{printf "%s,", $2}')
|
||||
rss=$(grep '^VmRSS:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}')
|
||||
vsz=$(grep '^VmSize:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}')
|
||||
mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}')
|
||||
echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT"
|
||||
|
||||
reason=""
|
||||
[ "$maps" -gt "$MAPS_MAX" ] && reason="mappings $maps > MAPS_MAX $MAPS_MAX"
|
||||
[ -n "${rss:-}" ] && [ "$rss" -gt "$RSS_MAX_KB" ] && reason="VmRSS ${rss} kB > RSS_MAX_GB"
|
||||
[ -n "${vsz:-}" ] && [ "$vsz" -gt "$VMSIZE_MAX_KB" ] && reason="VmSize ${vsz} kB > VMSIZE_MAX_GB"
|
||||
if [ -n "$reason" ]; then
|
||||
echo "$(date +%T) GUARD: $reason — sending SIGTERM to $PID" | tee "$OUT.reason" >&2
|
||||
kill -TERM "$PID" 2>/dev/null
|
||||
for _ in $(seq 30); do kill -0 "$PID" 2>/dev/null || exit 0; sleep 1; done
|
||||
kill -KILL "$PID" 2>/dev/null
|
||||
exit 0
|
||||
fi
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
Reference in new issue
Block a user