feat: add pack optimization prompt, debug infrastructure, and monitoring scripts
Add a prompt template for optimizing the `pack` step and a suite of Docker-based debug tools including build, run, and monitoring scripts. Enhance `pack_matrices` with LFD partition sizing and detailed timing reports, and update `stop` with additional system statistics. - Add `prompts/pack_optimization.md` to guide optimization of the `pack` step, specifying context, performance measurements, and deliverable structures. - Add `scripts/docker_debug/` infrastructure: Dockerfile for musl targets, `build.sh` for image creation, `run.sh` for parallel indexing and merging, `monitor.sh` for PID tracking, `gen_fasta.py` for synthetic genomes, and `gen_fasta_real.py` for real genome generation with `read_fna`. - Add `scripts/monitor_pack.sh` as a 1 Hz sampler for I/O and memory metrics, writing samples to CSV and summary reports. - Add `scripts/watch_merge.sh` to monitor process resource usage, enforcing memory limits via SIGTERM/SIGKILL guards based on VmRSS and VmSize. - Modify `src/obikindex/src/index/kmer_index.rs`: add `pending_matrix_bytes` and `report_pack_timings`; update `pack_matrices` to use LFD strategy for partition sizing and clarify documentation regarding `stat()` proxy bytes. - Modify `src/obisys/src/stage.rs`: update `stop` definition to include new variables (`user_secs`, `sys_secs`, faults, context switches, I/O blocks) and logic checking for swap usage.
This commit is contained in:
1 parent
24291a4484
commit
03eaa4544a
11 files changed
+497
-2
No files matched your search
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# watch_merge.sh PID OUT.csv — samples a running process and kills it past a threshold.
|
||||
#
|
||||
# Every INTERVAL seconds appends one CSV row: number of memory mappings
|
||||
# (lines of /proc/PID/maps), VmSize, VmData, VmRSS, threads, and the
|
||||
# system-wide Committed_AS / CommitLimit. If the mapping count exceeds
|
||||
# MAPS_MAX, VmRSS exceeds RSS_MAX_GB, or VmSize exceeds VMSIZE_MAX_GB (stay
|
||||
# below `ulimit -v`: past it, allocations fail instead of the guard firing), the process receives SIGTERM (then
|
||||
# SIGKILL after 30 s) and the reason is written to OUT.csv.reason — a guard
|
||||
# so a runaway merge ends with a clean message instead of a failed
|
||||
# allocation (or the OOM killer on a shared node).
|
||||
#
|
||||
# Reading /proc/PID/maps is itself slow when the process has hundreds of
|
||||
# thousands of mappings (it held the kernel's mmap lock in earlier
|
||||
# measurements): keep INTERVAL >= 5. It never reads /proc/PID/smaps.
|
||||
#
|
||||
# Usage: scripts/watch_merge.sh PID OUT.csv
|
||||
# env: INTERVAL=5 MAPS_MAX=500000 RSS_MAX_GB=400 VMSIZE_MAX_GB=440
|
||||
#
|
||||
# Typical use (Linux only):
|
||||
# nohup scripts/watch_merge.sh "$(pgrep -f 'obikmer merge' | head -1)" /tmp/watch.csv &
|
||||
set -u
|
||||
PID=${1:?usage: watch_merge.sh PID OUT.csv}
|
||||
OUT=${2:?usage: watch_merge.sh PID OUT.csv}
|
||||
INTERVAL=${INTERVAL:-5}
|
||||
MAPS_MAX=${MAPS_MAX:-500000}
|
||||
RSS_MAX_KB=$(( ${RSS_MAX_GB:-400} * 1024 * 1024 ))
|
||||
VMSIZE_MAX_KB=$(( ${VMSIZE_MAX_GB:-440} * 1024 * 1024 ))
|
||||
|
||||
[ -r "/proc/$PID/maps" ] || { echo "cannot read /proc/$PID/maps" >&2; exit 1; }
|
||||
printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT"
|
||||
|
||||
while kill -0 "$PID" 2>/dev/null; do
|
||||
maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break
|
||||
st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" 2>/dev/null | awk '{printf "%s,", $2}')
|
||||
rss=$(grep '^VmRSS:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}')
|
||||
vsz=$(grep '^VmSize:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}')
|
||||
mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}')
|
||||
echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT"
|
||||
|
||||
reason=""
|
||||
[ "$maps" -gt "$MAPS_MAX" ] && reason="mappings $maps > MAPS_MAX $MAPS_MAX"
|
||||
[ -n "${rss:-}" ] && [ "$rss" -gt "$RSS_MAX_KB" ] && reason="VmRSS ${rss} kB > RSS_MAX_GB"
|
||||
[ -n "${vsz:-}" ] && [ "$vsz" -gt "$VMSIZE_MAX_KB" ] && reason="VmSize ${vsz} kB > VMSIZE_MAX_GB"
|
||||
if [ -n "$reason" ]; then
|
||||
echo "$(date +%T) GUARD: $reason — sending SIGTERM to $PID" | tee "$OUT.reason" >&2
|
||||
kill -TERM "$PID" 2>/dev/null
|
||||
for _ in $(seq 30); do kill -0 "$PID" 2>/dev/null || exit 0; sleep 1; done
|
||||
kill -KILL "$PID" 2>/dev/null
|
||||
exit 0
|
||||
fi
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
Reference in new issue
Block a user