feat: add pack optimization prompt, debug infrastructure, and monitoring scripts

Add a prompt template for optimizing the `pack` step and a suite of Docker-based
debug tools including build, run, and monitoring scripts. Enhance `pack_matrices`
with LFD partition sizing and detailed timing reports, and update `stop` with
additional system statistics.

- Add `prompts/pack_optimization.md` to guide optimization of the `pack` step,
  specifying context, performance measurements, and deliverable structures.
- Add `scripts/docker_debug/` infrastructure: Dockerfile for musl targets,
  `build.sh` for image creation, `run.sh` for parallel indexing and merging,
  `monitor.sh` for PID tracking, `gen_fasta.py` for synthetic genomes, and
  `gen_fasta_real.py` for real genome generation with `read_fna`.
- Add `scripts/monitor_pack.sh` as a 1 Hz sampler for I/O and memory metrics,
  writing samples to CSV and summary reports.
- Add `scripts/watch_merge.sh` to monitor process resource usage, enforcing
  memory limits via SIGTERM/SIGKILL guards based on VmRSS and VmSize.
- Modify `src/obikindex/src/index/kmer_index.rs`: add `pending_matrix_bytes` and
  `report_pack_timings`; update `pack_matrices` to use LFD strategy for partition
  sizing and clarify documentation regarding `stat()` proxy bytes.
- Modify `src/obisys/src/stage.rs`: update `stop` definition to include new
  variables (`user_secs`, `sys_secs`, faults, context switches, I/O blocks) and
  logic checking for swap usage.
This commit is contained in:
Eric Coissac committed 2026-11-19 07:07:50 +01:00
1 parent 24291a4484
commit 03eaa4544a
11 files changed
+497 -2

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
# Linux debug environment for the merge mapping/memory-growth hunt.
# Same toolchain as .gitea/workflows/release.yml for the musl target
# (cargo-zigbuild + zig cc), plus a glibc toolchain and diagnostic tools.
FROM rust:1-bookworm
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake pkg-config autoconf automake libtool \
heaptrack valgrind gdb strace procps python3 python3-pip \
&& rm -rf /var/lib/apt/lists/*
RUN pip install ziglang --quiet --break-system-packages \
&& cargo install cargo-zigbuild \
&& rustup target add x86_64-unknown-linux-musl
RUN ZIG=$(python3 -c "import ziglang, os; print(os.path.join(os.path.dirname(ziglang.__file__), 'zig'))") \
&& printf '#!/bin/sh\nexec "%s" cc -target x86_64-linux-musl "$@"\n' "$ZIG" > /usr/local/bin/x86_64-linux-musl-gcc \
&& printf '#!/bin/sh\nexec "%s" c++ -target x86_64-linux-musl "$@"\n' "$ZIG" > /usr/local/bin/x86_64-linux-musl-g++ \
&& chmod +x /usr/local/bin/x86_64-linux-musl-gcc /usr/local/bin/x86_64-linux-musl-g++
WORKDIR /work
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
# Builds the debug image. PLATFORM defaults to linux/amd64 (Rosetta on Apple Silicon).
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
PLATFORM="${PLATFORM:-linux/amd64}"
docker build --platform "$PLATFORM" -t "obikmer-debug:${PLATFORM//\//-}" .
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""gen_fasta.py N OUTDIR [GENOME_LEN] — N deterministic synthetic genomes.
Each genome = unique random sequence + a random subset of a shared fragment
pool (some fragments lightly mutated), so a merge both hits k-mers already
in the destination and creates new ones, like related genomes would.
"""
import random, sys
n, out = int(sys.argv[1]), sys.argv[2]
glen = int(sys.argv[3]) if len(sys.argv) > 3 else 150_000
seed = int(sys.argv[4]) if len(sys.argv) > 4 else 20261118
rng = random.Random(seed)
def rnd(m):
return "".join(rng.choices("ACGT", k=m))
def mutate(s, rate=0.01):
s = list(s)
for i in range(len(s)):
if rng.random() < rate:
s[i] = rng.choice("ACGT")
return "".join(s)
pool = [rnd(2000) for _ in range(300)] # 600 kb shared pool
for g in range(n):
parts = [rnd(glen)]
for f in rng.sample(range(len(pool)), 60):
parts.append(mutate(pool[f]) if rng.random() < 0.3 else pool[f])
rng.shuffle(parts)
with open(f"{out}/G{g:03d}.fasta", "w") as fh:
for j, p in enumerate(parts):
fh.write(f">G{g:03d}_{j}\n{p}\n")
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""gen_fasta_real.py N OUTDIR GENOMES_DIR [WINDOW] [SEED] [MUT]
N derived genomes built from *real* genomes (GENOMES_DIR/*.fna.gz): each one is
a random window of WINDOW bp of a randomly chosen real genome, point-mutated at
rate MUT (default 3 %, which alters ~47 % of 21-mers), so a merge both finds
k-mers already present in the destination and creates new ones, with realistic
sequence statistics (no dedicated degenerate structure).
"""
import glob, gzip, random, sys
n, out, gdir = int(sys.argv[1]), sys.argv[2], sys.argv[3]
window = int(sys.argv[4]) if len(sys.argv) > 4 else 300_000
seed = int(sys.argv[5]) if len(sys.argv) > 5 else 20261118
mut = float(sys.argv[6]) if len(sys.argv) > 6 else 0.03
rng = random.Random(seed)
def read_fna(path):
seq = []
with gzip.open(path, "rt") as fh:
for line in fh:
if not line.startswith(">"):
seq.append(line.strip().upper())
return "".join(seq)
genomes = [read_fna(p) for p in sorted(glob.glob(f"{gdir}/*.fna.gz"))]
genomes = [g for g in genomes if len(g) > window]
for i in range(n):
g = rng.choice(genomes)
start = rng.randrange(0, len(g) - window)
s = list(g[start:start + window])
for j in range(len(s)):
if s[j] in "ACGT" and rng.random() < mut:
s[j] = rng.choice("ACGT")
seq = "".join(c if c in "ACGT" else "A" for c in s)
with open(f"{out}/R{i:03d}.fasta", "w") as fh:
fh.write(f">R{i:03d}\n")
for k in range(0, len(seq), 80):
fh.write(seq[k:k + 80] + "\n")
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# monitor.sh PID OUT [INTERVAL_S] — samples mappings / VM / commit figures until PID exits.
PID=$1; OUT=$2; DT=${3:-5}
printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT"
while kill -0 "$PID" 2>/dev/null; do
maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break
st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" | awk '{printf "%s,", $2}')
mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}')
echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT"
sleep "$DT"
done
+92
View File
@@ -0,0 +1,92 @@
#!/usr/bin/env bash
# run.sh — builds obikmer (release, debug symbols) and merges the benchmark presence
# indexes in the container while sampling mappings/RSS/commit in parallel.
#
# Env:
# PLATFORM linux/amd64 (default, Rosetta) | linux/arm64
# GEN if set: build GEN synthetic indexes (gen_fasta.py) instead of using
# the benchmark ones; GENOME_LEN (default 150000), K (21), M (9), PART (256)
# REAL if set (with GEN): derive genomes from benchmark/run/genomes (gen_fasta_real.py)
# SRC_DIR alternative source tree (default: <repo>/src); VARIANT: volume suffix;
# LOCKED=0 to allow Cargo.lock changes (needed when a dep is added)
# NSRC number of source indexes to merge (default: all 20)
# TARGET gnu (default, glibc) | musl (static, same chain as the release CI)
# TOOL none (default) | heaptrack (heaptrack output goes to OUT/)
# MAXMAP if set, vm.max_map_count is lowered to this inside the VM for the
# run and restored afterwards (needs --privileged; affects the whole
# OrbStack VM while it runs)
# MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_, MALLOC_TRIM_THRESHOLD_ passed through
# OUT host output directory (default: ./out/<timestamp>)
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
PLATFORM="${PLATFORM:-linux/amd64}"
NSRC="${NSRC:-20}"
TOOL="${TOOL:-none}"
TARGET="${TARGET:-gnu}"
OUT="${OUT:-$HERE/out/$(date +%Y%m%d-%H%M%S)}"
mkdir -p "$OUT"
IMG="obikmer-debug:${PLATFORM//\//-}"
VOL="obikmer-debug-target-${PLATFORM//\//-}-${TARGET}${VARIANT:+-$VARIANT}"
docker run --rm --privileged --platform "$PLATFORM" \
-v "${SRC_DIR:-$REPO/src}:/work/src:$([ -n "${SRC_DIR:-}" ] && echo rw || echo ro)" \
-v "$REPO/benchmark/run/specimen_index_presence:/idx_ro:ro" \
-v "$VOL:/target" \
-v "$OUT:/out" \
-v "$HERE/monitor.sh:/monitor.sh:ro" \
-v "$HERE/gen_fasta.py:/gen_fasta.py:ro" \
-v "$HERE/gen_fasta_real.py:/gen_fasta_real.py:ro" \
-v "$REPO/benchmark/run/genomes:/genomes_ro:ro" \
-e CARGO_TARGET_DIR=/target -e CARGO_PROFILE_RELEASE_DEBUG=1 \
-e GEN="${GEN:-}" -e LOCKED_FLAG="$([ "${LOCKED:-1}" = 0 ] || echo --locked)" -e SEED="${SEED:-20261118}" -e REAL="${REAL:-}" -e GENOME_LEN="${GENOME_LEN:-150000}" -e K="${K:-21}" -e M="${M:-9}" -e PART="${PART:-256}" -e RUST_LOG="${RUST_LOG:-}" -e NSRC="$NSRC" -e TARGET="$TARGET" -e TOOL="$TOOL" -e MAXMAP="${MAXMAP:-}" \
-e MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-}" \
-e CXXFLAGS="${CXXFLAGS:-}" \
-e MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-}" \
-e MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:-}" \
"$IMG" bash -c '
set -euo pipefail
for v in MALLOC_ARENA_MAX MALLOC_MMAP_THRESHOLD_ MALLOC_TRIM_THRESHOLD_ RUST_LOG; do
[ -n "${!v}" ] || unset $v
done
if [ "$TARGET" = musl ]; then
PKG_CONFIG_ALLOW_CROSS=1 cargo zigbuild --release $LOCKED_FLAG --target x86_64-unknown-linux-musl \
--manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
BIN=/target/x86_64-unknown-linux-musl/release/obikmer
else
cargo build --release $LOCKED_FLAG --manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
BIN=/target/release/obikmer
fi
file "$BIN" 2>/dev/null | cut -c1-150 || true
# merge writes into its sources (lock files…): work on a copy, never on benchmark/
rm -rf /tmp/idx && mkdir -p /tmp/idx
if [ -n "$GEN" ]; then
mkdir -p /tmp/fa
if [ -n "$REAL" ]; then python3 /gen_fasta_real.py "$GEN" /tmp/fa /genomes_ro "$GENOME_LEN" "$SEED"
else python3 /gen_fasta.py "$GEN" /tmp/fa "$GENOME_LEN" "$SEED"; fi
NSRC=$GEN
echo "indexing $GEN synthetic genomes (k=$K m=$M p=$PART)…"
ls /tmp/fa/*.fasta | xargs -P 8 -I{} sh -c \
"b=\$(basename {} .fasta); $BIN index -k $K -m $M --theta 0 -p $PART --label \$b -o /tmp/idx/\$b.idx {} >/dev/null 2>&1 || echo index failed: \$b"
else
cp -r /idx_ro/. /tmp/idx/
fi
mapfile -t SRC < <(find /tmp/idx -mindepth 1 -maxdepth 1 -type d | sort | head -n "$NSRC")
echo "sources: ${#SRC[@]} target: $TARGET tool: $TOOL max_map_count: $(cat /proc/sys/vm/max_map_count)"
if [ -n "$MAXMAP" ]; then
OLD=$(cat /proc/sys/vm/max_map_count); sysctl -w vm.max_map_count="$MAXMAP" >/dev/null
trap "sysctl -w vm.max_map_count=$OLD >/dev/null" EXIT
fi
rm -rf /tmp/merged
CMD=("$BIN" merge --output /tmp/merged --force --force-presence "${SRC[@]}")
if [ "$TOOL" = heaptrack ]; then CMD=(heaptrack -o /out/heaptrack "${CMD[@]}"); fi
"${CMD[@]}" 2>/out/merge.stderr &
PID=$!
/monitor.sh "$PID" /out/monitor.csv 1 &
MON=$!
set +e; wait "$PID"; RC=$?; set -e
wait "$MON" || true
echo "merge exit code: $RC" | tee /out/exit_code.txt
tail -5 /out/merge.stderr
'
echo "outputs: $OUT"