From 03eaa4544af08848e54e41463e4d33129d5419fe Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Wed, 18 Nov 2026 08:01:32 +0100 Subject: [PATCH] feat: add pack optimization prompt, debug infrastructure, and monitoring scripts Add a prompt template for optimizing the `pack` step and a suite of Docker-based debug tools including build, run, and monitoring scripts. Enhance `pack_matrices` with LFD partition sizing and detailed timing reports, and update `stop` with additional system statistics. - Add `prompts/pack_optimization.md` to guide optimization of the `pack` step, specifying context, performance measurements, and deliverable structures. - Add `scripts/docker_debug/` infrastructure: Dockerfile for musl targets, `build.sh` for image creation, `run.sh` for parallel indexing and merging, `monitor.sh` for PID tracking, `gen_fasta.py` for synthetic genomes, and `gen_fasta_real.py` for real genome generation with `read_fna`. - Add `scripts/monitor_pack.sh` as a 1 Hz sampler for I/O and memory metrics, writing samples to CSV and summary reports. - Add `scripts/watch_merge.sh` to monitor process resource usage, enforcing memory limits via SIGTERM/SIGKILL guards based on VmRSS and VmSize. - Modify `src/obikindex/src/index/kmer_index.rs`: add `pending_matrix_bytes` and `report_pack_timings`; update `pack_matrices` to use LFD strategy for partition sizing and clarify documentation regarding `stat()` proxy bytes. - Modify `src/obisys/src/stage.rs`: update `stop` definition to include new variables (`user_secs`, `sys_secs`, faults, context switches, I/O blocks) and logic checking for swap usage. --- prompts/pack_optimization.md | 61 +++++++++++++++ scripts/docker_debug/Dockerfile | 16 ++++ scripts/docker_debug/build.sh | 6 ++ scripts/docker_debug/gen_fasta.py | 33 ++++++++ scripts/docker_debug/gen_fasta_real.py | 39 ++++++++++ scripts/docker_debug/monitor.sh | 11 +++ scripts/docker_debug/run.sh | 92 ++++++++++++++++++++++ scripts/monitor_pack.sh | 102 +++++++++++++++++++++++++ scripts/watch_merge.sh | 53 +++++++++++++ src/obikindex/src/index/kmer_index.rs | 78 ++++++++++++++++++- src/obisys/src/stage.rs | 8 ++ 11 files changed, 497 insertions(+), 2 deletions(-) create mode 100644 prompts/pack_optimization.md create mode 100644 scripts/docker_debug/Dockerfile create mode 100755 scripts/docker_debug/build.sh create mode 100644 scripts/docker_debug/gen_fasta.py create mode 100644 scripts/docker_debug/gen_fasta_real.py create mode 100755 scripts/docker_debug/monitor.sh create mode 100755 scripts/docker_debug/run.sh create mode 100755 scripts/monitor_pack.sh create mode 100755 scripts/watch_merge.sh diff --git a/prompts/pack_optimization.md b/prompts/pack_optimization.md new file mode 100644 index 00000000..4942eff5 --- /dev/null +++ b/prompts/pack_optimization.md @@ -0,0 +1,61 @@ +# Prompt — où optimiser l'étape `pack` du merge (diagnostic d'abord) + +> À coller tel quel au début d'une nouvelle session Claude Code ouverte à la racine du dépôt obikmer. +> Cette session est un **diagnostic** : elle produit une analyse chiffrée et des pistes classées, pas du code. Toute modification du code attend mon feu vert explicite, piste par piste. + +--- + +## Contexte + +`obikmer merge` se termine par une étape `finalize`, dont l'essentiel est `pack` : les matrices de présence/comptage, écrites en colonnes pendant le merge (`MatrixBuilder`), sont transposées en un format compact par partition (sparse par défaut). Le point d'entrée est `KmerIndex::finalize_indexed` (`src/obikindex/src/index/builder.rs`, étape « pack » vers la ligne 109), qui appelle `KmerIndex::pack_matrices(sparse)` (`src/obikindex/src/index/kmer_index.rs`). Les primitives sont dans `src/obicompactvec/src/` : `pack_bit_matrix` (`bitmatrix/packed.rs`), `pack_compact_int_matrix` (`intmatrix.rs`), `pack_sparse_compact_int_matrix` (`sparse_intmatrix.rs`), et le côté sparse des bits dans `bitmatrix/`. + +Lis d'abord `DevDocMD/implementation/merge_scaling.md` (état du pack sparse : passage par une matrice dense séquentielle puis sparse, un seul mapping par partition) et `DevDocMD/implementation/merge_mmap_audit.md`. + +## Mesure de référence (production, 2026-11-18) + +Merge de 186 sources (`Cedrus`), 256 partitions, binaire glibc compilé localement, **192 CPU non contraints**, 17,5 milliards de k-mers nouveaux au total. Mode **présence**, compaction par défaut, donc **sparse** (pas de `--dense`) : + +| Étape | Durée | Efficacité CPU | RSS max | +|---|---|---|---| +| `merge_partitions` | 748 s | 141×/192 (73 %) | 354 Go | +| `pack` | 231 s | 65×/192 (34 %) | **503,7 Go** | + +La RAM de la machine est de 754 Go. Le RSS max est celui du process : il inclut les pages de fichiers mappés, pas seulement la mémoire anonyme. **Je n'ai pas encore séparé les deux.** + +Progression de `pack` (partitions terminées par tranche de 10 %, début à 04:08:09) : +`+53 s` (10 %) · `+35 s` (20 %) · `+13 s` (30 %) · `+48 s` (40 %) · `+24 s` (50 %) · `+9 s` · `+6 s` · `+6 s` · `+6 s` · `+11 s` (100 %), puis `+20 s` avant la fin de l'étape. Les partitions les plus grosses passent en premier (tri décroissant dans `merge.rs`), mais je ne sais pas si `pack` les traite dans le même ordre ni si l'ordre est le bon. + +## Ce que je veux savoir + +1. **Où part le temps** de `pack` ? Lecture des colonnes, transposition, écriture, `sync`/`fsync`, attente d'un verrou, déséquilibre entre partitions ? Efficacité de 34 % sur 192 workers : qu'est-ce qui les bloque ? +2. **Où part la mémoire** (504 Go) ? Combien est de la mémoire anonyme, combien des pages de fichiers mappés ? Est-elle proportionnelle au nombre de workers ? Quelle est la mémoire vivante par partition en cours de `pack` ? +3. **Quel est le parallélisme réel** : combien de partitions `pack` traite-t-il en même temps, et qu'est-ce qui le fixe (`PartitionRunner`, `--cpu-max`, NUMA) ? +4. Y a-t-il du **travail inutile** (relectures, copies intermédiaires, colonnes vides ou toutes-absentes ré-écrites, étape dense qui pourrait être évitée) ? + +## Méthode (impérative) + +- **Lire et mesurer avant de proposer.** Pas de récit plausible sans preuve : tout ce qui n'est pas vérifié par lecture du code ou par mesure est marqué « hypothèse ». +- Pour mesurer sans la production : `scripts/docker_debug/` fournit un conteneur Linux (build glibc ou musl) qui lance un merge de N sources synthétiques ou dérivées de génomes réels (`run.sh`, variables `GEN`, `REAL`, `GENOME_LEN`, `TARGET`, `TOOL=heaptrack` pour glibc) avec un moniteur CSV de mappings et de RSS. Les génomes de 2 Mb donnent des partitions assez grosses pour que `pack` ait quelque chose à faire. `heaptrack` fonctionne sur le build glibc (pas sur le build musl statique). `gdb` ne fonctionne pas sous Rosetta. +- Pour séparer mémoire anonyme et fichiers mappés, utilise `/proc/PID/status` (`RssAnon`, `RssFile`, `RssShmem`), **jamais `smaps` en entier** sur un process à beaucoup de mappings (il verrouille `mmap_lock` et ralentit le process). +- Les estimations de coût (octets, secondes) sont marquées **estimées** ; les chiffres observés sont marqués **mesurés**. + +## Règles du projet (non négociables) + +- Avant tout travail de code : `mcp__serena__initial_instructions`, puis Serena (`find_symbol`, `get_symbols_overview`, `find_referencing_symbols`) pour naviguer. `rust-analyzer-mcp` et `qmd` quand ils servent. Consulte `context7` avant d'utiliser une API de bibliothèque externe. +- **Aucune commande VCS mutante** (git/jj : commit, add, mv, rm, describe, push…). Lecture seule autorisée. +- **Aucune nouvelle dépendance, aucun changement de version** sans mon accord explicite. Si un problème de bibliothèque se pose, expose-le et propose des alternatives. +- **Une question appelle une réponse, pas une action.** Ne modifie aucun fichier de `src/` sans demande explicite. Cette session est en lecture seule sur le code. +- Le code existant est une hypothèse, pas une vérité : signale tout ce qui est devenu injustifié plutôt que de le garder par inertie. +- Si tu proposes d'ajouter une opération à une structure partagée, mets-la comme méthode générique sur cette structure (crate `obicompactvec`), pas comme fonction libre dans le consommateur. +- Ne jamais matérialiser en mémoire une structure volumineuse qui peut être streamée. +- Échanges en **français**, documentation en **anglais**. + +## Livrable + +Un rapport (dans la conversation, pas dans un fichier, sauf si je le demande) avec : +1. Le chemin de code de `pack`, de `finalize_indexed` jusqu'aux écritures disque, résumé en quelques lignes par étape. +2. Pour chaque question ci-dessus : ce qui est **mesuré**, ce qui est **hypothèse**, et comment vérifier l'hypothèse. +3. Une liste de **pistes d'optimisation classées** par gain attendu/coût/risque. Pour chaque piste : ce qu'elle changerait, ce qu'elle risque, et comment on verrait qu'elle marche. Aucune implémentation. +4. Les vérifications de non-régression à prévoir : le résultat sur disque doit rester **identique octet pour octet** (voir les oracles `scripts/smoke_test_merge.sh` et `scripts/bench_merge_oracle.sh`). + +Termine par la question : « Quelle piste veux-tu que j'implémente ? » et attends ma réponse. diff --git a/scripts/docker_debug/Dockerfile b/scripts/docker_debug/Dockerfile new file mode 100644 index 00000000..21e179fe --- /dev/null +++ b/scripts/docker_debug/Dockerfile @@ -0,0 +1,16 @@ +# Linux debug environment for the merge mapping/memory-growth hunt. +# Same toolchain as .gitea/workflows/release.yml for the musl target +# (cargo-zigbuild + zig cc), plus a glibc toolchain and diagnostic tools. +FROM rust:1-bookworm +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake pkg-config autoconf automake libtool \ + heaptrack valgrind gdb strace procps python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* +RUN pip install ziglang --quiet --break-system-packages \ + && cargo install cargo-zigbuild \ + && rustup target add x86_64-unknown-linux-musl +RUN ZIG=$(python3 -c "import ziglang, os; print(os.path.join(os.path.dirname(ziglang.__file__), 'zig'))") \ + && printf '#!/bin/sh\nexec "%s" cc -target x86_64-linux-musl "$@"\n' "$ZIG" > /usr/local/bin/x86_64-linux-musl-gcc \ + && printf '#!/bin/sh\nexec "%s" c++ -target x86_64-linux-musl "$@"\n' "$ZIG" > /usr/local/bin/x86_64-linux-musl-g++ \ + && chmod +x /usr/local/bin/x86_64-linux-musl-gcc /usr/local/bin/x86_64-linux-musl-g++ +WORKDIR /work diff --git a/scripts/docker_debug/build.sh b/scripts/docker_debug/build.sh new file mode 100755 index 00000000..9ca819b7 --- /dev/null +++ b/scripts/docker_debug/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Builds the debug image. PLATFORM defaults to linux/amd64 (Rosetta on Apple Silicon). +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" +PLATFORM="${PLATFORM:-linux/amd64}" +docker build --platform "$PLATFORM" -t "obikmer-debug:${PLATFORM//\//-}" . diff --git a/scripts/docker_debug/gen_fasta.py b/scripts/docker_debug/gen_fasta.py new file mode 100644 index 00000000..1c8fea5d --- /dev/null +++ b/scripts/docker_debug/gen_fasta.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""gen_fasta.py N OUTDIR [GENOME_LEN] — N deterministic synthetic genomes. + +Each genome = unique random sequence + a random subset of a shared fragment +pool (some fragments lightly mutated), so a merge both hits k-mers already +in the destination and creates new ones, like related genomes would. +""" +import random, sys + +n, out = int(sys.argv[1]), sys.argv[2] +glen = int(sys.argv[3]) if len(sys.argv) > 3 else 150_000 +seed = int(sys.argv[4]) if len(sys.argv) > 4 else 20261118 +rng = random.Random(seed) + +def rnd(m): + return "".join(rng.choices("ACGT", k=m)) + +def mutate(s, rate=0.01): + s = list(s) + for i in range(len(s)): + if rng.random() < rate: + s[i] = rng.choice("ACGT") + return "".join(s) + +pool = [rnd(2000) for _ in range(300)] # 600 kb shared pool +for g in range(n): + parts = [rnd(glen)] + for f in rng.sample(range(len(pool)), 60): + parts.append(mutate(pool[f]) if rng.random() < 0.3 else pool[f]) + rng.shuffle(parts) + with open(f"{out}/G{g:03d}.fasta", "w") as fh: + for j, p in enumerate(parts): + fh.write(f">G{g:03d}_{j}\n{p}\n") diff --git a/scripts/docker_debug/gen_fasta_real.py b/scripts/docker_debug/gen_fasta_real.py new file mode 100644 index 00000000..0838b818 --- /dev/null +++ b/scripts/docker_debug/gen_fasta_real.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""gen_fasta_real.py N OUTDIR GENOMES_DIR [WINDOW] [SEED] [MUT] + +N derived genomes built from *real* genomes (GENOMES_DIR/*.fna.gz): each one is +a random window of WINDOW bp of a randomly chosen real genome, point-mutated at +rate MUT (default 3 %, which alters ~47 % of 21-mers), so a merge both finds +k-mers already present in the destination and creates new ones, with realistic +sequence statistics (no dedicated degenerate structure). +""" +import glob, gzip, random, sys + +n, out, gdir = int(sys.argv[1]), sys.argv[2], sys.argv[3] +window = int(sys.argv[4]) if len(sys.argv) > 4 else 300_000 +seed = int(sys.argv[5]) if len(sys.argv) > 5 else 20261118 +mut = float(sys.argv[6]) if len(sys.argv) > 6 else 0.03 +rng = random.Random(seed) + +def read_fna(path): + seq = [] + with gzip.open(path, "rt") as fh: + for line in fh: + if not line.startswith(">"): + seq.append(line.strip().upper()) + return "".join(seq) + +genomes = [read_fna(p) for p in sorted(glob.glob(f"{gdir}/*.fna.gz"))] +genomes = [g for g in genomes if len(g) > window] +for i in range(n): + g = rng.choice(genomes) + start = rng.randrange(0, len(g) - window) + s = list(g[start:start + window]) + for j in range(len(s)): + if s[j] in "ACGT" and rng.random() < mut: + s[j] = rng.choice("ACGT") + seq = "".join(c if c in "ACGT" else "A" for c in s) + with open(f"{out}/R{i:03d}.fasta", "w") as fh: + fh.write(f">R{i:03d}\n") + for k in range(0, len(seq), 80): + fh.write(seq[k:k + 80] + "\n") diff --git a/scripts/docker_debug/monitor.sh b/scripts/docker_debug/monitor.sh new file mode 100755 index 00000000..8b660eae --- /dev/null +++ b/scripts/docker_debug/monitor.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# monitor.sh PID OUT [INTERVAL_S] — samples mappings / VM / commit figures until PID exits. +PID=$1; OUT=$2; DT=${3:-5} +printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT" +while kill -0 "$PID" 2>/dev/null; do + maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break + st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" | awk '{printf "%s,", $2}') + mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}') + echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT" + sleep "$DT" +done diff --git a/scripts/docker_debug/run.sh b/scripts/docker_debug/run.sh new file mode 100755 index 00000000..22fa37ea --- /dev/null +++ b/scripts/docker_debug/run.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# run.sh — builds obikmer (release, debug symbols) and merges the benchmark presence +# indexes in the container while sampling mappings/RSS/commit in parallel. +# +# Env: +# PLATFORM linux/amd64 (default, Rosetta) | linux/arm64 +# GEN if set: build GEN synthetic indexes (gen_fasta.py) instead of using +# the benchmark ones; GENOME_LEN (default 150000), K (21), M (9), PART (256) +# REAL if set (with GEN): derive genomes from benchmark/run/genomes (gen_fasta_real.py) +# SRC_DIR alternative source tree (default: /src); VARIANT: volume suffix; +# LOCKED=0 to allow Cargo.lock changes (needed when a dep is added) +# NSRC number of source indexes to merge (default: all 20) +# TARGET gnu (default, glibc) | musl (static, same chain as the release CI) +# TOOL none (default) | heaptrack (heaptrack output goes to OUT/) +# MAXMAP if set, vm.max_map_count is lowered to this inside the VM for the +# run and restored afterwards (needs --privileged; affects the whole +# OrbStack VM while it runs) +# MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_, MALLOC_TRIM_THRESHOLD_ passed through +# OUT host output directory (default: ./out/) +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +PLATFORM="${PLATFORM:-linux/amd64}" +NSRC="${NSRC:-20}" +TOOL="${TOOL:-none}" +TARGET="${TARGET:-gnu}" +OUT="${OUT:-$HERE/out/$(date +%Y%m%d-%H%M%S)}" +mkdir -p "$OUT" +IMG="obikmer-debug:${PLATFORM//\//-}" +VOL="obikmer-debug-target-${PLATFORM//\//-}-${TARGET}${VARIANT:+-$VARIANT}" + +docker run --rm --privileged --platform "$PLATFORM" \ + -v "${SRC_DIR:-$REPO/src}:/work/src:$([ -n "${SRC_DIR:-}" ] && echo rw || echo ro)" \ + -v "$REPO/benchmark/run/specimen_index_presence:/idx_ro:ro" \ + -v "$VOL:/target" \ + -v "$OUT:/out" \ + -v "$HERE/monitor.sh:/monitor.sh:ro" \ + -v "$HERE/gen_fasta.py:/gen_fasta.py:ro" \ + -v "$HERE/gen_fasta_real.py:/gen_fasta_real.py:ro" \ + -v "$REPO/benchmark/run/genomes:/genomes_ro:ro" \ + -e CARGO_TARGET_DIR=/target -e CARGO_PROFILE_RELEASE_DEBUG=1 \ + -e GEN="${GEN:-}" -e LOCKED_FLAG="$([ "${LOCKED:-1}" = 0 ] || echo --locked)" -e SEED="${SEED:-20261118}" -e REAL="${REAL:-}" -e GENOME_LEN="${GENOME_LEN:-150000}" -e K="${K:-21}" -e M="${M:-9}" -e PART="${PART:-256}" -e RUST_LOG="${RUST_LOG:-}" -e NSRC="$NSRC" -e TARGET="$TARGET" -e TOOL="$TOOL" -e MAXMAP="${MAXMAP:-}" \ + -e MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-}" \ + -e CXXFLAGS="${CXXFLAGS:-}" \ + -e MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-}" \ + -e MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:-}" \ + "$IMG" bash -c ' +set -euo pipefail +for v in MALLOC_ARENA_MAX MALLOC_MMAP_THRESHOLD_ MALLOC_TRIM_THRESHOLD_ RUST_LOG; do + [ -n "${!v}" ] || unset $v +done +if [ "$TARGET" = musl ]; then + PKG_CONFIG_ALLOW_CROSS=1 cargo zigbuild --release $LOCKED_FLAG --target x86_64-unknown-linux-musl \ + --manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3 + BIN=/target/x86_64-unknown-linux-musl/release/obikmer +else + cargo build --release $LOCKED_FLAG --manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3 + BIN=/target/release/obikmer +fi +file "$BIN" 2>/dev/null | cut -c1-150 || true +# merge writes into its sources (lock files…): work on a copy, never on benchmark/ +rm -rf /tmp/idx && mkdir -p /tmp/idx +if [ -n "$GEN" ]; then + mkdir -p /tmp/fa + if [ -n "$REAL" ]; then python3 /gen_fasta_real.py "$GEN" /tmp/fa /genomes_ro "$GENOME_LEN" "$SEED" + else python3 /gen_fasta.py "$GEN" /tmp/fa "$GENOME_LEN" "$SEED"; fi + NSRC=$GEN + echo "indexing $GEN synthetic genomes (k=$K m=$M p=$PART)…" + ls /tmp/fa/*.fasta | xargs -P 8 -I{} sh -c \ + "b=\$(basename {} .fasta); $BIN index -k $K -m $M --theta 0 -p $PART --label \$b -o /tmp/idx/\$b.idx {} >/dev/null 2>&1 || echo index failed: \$b" +else + cp -r /idx_ro/. /tmp/idx/ +fi +mapfile -t SRC < <(find /tmp/idx -mindepth 1 -maxdepth 1 -type d | sort | head -n "$NSRC") +echo "sources: ${#SRC[@]} target: $TARGET tool: $TOOL max_map_count: $(cat /proc/sys/vm/max_map_count)" +if [ -n "$MAXMAP" ]; then + OLD=$(cat /proc/sys/vm/max_map_count); sysctl -w vm.max_map_count="$MAXMAP" >/dev/null + trap "sysctl -w vm.max_map_count=$OLD >/dev/null" EXIT +fi +rm -rf /tmp/merged +CMD=("$BIN" merge --output /tmp/merged --force --force-presence "${SRC[@]}") +if [ "$TOOL" = heaptrack ]; then CMD=(heaptrack -o /out/heaptrack "${CMD[@]}"); fi +"${CMD[@]}" 2>/out/merge.stderr & +PID=$! +/monitor.sh "$PID" /out/monitor.csv 1 & +MON=$! +set +e; wait "$PID"; RC=$?; set -e +wait "$MON" || true +echo "merge exit code: $RC" | tee /out/exit_code.txt +tail -5 /out/merge.stderr +' +echo "outputs: $OUT" diff --git a/scripts/monitor_pack.sh b/scripts/monitor_pack.sh new file mode 100755 index 00000000..8adbe57c --- /dev/null +++ b/scripts/monitor_pack.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# monitor_pack.sh — 1 Hz sampler for I/O- and memory-bound stages (pack, merge_partitions). +# +# Usage: +# scripts/monitor_pack.sh --pid PID OUT_DIR [OUTPUT_INDEX_DIR] # attach to a running process +# scripts/monitor_pack.sh --cmd OUT_DIR [OUTPUT_INDEX_DIR] -- CMD... # start CMD and sample it +# env: DT=1 (seconds between samples) +# +# Writes OUT_DIR/samples.csv (one row per sample) and OUT_DIR/env.txt (context +# needed to read it). Linux only. Never reads smaps (it takes mmap_lock and +# slows a process with many mappings); only /proc/PID/{status,io,task/*/stat} +# and /proc/meminfo. +# +# Columns of samples.csv +# t epoch seconds +# rss_anon_kb, rss_file_kb, rss_shmem_kb from /proc/PID/status +# threads thread count +# n_R n_D n_S threads running / in uninterruptible sleep (I/O wait) / sleeping +# (futex, channel, sleep: NOT disk). n_D is the direct count of +# workers blocked on I/O. +# maps number of mappings (wc -l /proc/PID/maps) +# read_bytes write_bytes /proc/PID/io (cumulative, bytes submitted to the +# block layer; buffered writes counted at page dirtying) +# rchar wchar /proc/PID/io (cumulative, bytes through read()/write() syscalls) +# dirty_kb writeback_kb cached_kb mem_avail_kb /proc/meminfo +# +# Pair it with the process's own log (RUST_LOG=info, plus obisys=debug for the +# "activated N worker(s)" lines and obikindex=debug for per-partition timings) +# to align stages on wall-clock time: the log timestamps are UTC ISO-8601, `t` +# is epoch seconds. +set -euo pipefail + +MODE=${1:-}; shift || true +case "$MODE" in + --pid) PID=${1:?PID}; shift ;; + --cmd) ;; + *) sed -n 2,12p "$0"; exit 2 ;; +esac +OUT=${1:?OUT_DIR}; shift +IDX="" +if [ "${1:-}" != "" ] && [ "${1:-}" != "--" ]; then IDX=$1; shift; fi +mkdir -p "$OUT" + +if [ "$MODE" = "--cmd" ]; then + [ "${1:-}" = "--" ] || { echo "--cmd needs: -- CMD..." >&2; exit 2; } + shift + "$@" 2>"$OUT/cmd.stderr" & + PID=$! +fi + +{ + echo "date_utc: $(date -u +%FT%TZ)" + echo "pid: $PID" + echo "nproc: $(nproc)" + echo "kernel: $(uname -r)" + grep -E 'MemTotal|SwapTotal' /proc/meminfo + for k in vm.dirty_ratio vm.dirty_background_ratio vm.dirty_bytes vm.dirty_background_bytes \ + vm.dirty_expire_centisecs vm.dirty_writeback_centisecs vm.max_map_count; do + echo "$k: $(sysctl -n $k 2>/dev/null || echo '?')" + done + echo "ulimit -n: $(ulimit -n)" + echo "cmdline: $(tr '\0' ' ' /dev/null)" + if [ -n "$IDX" ]; then + echo "output_dir: $IDX" + echo "fs_type: $(stat -f -c %T "$IDX" 2>/dev/null || echo '?')" + echo "mount: $(df -P "$IDX" 2>/dev/null | tail -1)" + command -v beegfs-ctl >/dev/null && echo "beegfs: $(beegfs-ctl --getentryinfo "$IDX" 2>&1 | head -8 | tr '\n' ' ')" + fi + env | grep -E '^(MALLOC_|RUST_LOG|RAYON_)' || true +} >"$OUT/env.txt" + +DT=${DT:-1} +printf 't,rss_anon_kb,rss_file_kb,rss_shmem_kb,threads,n_R,n_D,n_S,maps,read_bytes,write_bytes,rchar,wchar,dirty_kb,writeback_kb,cached_kb,mem_avail_kb\n' >"$OUT/samples.csv" + +while [ -d "/proc/$PID" ]; do + t=$(date +%s) + st=$(awk '/^RssAnon:/{a=$2} /^RssFile:/{f=$2} /^RssShmem:/{s=$2} /^Threads:/{th=$2} END{printf "%s,%s,%s,%s",a,f,s,th}' "/proc/$PID/status" 2>/dev/null) || break + # thread states: field 3 of /proc/PID/task/TID/stat, after the ")" closing comm + states=$(cat /proc/$PID/task/*/stat 2>/dev/null | sed -E 's/^.*\) (.).*/\1/' | sort | uniq -c \ + | awk '{c[$2]=$1} END{printf "%d,%d,%d", c["R"]+0, c["D"]+0, c["S"]+0}') + maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null || echo 0) + io=$(awk '/^rchar:/{rc=$2} /^wchar:/{wc=$2} /^read_bytes:/{r=$2} /^write_bytes:/{w=$2} END{printf "%s,%s,%s,%s",r,w,rc,wc}' "/proc/$PID/io" 2>/dev/null || echo ",,,") + mi=$(awk '/^Dirty:/{d=$2} /^Writeback:/{w=$2} /^Cached:/{c=$2} /^MemAvailable:/{m=$2} END{printf "%s,%s,%s,%s",d,w,c,m}' /proc/meminfo) + echo "$t,$st,$states,$maps,$io,$mi" >>"$OUT/samples.csv" + sleep "$DT" +done + +if [ "$MODE" = "--cmd" ]; then wait "$PID" || echo "command exit code: $?" >>"$OUT/env.txt"; fi + +# ── summary ────────────────────────────────────────────────────────────────── +awk -F, 'NR==1{next} + { n++; if(NR==2){t0=$1; r0=$10; w0=$11} + if($2>ma)ma=$2; if($3>mf)mf=$3; if($4>ms)ms=$4; if($5>mt)mt=$5 + if($6>mR)mR=$6; if($7>mD)mD=$7; if($8>mS)mS=$8; sR+=$6; sD+=$7; sS+=$8 + if($14>md)md=$14; if($15>mw)mw=$15; if($9>mm)mm=$9 + t1=$1; r1=$10; w1=$11 } + END{ if(!n)exit + printf "samples: %d over %ds\n", n, t1-t0 + printf "peak RssAnon %.1f GB | RssFile %.1f GB | RssShmem %.1f GB | threads %d | maps %d\n", ma/1e6, mf/1e6, ms/1e6, mt, mm + printf "threads by state (mean / max): R %.1f / %d | D(I/O wait) %.1f / %d | S %.1f / %d\n", sR/n, mR, sD/n, mD, sS/n, mS + printf "peak Dirty %.1f GB | Writeback %.1f GB\n", md/1e6, mw/1e6 + printf "block I/O over the run: read %.1f GB, write %.1f GB\n", (r1-r0)/1e9, (w1-w0)/1e9 }' "$OUT/samples.csv" | tee "$OUT/summary.txt" diff --git a/scripts/watch_merge.sh b/scripts/watch_merge.sh new file mode 100755 index 00000000..934c4eef --- /dev/null +++ b/scripts/watch_merge.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# watch_merge.sh PID OUT.csv — samples a running process and kills it past a threshold. +# +# Every INTERVAL seconds appends one CSV row: number of memory mappings +# (lines of /proc/PID/maps), VmSize, VmData, VmRSS, threads, and the +# system-wide Committed_AS / CommitLimit. If the mapping count exceeds +# MAPS_MAX, VmRSS exceeds RSS_MAX_GB, or VmSize exceeds VMSIZE_MAX_GB (stay +# below `ulimit -v`: past it, allocations fail instead of the guard firing), the process receives SIGTERM (then +# SIGKILL after 30 s) and the reason is written to OUT.csv.reason — a guard +# so a runaway merge ends with a clean message instead of a failed +# allocation (or the OOM killer on a shared node). +# +# Reading /proc/PID/maps is itself slow when the process has hundreds of +# thousands of mappings (it held the kernel's mmap lock in earlier +# measurements): keep INTERVAL >= 5. It never reads /proc/PID/smaps. +# +# Usage: scripts/watch_merge.sh PID OUT.csv +# env: INTERVAL=5 MAPS_MAX=500000 RSS_MAX_GB=400 VMSIZE_MAX_GB=440 +# +# Typical use (Linux only): +# nohup scripts/watch_merge.sh "$(pgrep -f 'obikmer merge' | head -1)" /tmp/watch.csv & +set -u +PID=${1:?usage: watch_merge.sh PID OUT.csv} +OUT=${2:?usage: watch_merge.sh PID OUT.csv} +INTERVAL=${INTERVAL:-5} +MAPS_MAX=${MAPS_MAX:-500000} +RSS_MAX_KB=$(( ${RSS_MAX_GB:-400} * 1024 * 1024 )) +VMSIZE_MAX_KB=$(( ${VMSIZE_MAX_GB:-440} * 1024 * 1024 )) + +[ -r "/proc/$PID/maps" ] || { echo "cannot read /proc/$PID/maps" >&2; exit 1; } +printf 'time,maps,vmsize_kb,vmdata_kb,vmrss_kb,threads,committed_as_kb,commit_limit_kb\n' >"$OUT" + +while kill -0 "$PID" 2>/dev/null; do + maps=$(wc -l <"/proc/$PID/maps" 2>/dev/null) || break + st=$(grep -E '^(VmSize|VmData|VmRSS|Threads):' "/proc/$PID/status" 2>/dev/null | awk '{printf "%s,", $2}') + rss=$(grep '^VmRSS:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}') + vsz=$(grep '^VmSize:' "/proc/$PID/status" 2>/dev/null | awk '{print $2}') + mi=$(grep -E '^(Committed_AS|CommitLimit):' /proc/meminfo | awk '{printf "%s,", $2}') + echo "$(date +%T),$maps,${st}${mi%,}" >>"$OUT" + + reason="" + [ "$maps" -gt "$MAPS_MAX" ] && reason="mappings $maps > MAPS_MAX $MAPS_MAX" + [ -n "${rss:-}" ] && [ "$rss" -gt "$RSS_MAX_KB" ] && reason="VmRSS ${rss} kB > RSS_MAX_GB" + [ -n "${vsz:-}" ] && [ "$vsz" -gt "$VMSIZE_MAX_KB" ] && reason="VmSize ${vsz} kB > VMSIZE_MAX_GB" + if [ -n "$reason" ]; then + echo "$(date +%T) GUARD: $reason — sending SIGTERM to $PID" | tee "$OUT.reason" >&2 + kill -TERM "$PID" 2>/dev/null + for _ in $(seq 30); do kill -0 "$PID" 2>/dev/null || exit 0; sleep 1; done + kill -KILL "$PID" 2>/dev/null + exit 0 + fi + sleep "$INTERVAL" +done diff --git a/src/obikindex/src/index/kmer_index.rs b/src/obikindex/src/index/kmer_index.rs index 4f727a71..9fe2b112 100644 --- a/src/obikindex/src/index/kmer_index.rs +++ b/src/obikindex/src/index/kmer_index.rs @@ -296,7 +296,14 @@ impl KmerIndex { }; let n = self.n_partitions(); - let order: Vec = (0..n).collect(); + // Largest partitions first (LFD), so the long jobs do not end up as + // stragglers at the tail. Sizes are `stat()` only; the sort is stable, + // ties keep partition-number order. Partitions are independent, so + // the order has no effect on what is written. + let sizes: Vec = (0..n).map(|i| self.pending_matrix_bytes(i)).collect(); + let mut order: Vec = (0..n).collect(); + order.sort_by_key(|&i| std::cmp::Reverse(sizes[i])); + let mut timings: Vec<(usize, u64, std::time::Duration)> = Vec::with_capacity(n); let pb = progress_bar("pack", n as u64, "partitions"); obisys::PartitionRunner::new().run( &order, @@ -327,14 +334,50 @@ impl KmerIndex { } Ok(()) }, - |_, _, _| { + |i, _, elapsed| { + tracing::debug!( + partition = i, + matrix_bytes = sizes[i], + secs = elapsed.as_secs_f64(), + "pack: partition done" + ); + timings.push((i, sizes[i], elapsed)); pb.inc(1); }, )?; pb.finish_and_clear(); + report_pack_timings(&timings); Ok(()) } + /// Bytes of matrix data (`presence/` and `counts/` of every layer) still + /// present in partition `i`, from `stat()` only. Before packing this is + /// the columnar data, i.e. a proxy for the work `pack_matrices` has to do + /// on that partition. Missing partition or directories count as 0. + pub fn pending_matrix_bytes(&self, i: usize) -> u64 { + let Ok(n_layers) = self.n_layers(i) else { + return 0; + }; + let mut total = 0u64; + for l in 0..n_layers { + let Ok(layer_dir) = self.layer_dir(i, l) else { + continue; + }; + for sub in ["presence", "counts"] { + let Ok(entries) = fs::read_dir(layer_dir.join(sub)) else { + continue; + }; + total += entries + .flatten() + .filter_map(|e| e.metadata().ok()) + .filter(|m| m.is_file()) + .map(|m| m.len()) + .sum::(); + } + } + total + } + // /// Write a `layer_meta.json` in any layer directory that is missing one. // /// // /// Old indexes were built before this file was required. The number of @@ -394,3 +437,34 @@ impl KmerIndex { // Ok(()) // } } + +/// One `info!` summary of `pack_matrices` per-partition timings: the sum of +/// partition times (divide by the stage wall time to get the mean number of +/// concurrently busy workers), the median and maximum, and the slowest +/// partitions with their pre-pack matrix size. `timings` holds +/// `(partition, matrix_bytes, elapsed)`. +fn report_pack_timings(timings: &[(usize, u64, std::time::Duration)]) { + if timings.is_empty() { + return; + } + let mut by_time: Vec<&(usize, u64, std::time::Duration)> = timings.iter().collect(); + by_time.sort_by_key(|t| std::cmp::Reverse(t.2)); + let sum: f64 = timings.iter().map(|t| t.2.as_secs_f64()).sum(); + let bytes: u64 = timings.iter().map(|t| t.1).sum(); + let median = by_time[by_time.len() / 2].2.as_secs_f64(); + tracing::info!( + "pack: {} partition(s), sum of partition times {:.0}s, median {:.1}s, max {:.1}s, {:.1} GB of matrix data", + timings.len(), + sum, + median, + by_time[0].2.as_secs_f64(), + bytes as f64 / 1e9, + ); + for (i, b, d) in by_time.iter().take(5).map(|t| (t.0, t.1, t.2)) { + tracing::info!( + "pack: slowest partition {i}: {:.1}s, {:.2} GB of matrix data", + d.as_secs_f64(), + b as f64 / 1e9, + ); + } +} diff --git a/src/obisys/src/stage.rs b/src/obisys/src/stage.rs index 2cb75647..91608cac 100644 --- a/src/obisys/src/stage.rs +++ b/src/obisys/src/stage.rs @@ -49,6 +49,14 @@ impl Stage { wall_secs = format_args!("{:.3}", stats.wall_secs), rss = %fmt_bytes(stats.max_rss_bytes), swaps = stats.swaps, + user_secs = format_args!("{:.1}", stats.user_secs), + sys_secs = format_args!("{:.1}", stats.sys_secs), + minor_faults = stats.minor_faults, + major_faults = stats.major_faults, + vol_ctx = stats.vol_ctx, + invol_ctx = stats.invol_ctx, + in_blocks = stats.in_blocks, + out_blocks = stats.out_blocks, "done" ); if stats.swaps > 0 {