Files
obikmer/scripts/bench_merge_oracle.sh
Eric Coissac 07bc5fc634
ci.yml / build (pull_request) Successful in 4m53s
Release / create-release (push) Successful in 28s
Release / build-linux-x86_64 (push) Successful in 6m46s
Release / build-macos-arm64 (push) Successful in 4m8s
docs: update sparse packing strategy and benchmark scripts
Update documentation to reflect that sparse packing now maps all columns of a layer at once and packs layers sequentially, reversing the previous choice to avoid materializing a packed copy. This change introduces an extra write/read cycle for the dense matrix, increasing pack benchmark time from 1.11s to 1.60s.

- Modify `pack_sparse_bit_matrix` in `src/obicompactvec/src/bitmatrix/sparse.rs` to always convert via the dense packed format, simplifying logic by removing conditional handling of existing files and ensuring a single mapping.
- Update `pack_sparse_compact_int_matrix` in `src/obicompactvec/src/sparse_intmatrix.rs` to exclusively handle conversion from the dense packed format (`matrix.pcmx`) to sparse, removing columnar format handling and simplifying file cleanup.
- Adjust `scripts/bench_merge_oracle.sh` to extract `PACK_WALL` metrics via `stage_field` for merge operations, updating grep patterns and output messages to report pack wall time.
- Revise `DevDocMD/implementation/merge_mmap_audit.md` and `merge_scaling.md` to document the new sequential packing strategy, noting that `N_part` scaling applies only to incremental merges and that `obisys::MemoryBudget` is no longer used by merge.
- Bump package version in `src/obikmer/Cargo.toml` from 1.4.3 to 1.4.4.
2026-11-17 15:35:41 +01:00

157 lines
6.6 KiB
Bash
Executable File

#!/usr/bin/env bash
# bench_merge_oracle.sh — golden-output + timing oracle for `obikmer merge`
# on the real-data benchmark (benchmark/run/specimen_index_presence, 20
# presence indexes built from real genomes).
#
# Companion of smoke_test_merge.sh for refactors of obikmerge (see
# DevDocMD/implementation/merge_mmap_audit.md). Same merge options as
# benchmark/merge_presence.sh (--force --force-presence), but the output goes
# to a dedicated directory, never to benchmark/run/global_index_presence.
#
# What it does:
# 1. builds the release `obikmer`
# 2. merges every specimen presence index (sorted, as merge_presence.sh)
# and keeps the stderr log (stage report: wall/RSS of merge_partitions)
# 3. `obikmer dump --debug` of the result → header first, rows sorted
# (LC_ALL=C) → sha256 of that sorted text, plus a zstd copy
# 4. --record DIR: writes DIR/merged (the merged index), DIR/merge.stderr.log, DIR/dump.sha256,
# DIR/dump.sorted.csv.zst, and (informative cross-check)
# DIR/global_index_presence.sha256 = same dump hash computed on the
# existing benchmark/run/global_index_presence (older binary — a
# difference there is reported, not judged)
# --check DIR: merges into a scratch directory, compares the hash with
# DIR/dump.sha256 (full diff against the zstd copy only on mismatch),
# and prints merge_partitions wall/RSS and pack wall before → after
#
# Usage:
# scripts/bench_merge_oracle.sh (--record DIR | --check DIR) [--keep]
set -euo pipefail
KEEP=0
MODE=""
DIR=""
while [ $# -gt 0 ]; do
case "$1" in
--record) MODE=record; DIR="$2"; shift 2 ;;
--check) MODE=check; DIR="$2"; shift 2 ;;
--keep) KEEP=1; shift ;;
*) echo "unknown argument: $1" >&2; exit 2 ;;
esac
done
[ -n "$MODE" ] || { echo "usage: $0 (--record DIR | --check DIR) [--keep]" >&2; exit 2; }
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
case "$DIR" in /*) ;; *) DIR="$PWD/$DIR" ;; esac
RUN_DIR="$REPO_ROOT/benchmark/run"
IDX_DIR="$RUN_DIR/specimen_index_presence"
GLOBAL_OLD="$RUN_DIR/global_index_presence"
BIN="$REPO_ROOT/src/target/release/obikmer"
mkdir -p "$DIR" "$RUN_DIR/merge_oracle"
# Scratch space on the same file system as the benchmark data (merge's
# bootstrap hard-links from its base source; sort needs room for temp files).
WORK="$(mktemp -d "$RUN_DIR/merge_oracle/.work.XXXXXX")"
cleanup() {
if [ "$KEEP" -eq 1 ]; then echo "kept: $WORK"; else rm -rf "$WORK"; fi
}
trap cleanup EXIT
fail() {
echo "FAIL: $1" >&2
exit 1
}
# ── 1. build ────────────────────────────────────────────────────────────────
if ! (cd "$REPO_ROOT/src" && cargo build --release -q -p obikmer --bin obikmer) > "$WORK/build.log" 2>&1; then
cat "$WORK/build.log" >&2
fail "cargo build --release failed"
fi
sorted_dump() { # sorted_dump INDEX OUT_TXT — header first, rows sorted
local raw="$WORK/dump.raw.csv"
"$BIN" dump --debug "$1" > "$raw" 2> "$WORK/dump.log" \
|| { cat "$WORK/dump.log" >&2; fail "obikmer dump $1 exited non-zero"; }
{ head -n 1 "$raw"
tail -n +2 "$raw" | LC_ALL=C sort -S 25% --parallel=8 -T "$WORK"
} > "$2"
rm -f "$raw"
[ "$(head -n 2 "$2" | wc -l)" -eq 2 ] || fail "empty dump of $1"
}
hash_of() { sha256sum "$1" | cut -d' ' -f1; }
stage_field() { # stage_field LOG FIELD [STAGE] — FIELD of the 'done stage=STAGE' line (default merge_partitions)
perl -pe 's/\e\[[0-9;]*m//g' "$1" \
| grep "done stage=${3:-merge_partitions} " \
| sed -n "s/.*$2=\([^ ]*\( [KMG]*B\)\{0,1\}\).*/\1/p" | tail -n 1
}
# ── 2. merge ────────────────────────────────────────────────────────────────
mapfile -t SOURCES < <(find "$IDX_DIR" -mindepth 1 -maxdepth 1 -type d | sort)
[ "${#SOURCES[@]}" -gt 0 ] || fail "no indexes found in $IDX_DIR"
if [ "$MODE" = record ]; then
MERGED="$DIR/merged"
LOG="$DIR/merge.stderr.log"
else
MERGED="$WORK/merged"
LOG="$WORK/merge.stderr.log"
fi
echo "merging ${#SOURCES[@]} presence indexes → $MERGED"
if ! "$BIN" merge --output "$MERGED" --force --force-presence "${SOURCES[@]}" \
> /dev/null 2> "$LOG"; then
tail -n 40 "$LOG" >&2
fail "obikmer merge exited non-zero"
fi
WALL="$(stage_field "$LOG" wall_secs)"
RSS="$(stage_field "$LOG" rss)"
PACK_WALL="$(stage_field "$LOG" wall_secs pack)"
echo "merge_partitions: wall=${WALL}s rss=${RSS} pack: wall=${PACK_WALL}s"
# ── 3. sorted dump + hash ───────────────────────────────────────────────────
DUMP="$WORK/dump.sorted.csv"
sorted_dump "$MERGED" "$DUMP"
H="$(hash_of "$DUMP")"
echo "dump: $(($(wc -l < "$DUMP") - 1)) k-mers, sha256 $H"
# ── 4. record or check ──────────────────────────────────────────────────────
if [ "$MODE" = record ]; then
echo "$H" > "$DIR/dump.sha256"
zstd -q -T0 -f "$DUMP" -o "$DIR/dump.sorted.csv.zst"
if [ -d "$GLOBAL_OLD" ]; then
OLD_DUMP="$WORK/old.sorted.csv"
sorted_dump "$GLOBAL_OLD" "$OLD_DUMP"
HO="$(hash_of "$OLD_DUMP")"
echo "$HO" > "$DIR/global_index_presence.sha256"
if [ "$HO" = "$H" ]; then
echo "cross-check: global_index_presence (older binary) dump is identical"
else
echo "cross-check: global_index_presence (older binary) dump DIFFERS" \
"($(($(wc -l < "$OLD_DUMP") - 1)) k-mers, sha256 $HO) — informative only"
fi
fi
echo "RECORDED → $DIR"
exit 0
fi
[ -f "$DIR/dump.sha256" ] || fail "no reference in $DIR (run --record first)"
REF_H="$(cat "$DIR/dump.sha256")"
REF_WALL="$(stage_field "$DIR/merge.stderr.log" wall_secs)"
REF_RSS="$(stage_field "$DIR/merge.stderr.log" rss)"
REF_PACK="$(stage_field "$DIR/merge.stderr.log" wall_secs pack)"
echo "merge_partitions wall: ${REF_WALL}s → ${WALL}s rss: ${REF_RSS} → ${RSS}"
echo "pack wall: ${REF_PACK}s → ${PACK_WALL}s"
python3 -c "import sys; r,n=float(sys.argv[1]),float(sys.argv[2]); print(f'merge_partitions wall ratio: {n/r:.2f}' + (' (> 1.20: SLOWER THAN 20 %)' if n > 1.2*r else ''))" \
"$REF_WALL" "$WALL" || true
if [ "$H" = "$REF_H" ]; then
echo "PASS: dump identical to reference ($H)"
exit 0
fi
echo "FAIL: dump hash differs (ref $REF_H, now $H)" >&2
zstd -q -d -c "$DIR/dump.sorted.csv.zst" | { diff - "$DUMP" || true; } | head -n 40 >&2
exit 1