Update documentation to reflect that sparse packing now maps all columns of a layer at once and packs layers sequentially, reversing the previous choice to avoid materializing a packed copy. This change introduces an extra write/read cycle for the dense matrix, increasing pack benchmark time from 1.11s to 1.60s. - Modify `pack_sparse_bit_matrix` in `src/obicompactvec/src/bitmatrix/sparse.rs` to always convert via the dense packed format, simplifying logic by removing conditional handling of existing files and ensuring a single mapping. - Update `pack_sparse_compact_int_matrix` in `src/obicompactvec/src/sparse_intmatrix.rs` to exclusively handle conversion from the dense packed format (`matrix.pcmx`) to sparse, removing columnar format handling and simplifying file cleanup. - Adjust `scripts/bench_merge_oracle.sh` to extract `PACK_WALL` metrics via `stage_field` for merge operations, updating grep patterns and output messages to report pack wall time. - Revise `DevDocMD/implementation/merge_mmap_audit.md` and `merge_scaling.md` to document the new sequential packing strategy, noting that `N_part` scaling applies only to incremental merges and that `obisys::MemoryBudget` is no longer used by merge. - Bump package version in `src/obikmer/Cargo.toml` from 1.4.3 to 1.4.4.
157 lines
6.6 KiB
Bash
Executable File
157 lines
6.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bench_merge_oracle.sh — golden-output + timing oracle for `obikmer merge`
|
|
# on the real-data benchmark (benchmark/run/specimen_index_presence, 20
|
|
# presence indexes built from real genomes).
|
|
#
|
|
# Companion of smoke_test_merge.sh for refactors of obikmerge (see
|
|
# DevDocMD/implementation/merge_mmap_audit.md). Same merge options as
|
|
# benchmark/merge_presence.sh (--force --force-presence), but the output goes
|
|
# to a dedicated directory, never to benchmark/run/global_index_presence.
|
|
#
|
|
# What it does:
|
|
# 1. builds the release `obikmer`
|
|
# 2. merges every specimen presence index (sorted, as merge_presence.sh)
|
|
# and keeps the stderr log (stage report: wall/RSS of merge_partitions)
|
|
# 3. `obikmer dump --debug` of the result → header first, rows sorted
|
|
# (LC_ALL=C) → sha256 of that sorted text, plus a zstd copy
|
|
# 4. --record DIR: writes DIR/merged (the merged index), DIR/merge.stderr.log, DIR/dump.sha256,
|
|
# DIR/dump.sorted.csv.zst, and (informative cross-check)
|
|
# DIR/global_index_presence.sha256 = same dump hash computed on the
|
|
# existing benchmark/run/global_index_presence (older binary — a
|
|
# difference there is reported, not judged)
|
|
# --check DIR: merges into a scratch directory, compares the hash with
|
|
# DIR/dump.sha256 (full diff against the zstd copy only on mismatch),
|
|
# and prints merge_partitions wall/RSS and pack wall before → after
|
|
#
|
|
# Usage:
|
|
# scripts/bench_merge_oracle.sh (--record DIR | --check DIR) [--keep]
|
|
|
|
set -euo pipefail
|
|
|
|
KEEP=0
|
|
MODE=""
|
|
DIR=""
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--record) MODE=record; DIR="$2"; shift 2 ;;
|
|
--check) MODE=check; DIR="$2"; shift 2 ;;
|
|
--keep) KEEP=1; shift ;;
|
|
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
[ -n "$MODE" ] || { echo "usage: $0 (--record DIR | --check DIR) [--keep]" >&2; exit 2; }
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
case "$DIR" in /*) ;; *) DIR="$PWD/$DIR" ;; esac
|
|
RUN_DIR="$REPO_ROOT/benchmark/run"
|
|
IDX_DIR="$RUN_DIR/specimen_index_presence"
|
|
GLOBAL_OLD="$RUN_DIR/global_index_presence"
|
|
BIN="$REPO_ROOT/src/target/release/obikmer"
|
|
|
|
mkdir -p "$DIR" "$RUN_DIR/merge_oracle"
|
|
# Scratch space on the same file system as the benchmark data (merge's
|
|
# bootstrap hard-links from its base source; sort needs room for temp files).
|
|
WORK="$(mktemp -d "$RUN_DIR/merge_oracle/.work.XXXXXX")"
|
|
cleanup() {
|
|
if [ "$KEEP" -eq 1 ]; then echo "kept: $WORK"; else rm -rf "$WORK"; fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
fail() {
|
|
echo "FAIL: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
# ── 1. build ────────────────────────────────────────────────────────────────
|
|
if ! (cd "$REPO_ROOT/src" && cargo build --release -q -p obikmer --bin obikmer) > "$WORK/build.log" 2>&1; then
|
|
cat "$WORK/build.log" >&2
|
|
fail "cargo build --release failed"
|
|
fi
|
|
|
|
sorted_dump() { # sorted_dump INDEX OUT_TXT — header first, rows sorted
|
|
local raw="$WORK/dump.raw.csv"
|
|
"$BIN" dump --debug "$1" > "$raw" 2> "$WORK/dump.log" \
|
|
|| { cat "$WORK/dump.log" >&2; fail "obikmer dump $1 exited non-zero"; }
|
|
{ head -n 1 "$raw"
|
|
tail -n +2 "$raw" | LC_ALL=C sort -S 25% --parallel=8 -T "$WORK"
|
|
} > "$2"
|
|
rm -f "$raw"
|
|
[ "$(head -n 2 "$2" | wc -l)" -eq 2 ] || fail "empty dump of $1"
|
|
}
|
|
|
|
hash_of() { sha256sum "$1" | cut -d' ' -f1; }
|
|
|
|
stage_field() { # stage_field LOG FIELD [STAGE] — FIELD of the 'done stage=STAGE' line (default merge_partitions)
|
|
perl -pe 's/\e\[[0-9;]*m//g' "$1" \
|
|
| grep "done stage=${3:-merge_partitions} " \
|
|
| sed -n "s/.*$2=\([^ ]*\( [KMG]*B\)\{0,1\}\).*/\1/p" | tail -n 1
|
|
}
|
|
|
|
# ── 2. merge ────────────────────────────────────────────────────────────────
|
|
mapfile -t SOURCES < <(find "$IDX_DIR" -mindepth 1 -maxdepth 1 -type d | sort)
|
|
[ "${#SOURCES[@]}" -gt 0 ] || fail "no indexes found in $IDX_DIR"
|
|
|
|
if [ "$MODE" = record ]; then
|
|
MERGED="$DIR/merged"
|
|
LOG="$DIR/merge.stderr.log"
|
|
else
|
|
MERGED="$WORK/merged"
|
|
LOG="$WORK/merge.stderr.log"
|
|
fi
|
|
|
|
echo "merging ${#SOURCES[@]} presence indexes → $MERGED"
|
|
if ! "$BIN" merge --output "$MERGED" --force --force-presence "${SOURCES[@]}" \
|
|
> /dev/null 2> "$LOG"; then
|
|
tail -n 40 "$LOG" >&2
|
|
fail "obikmer merge exited non-zero"
|
|
fi
|
|
WALL="$(stage_field "$LOG" wall_secs)"
|
|
RSS="$(stage_field "$LOG" rss)"
|
|
PACK_WALL="$(stage_field "$LOG" wall_secs pack)"
|
|
echo "merge_partitions: wall=${WALL}s rss=${RSS} pack: wall=${PACK_WALL}s"
|
|
|
|
# ── 3. sorted dump + hash ───────────────────────────────────────────────────
|
|
DUMP="$WORK/dump.sorted.csv"
|
|
sorted_dump "$MERGED" "$DUMP"
|
|
H="$(hash_of "$DUMP")"
|
|
echo "dump: $(($(wc -l < "$DUMP") - 1)) k-mers, sha256 $H"
|
|
|
|
# ── 4. record or check ──────────────────────────────────────────────────────
|
|
if [ "$MODE" = record ]; then
|
|
echo "$H" > "$DIR/dump.sha256"
|
|
zstd -q -T0 -f "$DUMP" -o "$DIR/dump.sorted.csv.zst"
|
|
|
|
if [ -d "$GLOBAL_OLD" ]; then
|
|
OLD_DUMP="$WORK/old.sorted.csv"
|
|
sorted_dump "$GLOBAL_OLD" "$OLD_DUMP"
|
|
HO="$(hash_of "$OLD_DUMP")"
|
|
echo "$HO" > "$DIR/global_index_presence.sha256"
|
|
if [ "$HO" = "$H" ]; then
|
|
echo "cross-check: global_index_presence (older binary) dump is identical"
|
|
else
|
|
echo "cross-check: global_index_presence (older binary) dump DIFFERS" \
|
|
"($(($(wc -l < "$OLD_DUMP") - 1)) k-mers, sha256 $HO) — informative only"
|
|
fi
|
|
fi
|
|
echo "RECORDED → $DIR"
|
|
exit 0
|
|
fi
|
|
|
|
[ -f "$DIR/dump.sha256" ] || fail "no reference in $DIR (run --record first)"
|
|
REF_H="$(cat "$DIR/dump.sha256")"
|
|
REF_WALL="$(stage_field "$DIR/merge.stderr.log" wall_secs)"
|
|
REF_RSS="$(stage_field "$DIR/merge.stderr.log" rss)"
|
|
REF_PACK="$(stage_field "$DIR/merge.stderr.log" wall_secs pack)"
|
|
echo "merge_partitions wall: ${REF_WALL}s → ${WALL}s rss: ${REF_RSS} → ${RSS}"
|
|
echo "pack wall: ${REF_PACK}s → ${PACK_WALL}s"
|
|
python3 -c "import sys; r,n=float(sys.argv[1]),float(sys.argv[2]); print(f'merge_partitions wall ratio: {n/r:.2f}' + (' (> 1.20: SLOWER THAN 20 %)' if n > 1.2*r else ''))" \
|
|
"$REF_WALL" "$WALL" || true
|
|
|
|
if [ "$H" = "$REF_H" ]; then
|
|
echo "PASS: dump identical to reference ($H)"
|
|
exit 0
|
|
fi
|
|
echo "FAIL: dump hash differs (ref $REF_H, now $H)" >&2
|
|
zstd -q -d -c "$DIR/dump.sorted.csv.zst" | { diff - "$DUMP" || true; } | head -n 40 >&2
|
|
exit 1
|