feat: implement in-memory column builders and refactor merge partitioning
Introduce `MemBitVecBuilder` and `MemCompactIntVecBuilder` to construct columns in RAM before writing, reducing the number of open file descriptors during merge operations. Refactor `partition_merge.rs` to use these new builders via `add_mem_col`, decoupling live memory mappings from source counts to prevent `SIGBUS` errors. - Add `mem_builder` module with `MemBitVecBuilder` and `MemCompactIntVecBuilder` structs for in-memory column construction and PCIV file writing. - Extend `MatrixBuilder` with `add_mem_col` and `n_cols` methods, updating `ColBuilder` enum to support `MemBit` and `MemInt` variants. - Refactor `partition_merge.rs` to use `add_mem_col` instead of `add_col`, introducing `run_pass2` for batch processing and optimizing destination layer handling. - Update `format.rs` with `pciv_index_layout`, `write_pciv_header`, and `write_pciv_tail` functions to manage PCIV index layout and header/tail sections. - Modify `mmap_file.rs` to use `write_std_header` and introduce `write_built` for standardized file writing and flushing logic. - Make `MAGIC` constant public in `bitvec.rs` and update documentation for memory mapping alignment and structure. - Add test modules `tests/matrix_builder.rs` and `tests/mem_builder.rs` to verify column counts and byte-identical output of new builders against persistent versions. - Update `mkdocs.yml` navigation to include "Merge mmap audit (2026-11)" documentation entry. - Add implementation audit documentation in `DevDocMD/implementation/merge_mmap_audit.md` detailing memory mapping failures and refactoring plans. - Add merge testing scripts `scripts/smoke_test_merge.sh` and `scripts/bench_merge_oracle.sh` for validation and benchmarking. - Bump `obikmer` package version from 1.4.2 to 1.4.3 in `Cargo.toml`.
This commit is contained in:
1 parent
b35127e4e3
commit
1f199c69db
25 files changed
+1631
-687
No files matched your search
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# bench_merge_oracle.sh — golden-output + timing oracle for `obikmer merge`
|
||||
# on the real-data benchmark (benchmark/run/specimen_index_presence, 20
|
||||
# presence indexes built from real genomes).
|
||||
#
|
||||
# Companion of smoke_test_merge.sh for refactors of obikmerge (see
|
||||
# DevDocMD/implementation/merge_mmap_audit.md). Same merge options as
|
||||
# benchmark/merge_presence.sh (--force --force-presence), but the output goes
|
||||
# to a dedicated directory, never to benchmark/run/global_index_presence.
|
||||
#
|
||||
# What it does:
|
||||
# 1. builds the release `obikmer`
|
||||
# 2. merges every specimen presence index (sorted, as merge_presence.sh)
|
||||
# and keeps the stderr log (stage report: wall/RSS of merge_partitions)
|
||||
# 3. `obikmer dump --debug` of the result → header first, rows sorted
|
||||
# (LC_ALL=C) → sha256 of that sorted text, plus a zstd copy
|
||||
# 4. --record DIR: writes DIR/merged (the merged index), DIR/merge.stderr.log, DIR/dump.sha256,
|
||||
# DIR/dump.sorted.csv.zst, and (informative cross-check)
|
||||
# DIR/global_index_presence.sha256 = same dump hash computed on the
|
||||
# existing benchmark/run/global_index_presence (older binary — a
|
||||
# difference there is reported, not judged)
|
||||
# --check DIR: merges into a scratch directory, compares the hash with
|
||||
# DIR/dump.sha256 (full diff against the zstd copy only on mismatch),
|
||||
# and prints merge_partitions wall/RSS before → after
|
||||
#
|
||||
# Usage:
|
||||
# scripts/bench_merge_oracle.sh (--record DIR | --check DIR) [--keep]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
KEEP=0
|
||||
MODE=""
|
||||
DIR=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--record) MODE=record; DIR="$2"; shift 2 ;;
|
||||
--check) MODE=check; DIR="$2"; shift 2 ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$MODE" ] || { echo "usage: $0 (--record DIR | --check DIR) [--keep]" >&2; exit 2; }
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
case "$DIR" in /*) ;; *) DIR="$PWD/$DIR" ;; esac
|
||||
RUN_DIR="$REPO_ROOT/benchmark/run"
|
||||
IDX_DIR="$RUN_DIR/specimen_index_presence"
|
||||
GLOBAL_OLD="$RUN_DIR/global_index_presence"
|
||||
BIN="$REPO_ROOT/src/target/release/obikmer"
|
||||
|
||||
mkdir -p "$DIR" "$RUN_DIR/merge_oracle"
|
||||
# Scratch space on the same file system as the benchmark data (merge's
|
||||
# bootstrap hard-links from its base source; sort needs room for temp files).
|
||||
WORK="$(mktemp -d "$RUN_DIR/merge_oracle/.work.XXXXXX")"
|
||||
cleanup() {
|
||||
if [ "$KEEP" -eq 1 ]; then echo "kept: $WORK"; else rm -rf "$WORK"; fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── 1. build ────────────────────────────────────────────────────────────────
|
||||
if ! (cd "$REPO_ROOT/src" && cargo build --release -q -p obikmer --bin obikmer) > "$WORK/build.log" 2>&1; then
|
||||
cat "$WORK/build.log" >&2
|
||||
fail "cargo build --release failed"
|
||||
fi
|
||||
|
||||
sorted_dump() { # sorted_dump INDEX OUT_TXT — header first, rows sorted
|
||||
local raw="$WORK/dump.raw.csv"
|
||||
"$BIN" dump --debug "$1" > "$raw" 2> "$WORK/dump.log" \
|
||||
|| { cat "$WORK/dump.log" >&2; fail "obikmer dump $1 exited non-zero"; }
|
||||
{ head -n 1 "$raw"
|
||||
tail -n +2 "$raw" | LC_ALL=C sort -S 25% --parallel=8 -T "$WORK"
|
||||
} > "$2"
|
||||
rm -f "$raw"
|
||||
[ "$(head -n 2 "$2" | wc -l)" -eq 2 ] || fail "empty dump of $1"
|
||||
}
|
||||
|
||||
hash_of() { sha256sum "$1" | cut -d' ' -f1; }
|
||||
|
||||
stage_field() { # stage_field LOG FIELD — FIELD of the 'done stage=merge_partitions' line
|
||||
perl -pe 's/\e\[[0-9;]*m//g' "$1" \
|
||||
| grep 'done stage=merge_partitions' \
|
||||
| sed -n "s/.*$2=\([^ ]*\( [KMG]*B\)\{0,1\}\).*/\1/p" | tail -n 1
|
||||
}
|
||||
|
||||
# ── 2. merge ────────────────────────────────────────────────────────────────
|
||||
mapfile -t SOURCES < <(find "$IDX_DIR" -mindepth 1 -maxdepth 1 -type d | sort)
|
||||
[ "${#SOURCES[@]}" -gt 0 ] || fail "no indexes found in $IDX_DIR"
|
||||
|
||||
if [ "$MODE" = record ]; then
|
||||
MERGED="$DIR/merged"
|
||||
LOG="$DIR/merge.stderr.log"
|
||||
else
|
||||
MERGED="$WORK/merged"
|
||||
LOG="$WORK/merge.stderr.log"
|
||||
fi
|
||||
|
||||
echo "merging ${#SOURCES[@]} presence indexes → $MERGED"
|
||||
if ! "$BIN" merge --output "$MERGED" --force --force-presence "${SOURCES[@]}" \
|
||||
> /dev/null 2> "$LOG"; then
|
||||
tail -n 40 "$LOG" >&2
|
||||
fail "obikmer merge exited non-zero"
|
||||
fi
|
||||
WALL="$(stage_field "$LOG" wall_secs)"
|
||||
RSS="$(stage_field "$LOG" rss)"
|
||||
echo "merge_partitions: wall=${WALL}s rss=${RSS}"
|
||||
|
||||
# ── 3. sorted dump + hash ───────────────────────────────────────────────────
|
||||
DUMP="$WORK/dump.sorted.csv"
|
||||
sorted_dump "$MERGED" "$DUMP"
|
||||
H="$(hash_of "$DUMP")"
|
||||
echo "dump: $(($(wc -l < "$DUMP") - 1)) k-mers, sha256 $H"
|
||||
|
||||
# ── 4. record or check ──────────────────────────────────────────────────────
|
||||
if [ "$MODE" = record ]; then
|
||||
echo "$H" > "$DIR/dump.sha256"
|
||||
zstd -q -T0 -f "$DUMP" -o "$DIR/dump.sorted.csv.zst"
|
||||
|
||||
if [ -d "$GLOBAL_OLD" ]; then
|
||||
OLD_DUMP="$WORK/old.sorted.csv"
|
||||
sorted_dump "$GLOBAL_OLD" "$OLD_DUMP"
|
||||
HO="$(hash_of "$OLD_DUMP")"
|
||||
echo "$HO" > "$DIR/global_index_presence.sha256"
|
||||
if [ "$HO" = "$H" ]; then
|
||||
echo "cross-check: global_index_presence (older binary) dump is identical"
|
||||
else
|
||||
echo "cross-check: global_index_presence (older binary) dump DIFFERS" \
|
||||
"($(($(wc -l < "$OLD_DUMP") - 1)) k-mers, sha256 $HO) — informative only"
|
||||
fi
|
||||
fi
|
||||
echo "RECORDED → $DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ -f "$DIR/dump.sha256" ] || fail "no reference in $DIR (run --record first)"
|
||||
REF_H="$(cat "$DIR/dump.sha256")"
|
||||
REF_WALL="$(stage_field "$DIR/merge.stderr.log" wall_secs)"
|
||||
REF_RSS="$(stage_field "$DIR/merge.stderr.log" rss)"
|
||||
echo "merge_partitions wall: ${REF_WALL}s → ${WALL}s rss: ${REF_RSS} → ${RSS}"
|
||||
python3 -c "import sys; r,n=float(sys.argv[1]),float(sys.argv[2]); print(f'merge_partitions wall ratio: {n/r:.2f}' + (' (> 1.20: SLOWER THAN 20 %)' if n > 1.2*r else ''))" \
|
||||
"$REF_WALL" "$WALL" || true
|
||||
|
||||
if [ "$H" = "$REF_H" ]; then
|
||||
echo "PASS: dump identical to reference ($H)"
|
||||
exit 0
|
||||
fi
|
||||
echo "FAIL: dump hash differs (ref $REF_H, now $H)" >&2
|
||||
zstd -q -d -c "$DIR/dump.sorted.csv.zst" | { diff - "$DUMP" || true; } | head -n 40 >&2
|
||||
exit 1
|
||||
Executable
+186
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env bash
|
||||
# smoke_test_merge.sh — end-to-end golden-output oracle for `obikmer merge`
|
||||
#
|
||||
# Non-regression oracle for refactors of obikmerge (see
|
||||
# DevDocMD/implementation/merge_mmap_audit.md). Like smoke_test_index.sh it
|
||||
# drives the real CLI, never the library APIs.
|
||||
#
|
||||
# What it does:
|
||||
# 1. builds `obikmer` (debug)
|
||||
# 2. generates (on --record) 4 small deterministic FASTA files A, B, C, D sharing
|
||||
# fragments, so that merge both hits k-mers already present in the
|
||||
# destination ("existing layer" path) and creates new ones ("new layer"
|
||||
# path). D carries a short motif repeated > 255 times so the count track
|
||||
# exercises the compact-int overflow (counts >= 255).
|
||||
# 3. on --record, indexes each genome twice (presence, --with-counts) and
|
||||
# freezes the indexes in DIR/sources; --check reuses them (see below),
|
||||
# with -p PARTITIONS partitions
|
||||
# 4. runs the merge scenarios:
|
||||
# s1_presence / s1_count merge A B C D
|
||||
# s3_dense merge A B C D --dense (presence)
|
||||
# Scenario 2 (merge A B -> AB, then merge AB C D: multi-genome source,
|
||||
# multi-layer destination) is NOT included: the current code cannot
|
||||
# re-merge a merge output — its matrices are packed (sparse or dense)
|
||||
# at finalize, and `MatrixBuilder::resume` expects the columnar
|
||||
# `meta.json` layout (ENOENT). Known bug, to be fixed separately; see
|
||||
# DevDocMD/implementation/merge_mmap_audit.md, "Known issue".
|
||||
# 5. dumps every result with `obikmer dump --debug` (partition + layer
|
||||
# columns included, so a k-mer moving between layers is caught), keeps
|
||||
# the CSV header first and sorts the rows (LC_ALL=C)
|
||||
# 6. --record DIR writes the sorted dumps to DIR;
|
||||
# --check DIR regenerates them and diffs against DIR
|
||||
#
|
||||
# Usage:
|
||||
# scripts/smoke_test_merge.sh (--record DIR | --check DIR) [-p PARTITIONS] [--keep]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
K=21
|
||||
M=9
|
||||
PARTITIONS=4
|
||||
KEEP=0
|
||||
MODE=""
|
||||
DIR=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--record) MODE=record; DIR="$2"; shift 2 ;;
|
||||
--check) MODE=check; DIR="$2"; shift 2 ;;
|
||||
-p) PARTITIONS="$2"; shift 2 ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$MODE" ] || { echo "usage: $0 (--record DIR | --check DIR) [-p N] [--keep]" >&2; exit 2; }
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
case "$DIR" in /*) ;; *) DIR="$PWD/$DIR" ;; esac
|
||||
WORK="$(mktemp -d -t obikmer_merge_smoke.XXXXXX)"
|
||||
|
||||
cleanup() {
|
||||
if [ "$KEEP" -eq 1 ]; then echo "kept: $WORK"; else rm -rf "$WORK"; fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── 1. build ────────────────────────────────────────────────────────────────
|
||||
if ! (cd "$REPO_ROOT/src" && cargo build -q -p obikmer --bin obikmer) > "$WORK/build.log" 2>&1; then
|
||||
cat "$WORK/build.log" >&2
|
||||
fail "cargo build failed"
|
||||
fi
|
||||
BIN="$REPO_ROOT/src/target/debug/obikmer"
|
||||
|
||||
run() { # run LOGNAME CMD... — runs obikmer, dumps its log on failure
|
||||
local log="$WORK/$1.log"; shift
|
||||
if ! "$BIN" "$@" > "$log" 2>&1; then
|
||||
cat "$log" >&2
|
||||
fail "obikmer $1 exited non-zero"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 2. deterministic genomes with shared fragments ──────────────────────────
|
||||
python3 - "$WORK" <<'EOF'
|
||||
import random, sys
|
||||
out = sys.argv[1]
|
||||
rng = random.Random(20261117)
|
||||
def rnd(n): return "".join(rng.choice("ACGT") for _ in range(n))
|
||||
frag = {i: rnd(600) for i in range(5)} # shared fragments F0..F4
|
||||
motif = rnd(40) # tandem motif (counts >= 255)
|
||||
genomes = {
|
||||
"A": [rnd(1500), frag[0], rnd(300), frag[1]],
|
||||
"B": [frag[1], rnd(1200), frag[2], motif * 3],
|
||||
"C": [frag[0], rnd(800), frag[2], rnd(400), frag[3]],
|
||||
"D": [frag[3], rnd(1000), motif * 300, frag[4], rnd(200), frag[0][:250]],
|
||||
}
|
||||
for name, parts in genomes.items():
|
||||
with open(f"{out}/{name}.fasta", "w") as f:
|
||||
for j, p in enumerate(parts):
|
||||
f.write(f">{name}_{j}\n{p}\n")
|
||||
EOF
|
||||
|
||||
# ── 3. source indexes: built on --record, frozen in DIR/sources, reused on
|
||||
# --check. `obikmer index` is not byte-stable run to run (unitig
|
||||
# segmentation, hence unitigs.bin size, varies — measured 2026-11-17),
|
||||
# and the merge base is the source with the largest unitigs.bin
|
||||
# (`choose_base`), so re-indexing on every check would make the merge
|
||||
# base — and the column order — flip between A and C. Freezing the
|
||||
# sources makes this an oracle of `merge` alone. Copied (not linked)
|
||||
# into WORK, since merge's bootstrap hard-links from its base source.
|
||||
if [ "$MODE" = record ]; then
|
||||
for G in A B C D; do
|
||||
run "index_${G}_pres" index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
|
||||
--label "$G" -o "$WORK/$G.pres.idx" "$WORK/$G.fasta"
|
||||
run "index_${G}_cnt" index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
|
||||
--with-counts --label "$G" -o "$WORK/$G.cnt.idx" "$WORK/$G.fasta"
|
||||
done
|
||||
rm -rf "$DIR/sources"
|
||||
mkdir -p "$DIR/sources"
|
||||
cp -R "$WORK"/*.idx "$DIR/sources/"
|
||||
else
|
||||
[ -d "$DIR/sources" ] || fail "no frozen source indexes in $DIR/sources (run --record first)"
|
||||
cp -R "$DIR/sources"/*.idx "$WORK/"
|
||||
fi
|
||||
|
||||
# ── 4. merge scenarios ──────────────────────────────────────────────────────
|
||||
OUT="$WORK/out"
|
||||
mkdir -p "$OUT"
|
||||
dump() { # dump NAME INDEX — header first, rows sorted
|
||||
local raw="$WORK/$1.dump.csv"
|
||||
if ! "$BIN" dump --debug "$2" > "$raw" 2> "$WORK/dump_$1.log"; then
|
||||
cat "$WORK/dump_$1.log" >&2
|
||||
fail "obikmer dump $1 exited non-zero"
|
||||
fi
|
||||
{ head -n 1 "$raw"; tail -n +2 "$raw" | LC_ALL=C sort; } > "$OUT/$1.csv"
|
||||
}
|
||||
|
||||
for T in pres cnt; do
|
||||
run "s1_$T" merge -o "$WORK/s1_$T.idx" \
|
||||
"$WORK/A.$T.idx" "$WORK/B.$T.idx" "$WORK/C.$T.idx" "$WORK/D.$T.idx"
|
||||
dump "s1_$T" "$WORK/s1_$T.idx"
|
||||
done
|
||||
|
||||
run s3_dense merge --dense -o "$WORK/s3_dense.idx" \
|
||||
"$WORK/A.pres.idx" "$WORK/B.pres.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx"
|
||||
dump s3_dense "$WORK/s3_dense.idx"
|
||||
|
||||
# sanity: no dump may be empty (header + at least one row)
|
||||
for f in "$OUT"/*.csv; do
|
||||
[ "$(wc -l < "$f")" -gt 1 ] || fail "empty dump: $(basename "$f")"
|
||||
done
|
||||
|
||||
# ── 5. record or check ──────────────────────────────────────────────────────
|
||||
if [ "$MODE" = record ]; then
|
||||
mkdir -p "$DIR"
|
||||
rm -f "$DIR"/*.csv
|
||||
cp "$OUT"/*.csv "$DIR"/
|
||||
echo "RECORDED: $(ls "$OUT" | wc -l | tr -d ' ') dumps → $DIR"
|
||||
wc -l "$DIR"/*.csv
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ -d "$DIR" ] || fail "reference directory not found: $DIR"
|
||||
STATUS=0
|
||||
for ref in "$DIR"/*.csv; do
|
||||
name="$(basename "$ref")"
|
||||
if [ ! -f "$OUT/$name" ]; then
|
||||
echo "FAIL $name (not regenerated)"; STATUS=1
|
||||
elif diff -q "$ref" "$OUT/$name" > /dev/null; then
|
||||
echo "ok $name"
|
||||
else
|
||||
echo "FAIL $name"; { diff "$ref" "$OUT/$name" || true; } | head -20; STATUS=1
|
||||
fi
|
||||
done
|
||||
for new in "$OUT"/*.csv; do
|
||||
[ -f "$DIR/$(basename "$new")" ] || { echo "FAIL $(basename "$new") (no reference)"; STATUS=1; }
|
||||
done
|
||||
|
||||
if [ "$STATUS" -eq 0 ]; then
|
||||
echo "PASS: all merge dumps identical to $DIR"
|
||||
else
|
||||
echo "FAIL: merge dumps differ from $DIR" >&2
|
||||
fi
|
||||
exit "$STATUS"
|
||||
Reference in new issue
Block a user