Files
obikmer/scripts/smoke_test_merge.sh
T
Eric Coissac 7f829ea557 docs: clarify merge re-merge behavior and input file handling
Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content.

- refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check
- fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations
- feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion
- test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior
- test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes
- chore: remove obsolete cargo build environment variables from `run.sh` script
2026-11-19 07:10:58 +01:00

218 lines
9.0 KiB
Bash
Executable File

#!/usr/bin/env bash
# smoke_test_merge.sh — end-to-end golden-output oracle for `obikmer merge`
#
# Non-regression oracle for refactors of obikmerge (see
# DevDocMD/implementation/merge_mmap_audit.md). Like smoke_test_index.sh it
# drives the real CLI, never the library APIs.
#
# What it does:
# 1. builds `obikmer` (debug)
# 2. generates (on --record) 4 small deterministic FASTA files A, B, C, D sharing
# fragments, so that merge both hits k-mers already present in the
# destination ("existing layer" path) and creates new ones ("new layer"
# path). D carries a short motif repeated > 255 times so the count track
# exercises the compact-int overflow (counts >= 255).
# 3. on --record, indexes each genome twice (presence, --with-counts) and
# freezes the indexes in DIR/sources; --check reuses them (see below),
# with -p PARTITIONS partitions
# 4. runs the merge scenarios:
# s1_presence / s1_count merge A B C D
# s3_dense merge A B C D --dense (presence)
# Scenario 2 (merge A B -> AB, then merge AB C D: a packed merge output
# re-merged as a *source*) is checked relatively, not against the
# golden dumps: its table (k-mer x genome label) must equal s1_presence's,
# whatever the column order and layer placement. A packed index cannot
# be a merge *destination* (its matrices are no longer column-by-column,
# see `can_be_destination` in obikmerge); `choose_base` therefore picks
# an unpacked source, and merging only packed indexes is refused with a
# clear error, which this script also checks.
# 5. dumps every result with `obikmer dump --debug` (partition + layer
# columns included, so a k-mer moving between layers is caught), keeps
# the CSV header first and sorts the rows (LC_ALL=C)
# 6. --record DIR writes the sorted dumps to DIR;
# --check DIR regenerates them and diffs against DIR
#
# Usage:
# scripts/smoke_test_merge.sh (--record DIR | --check DIR) [-p PARTITIONS] [--keep]
set -euo pipefail
K=21
M=9
PARTITIONS=4
KEEP=0
MODE=""
DIR=""
while [ $# -gt 0 ]; do
case "$1" in
--record) MODE=record; DIR="$2"; shift 2 ;;
--check) MODE=check; DIR="$2"; shift 2 ;;
-p) PARTITIONS="$2"; shift 2 ;;
--keep) KEEP=1; shift ;;
*) echo "unknown argument: $1" >&2; exit 2 ;;
esac
done
[ -n "$MODE" ] || { echo "usage: $0 (--record DIR | --check DIR) [-p N] [--keep]" >&2; exit 2; }
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
case "$DIR" in /*) ;; *) DIR="$PWD/$DIR" ;; esac
WORK="$(mktemp -d -t obikmer_merge_smoke.XXXXXX)"
cleanup() {
if [ "$KEEP" -eq 1 ]; then echo "kept: $WORK"; else rm -rf "$WORK"; fi
}
trap cleanup EXIT
fail() {
echo "FAIL: $1" >&2
exit 1
}
# ── 1. build ────────────────────────────────────────────────────────────────
if ! (cd "$REPO_ROOT/src" && cargo build -q -p obikmer --bin obikmer) > "$WORK/build.log" 2>&1; then
cat "$WORK/build.log" >&2
fail "cargo build failed"
fi
BIN="$REPO_ROOT/src/target/debug/obikmer"
run() { # run LOGNAME CMD... — runs obikmer, dumps its log on failure
local log="$WORK/$1.log"; shift
if ! "$BIN" "$@" > "$log" 2>&1; then
cat "$log" >&2
fail "obikmer $1 exited non-zero"
fi
}
# ── 2. deterministic genomes with shared fragments ──────────────────────────
python3 - "$WORK" <<'EOF'
import random, sys
out = sys.argv[1]
rng = random.Random(20261117)
def rnd(n): return "".join(rng.choice("ACGT") for _ in range(n))
frag = {i: rnd(600) for i in range(5)} # shared fragments F0..F4
motif = rnd(40) # tandem motif (counts >= 255)
genomes = {
"A": [rnd(1500), frag[0], rnd(300), frag[1]],
"B": [frag[1], rnd(1200), frag[2], motif * 3],
"C": [frag[0], rnd(800), frag[2], rnd(400), frag[3]],
"D": [frag[3], rnd(1000), motif * 300, frag[4], rnd(200), frag[0][:250]],
}
for name, parts in genomes.items():
with open(f"{out}/{name}.fasta", "w") as f:
for j, p in enumerate(parts):
f.write(f">{name}_{j}\n{p}\n")
EOF
# ── 3. source indexes: built on --record, frozen in DIR/sources, reused on
# --check. `obikmer index` is not byte-stable run to run (unitig
# segmentation, hence unitigs.bin size, varies — measured 2026-11-17),
# and the merge base is the source with the largest unitigs.bin
# (`choose_base`), so re-indexing on every check would make the merge
# base — and the column order — flip between A and C. Freezing the
# sources makes this an oracle of `merge` alone. Copied (not linked)
# into WORK, since merge's bootstrap hard-links from its base source.
if [ "$MODE" = record ]; then
for G in A B C D; do
run "index_${G}_pres" index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
--label "$G" -o "$WORK/$G.pres.idx" "$WORK/$G.fasta"
run "index_${G}_cnt" index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
--with-counts --label "$G" -o "$WORK/$G.cnt.idx" "$WORK/$G.fasta"
done
rm -rf "$DIR/sources"
mkdir -p "$DIR/sources"
cp -R "$WORK"/*.idx "$DIR/sources/"
else
[ -d "$DIR/sources" ] || fail "no frozen source indexes in $DIR/sources (run --record first)"
cp -R "$DIR/sources"/*.idx "$WORK/"
fi
# ── 4. merge scenarios ──────────────────────────────────────────────────────
OUT="$WORK/out"
mkdir -p "$OUT"
dump() { # dump NAME INDEX — header first, rows sorted
local raw="$WORK/$1.dump.csv"
if ! "$BIN" dump --debug "$2" > "$raw" 2> "$WORK/dump_$1.log"; then
cat "$WORK/dump_$1.log" >&2
fail "obikmer dump $1 exited non-zero"
fi
{ head -n 1 "$raw"; tail -n +2 "$raw" | LC_ALL=C sort; } > "$OUT/$1.csv"
}
for T in pres cnt; do
run "s1_$T" merge -o "$WORK/s1_$T.idx" \
"$WORK/A.$T.idx" "$WORK/B.$T.idx" "$WORK/C.$T.idx" "$WORK/D.$T.idx"
dump "s1_$T" "$WORK/s1_$T.idx"
done
run s3_dense merge --dense -o "$WORK/s3_dense.idx" \
"$WORK/A.pres.idx" "$WORK/B.pres.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx"
dump s3_dense "$WORK/s3_dense.idx"
# scenario 2: re-merge a packed merge output as a source (checked against s1)
run s2_ab merge -o "$WORK/s2_ab.idx" "$WORK/A.pres.idx" "$WORK/B.pres.idx"
run s2 merge -o "$WORK/s2.idx" "$WORK/s2_ab.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx"
for N in s1_pres s2; do
"$BIN" dump "$WORK/$N.idx" > "$WORK/plain_$N.csv" 2> "$WORK/dump_plain_$N.log" \
|| { cat "$WORK/dump_plain_$N.log" >&2; fail "obikmer dump $N exited non-zero"; }
done
cat > "$WORK/compare_tables.py" <<'PY'
import csv, sys
def table(path):
rows = list(csv.reader(open(path)))
head = rows[0]
return {r[0]: {head[i]: r[i] for i in range(1, len(head))} for r in rows[1:]}
a, b = table(sys.argv[1]), table(sys.argv[2])
if a != b:
print(f"k-mers: {len(a)} vs {len(b)}; only in flat: {len(set(a) - set(b))}; "
f"only in re-merge: {len(set(b) - set(a))}", file=sys.stderr)
sys.exit(1)
PY
python3 "$WORK/compare_tables.py" "$WORK/plain_s1_pres.csv" "$WORK/plain_s2.csv" \
|| fail "scenario 2 (re-merge of a packed output) differs from the flat merge"
echo "ok s2 (re-merge of a packed output) equals the flat merge"
if "$BIN" merge -o "$WORK/s2_bad.idx" "$WORK/s2_ab.idx" "$WORK/s1_pres.idx" > "$WORK/s2_bad.log" 2>&1; then
fail "merging only packed indexes should be refused"
fi
grep -q "none of the sources can be the destination" "$WORK/s2_bad.log" \
|| { cat "$WORK/s2_bad.log" >&2; fail "refusal of an all-packed merge lacks the expected message"; }
echo "ok merge of only packed indexes is refused with a clear message"
# sanity: no dump may be empty (header + at least one row)
for f in "$OUT"/*.csv; do
[ "$(wc -l < "$f")" -gt 1 ] || fail "empty dump: $(basename "$f")"
done
# ── 5. record or check ──────────────────────────────────────────────────────
if [ "$MODE" = record ]; then
mkdir -p "$DIR"
rm -f "$DIR"/*.csv
cp "$OUT"/*.csv "$DIR"/
echo "RECORDED: $(ls "$OUT" | wc -l | tr -d ' ') dumps → $DIR"
wc -l "$DIR"/*.csv
exit 0
fi
[ -d "$DIR" ] || fail "reference directory not found: $DIR"
STATUS=0
for ref in "$DIR"/*.csv; do
name="$(basename "$ref")"
if [ ! -f "$OUT/$name" ]; then
echo "FAIL $name (not regenerated)"; STATUS=1
elif diff -q "$ref" "$OUT/$name" > /dev/null; then
echo "ok $name"
else
echo "FAIL $name"; { diff "$ref" "$OUT/$name" || true; } | head -20; STATUS=1
fi
done
for new in "$OUT"/*.csv; do
[ -f "$DIR/$(basename "$new")" ] || { echo "FAIL $(basename "$new") (no reference)"; STATUS=1; }
done
if [ "$STATUS" -eq 0 ]; then
echo "PASS: all merge dumps identical to $DIR"
else
echo "FAIL: merge dumps differ from $DIR" >&2
fi
exit "$STATUS"