#!/usr/bin/env bash # smoke_test_merge.sh — end-to-end golden-output oracle for `obikmer merge` # # Non-regression oracle for refactors of obikmerge (see # DevDocMD/implementation/merge_mmap_audit.md). Like smoke_test_index.sh it # drives the real CLI, never the library APIs. # # What it does: # 1. builds `obikmer` (debug) # 2. generates (on --record) 4 small deterministic FASTA files A, B, C, D sharing # fragments, so that merge both hits k-mers already present in the # destination ("existing layer" path) and creates new ones ("new layer" # path). D carries a short motif repeated > 255 times so the count track # exercises the compact-int overflow (counts >= 255). # 3. on --record, indexes each genome twice (presence, --with-counts) and # freezes the indexes in DIR/sources; --check reuses them (see below), # with -p PARTITIONS partitions # 4. runs the merge scenarios: # s1_presence / s1_count merge A B C D # s3_dense merge A B C D --dense (presence) # Scenario 2 (merge A B -> AB, then merge AB C D: multi-genome source, # multi-layer destination) is NOT included: the current code cannot # re-merge a merge output — its matrices are packed (sparse or dense) # at finalize, and `MatrixBuilder::resume` expects the columnar # `meta.json` layout (ENOENT). Known bug, to be fixed separately; see # DevDocMD/implementation/merge_mmap_audit.md, "Known issue". # 5. dumps every result with `obikmer dump --debug` (partition + layer # columns included, so a k-mer moving between layers is caught), keeps # the CSV header first and sorts the rows (LC_ALL=C) # 6. --record DIR writes the sorted dumps to DIR; # --check DIR regenerates them and diffs against DIR # # Usage: # scripts/smoke_test_merge.sh (--record DIR | --check DIR) [-p PARTITIONS] [--keep] set -euo pipefail K=21 M=9 PARTITIONS=4 KEEP=0 MODE="" DIR="" while [ $# -gt 0 ]; do case "$1" in --record) MODE=record; DIR="$2"; shift 2 ;; --check) MODE=check; DIR="$2"; shift 2 ;; -p) PARTITIONS="$2"; shift 2 ;; --keep) KEEP=1; shift ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac done [ -n "$MODE" ] || { echo "usage: $0 (--record DIR | --check DIR) [-p N] [--keep]" >&2; exit 2; } REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" case "$DIR" in /*) ;; *) DIR="$PWD/$DIR" ;; esac WORK="$(mktemp -d -t obikmer_merge_smoke.XXXXXX)" cleanup() { if [ "$KEEP" -eq 1 ]; then echo "kept: $WORK"; else rm -rf "$WORK"; fi } trap cleanup EXIT fail() { echo "FAIL: $1" >&2 exit 1 } # ── 1. build ──────────────────────────────────────────────────────────────── if ! (cd "$REPO_ROOT/src" && cargo build -q -p obikmer --bin obikmer) > "$WORK/build.log" 2>&1; then cat "$WORK/build.log" >&2 fail "cargo build failed" fi BIN="$REPO_ROOT/src/target/debug/obikmer" run() { # run LOGNAME CMD... — runs obikmer, dumps its log on failure local log="$WORK/$1.log"; shift if ! "$BIN" "$@" > "$log" 2>&1; then cat "$log" >&2 fail "obikmer $1 exited non-zero" fi } # ── 2. deterministic genomes with shared fragments ────────────────────────── python3 - "$WORK" <<'EOF' import random, sys out = sys.argv[1] rng = random.Random(20261117) def rnd(n): return "".join(rng.choice("ACGT") for _ in range(n)) frag = {i: rnd(600) for i in range(5)} # shared fragments F0..F4 motif = rnd(40) # tandem motif (counts >= 255) genomes = { "A": [rnd(1500), frag[0], rnd(300), frag[1]], "B": [frag[1], rnd(1200), frag[2], motif * 3], "C": [frag[0], rnd(800), frag[2], rnd(400), frag[3]], "D": [frag[3], rnd(1000), motif * 300, frag[4], rnd(200), frag[0][:250]], } for name, parts in genomes.items(): with open(f"{out}/{name}.fasta", "w") as f: for j, p in enumerate(parts): f.write(f">{name}_{j}\n{p}\n") EOF # ── 3. source indexes: built on --record, frozen in DIR/sources, reused on # --check. `obikmer index` is not byte-stable run to run (unitig # segmentation, hence unitigs.bin size, varies — measured 2026-11-17), # and the merge base is the source with the largest unitigs.bin # (`choose_base`), so re-indexing on every check would make the merge # base — and the column order — flip between A and C. Freezing the # sources makes this an oracle of `merge` alone. Copied (not linked) # into WORK, since merge's bootstrap hard-links from its base source. if [ "$MODE" = record ]; then for G in A B C D; do run "index_${G}_pres" index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \ --label "$G" -o "$WORK/$G.pres.idx" "$WORK/$G.fasta" run "index_${G}_cnt" index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \ --with-counts --label "$G" -o "$WORK/$G.cnt.idx" "$WORK/$G.fasta" done rm -rf "$DIR/sources" mkdir -p "$DIR/sources" cp -R "$WORK"/*.idx "$DIR/sources/" else [ -d "$DIR/sources" ] || fail "no frozen source indexes in $DIR/sources (run --record first)" cp -R "$DIR/sources"/*.idx "$WORK/" fi # ── 4. merge scenarios ────────────────────────────────────────────────────── OUT="$WORK/out" mkdir -p "$OUT" dump() { # dump NAME INDEX — header first, rows sorted local raw="$WORK/$1.dump.csv" if ! "$BIN" dump --debug "$2" > "$raw" 2> "$WORK/dump_$1.log"; then cat "$WORK/dump_$1.log" >&2 fail "obikmer dump $1 exited non-zero" fi { head -n 1 "$raw"; tail -n +2 "$raw" | LC_ALL=C sort; } > "$OUT/$1.csv" } for T in pres cnt; do run "s1_$T" merge -o "$WORK/s1_$T.idx" \ "$WORK/A.$T.idx" "$WORK/B.$T.idx" "$WORK/C.$T.idx" "$WORK/D.$T.idx" dump "s1_$T" "$WORK/s1_$T.idx" done run s3_dense merge --dense -o "$WORK/s3_dense.idx" \ "$WORK/A.pres.idx" "$WORK/B.pres.idx" "$WORK/C.pres.idx" "$WORK/D.pres.idx" dump s3_dense "$WORK/s3_dense.idx" # sanity: no dump may be empty (header + at least one row) for f in "$OUT"/*.csv; do [ "$(wc -l < "$f")" -gt 1 ] || fail "empty dump: $(basename "$f")" done # ── 5. record or check ────────────────────────────────────────────────────── if [ "$MODE" = record ]; then mkdir -p "$DIR" rm -f "$DIR"/*.csv cp "$OUT"/*.csv "$DIR"/ echo "RECORDED: $(ls "$OUT" | wc -l | tr -d ' ') dumps → $DIR" wc -l "$DIR"/*.csv exit 0 fi [ -d "$DIR" ] || fail "reference directory not found: $DIR" STATUS=0 for ref in "$DIR"/*.csv; do name="$(basename "$ref")" if [ ! -f "$OUT/$name" ]; then echo "FAIL $name (not regenerated)"; STATUS=1 elif diff -q "$ref" "$OUT/$name" > /dev/null; then echo "ok $name" else echo "FAIL $name"; { diff "$ref" "$OUT/$name" || true; } | head -20; STATUS=1 fi done for new in "$OUT"/*.csv; do [ -f "$DIR/$(basename "$new")" ] || { echo "FAIL $(basename "$new") (no reference)"; STATUS=1; } done if [ "$STATUS" -eq 0 ]; then echo "PASS: all merge dumps identical to $DIR" else echo "FAIL: merge dumps differ from $DIR" >&2 fi exit "$STATUS"