diff --git a/scripts/docker_debug/compat_ptrhash.sh b/scripts/docker_debug/compat_ptrhash.sh new file mode 100755 index 00000000..428cf277 --- /dev/null +++ b/scripts/docker_debug/compat_ptrhash.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Index compatibility test between two obikmer binaries that differ only by +# their ptr_hash version (A = baseline, B = candidate). +# +# Usage: compat_ptrhash.sh VOL_A VOL_B OUT +# VOL_A / VOL_B docker volumes holding a musl release build (CARGO_TARGET_DIR) +# OUT host directory receiving report.txt +# +# What it checks (everything compared by md5 of dump/query output): +# T0 on-disk bytes of an index built by A vs by B (informational) +# T1 A-built single-genome indexes read by A and by B +# T2 merges: A merging A-built sources (reference), B merging A-built +# sources, B merging B-built sources; each result dumped by A and by B +# T3 queries (hits and misses) on indexes read by A and by B +# T4 incremental merge into an existing A-built merged index, by A and B +set -u +VOL_A="$1"; VOL_B="$2"; OUT="$3" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" +mkdir -p "$OUT" + +docker run --rm --platform linux/amd64 \ + -v "$VOL_A:/va:ro" -v "$VOL_B:/vb:ro" \ + -v "$REPO/benchmark/run/genomes:/genomes_ro:ro" -v "$OUT:/out" \ + obikmer-debug:linux-amd64 bash -c ' +set -u +R=x86_64-unknown-linux-musl/release/obikmer +mkdir -p /tmp/bin && cp /va/$R /tmp/bin/A && cp /vb/$R /tmp/bin/B +A=/tmp/bin/A; B=/tmp/bin/B +W=/tmp/w; rm -rf $W; mkdir -p $W/ia $W/ib +exec > >(tee /out/report.txt) 2>&1 +echo "A sha: $(sha256sum $A | cut -c1-16) B sha: $(sha256sum $B | cut -c1-16)" +mapfile -t G < <(ls -S /genomes_ro/*.fna.gz | sed -n 8,12p) +echo "genomes: ${#G[@]}"; printf " %s\n" "${G[@]##*/}" +mkdir -p $W/fa; n=0; for gz in "${G[@]}"; do n=$((n+1)); g=$W/fa/L$n.fasta; zcat "$gz" > "$g" + $A index -k 21 -m 9 --theta 0 -p 256 --label L$n -o $W/ia/L$n.idx "$g" 2>$W/ia/L$n.err >/dev/null || { echo "A index failed L$n"; tail -2 $W/ia/L$n.err; } + $B index -k 21 -m 9 --theta 0 -p 256 --label L$n -o $W/ib/L$n.idx "$g" 2>$W/ib/L$n.err >/dev/null || { echo "B index failed L$n"; tail -2 $W/ib/L$n.err; } +done + +echo; echo "== T0 on-disk bytes, index built by A vs by B" +nd=$(diff -rq $W/ia $W/ib | wc -l); nt=$(find $W/ia -type f | wc -l) +echo "files differing: $nd / $nt" +diff -rq $W/ia $W/ib | sed "s#$W/##g" | awk "{print \$2}" | sed -E "s#/partitions/.*/#/…/#" | sort | uniq -c | sort -rn | head -8 + +h() { md5sum | cut -c1-12; } +echo; echo "== T1 A-built single-genome indexes: dump by A vs by B" +for i in 1 2 3 4 5; do + a=$($A dump $W/ia/L$i.idx 2>/dev/null | h); b=$($B dump $W/ia/L$i.idx 2>/dev/null | h) + echo "L$i A:$a B:$b $([ "$a" = "$b" ] && echo OK || echo DIFF)" +done + +cp_src() { rm -rf $2; mkdir -p $2; for i in $1; do cp -r $3/L$i.idx $2/; done; } +echo; echo "== T2 merges of 4 genomes (presence)" +cp_src "1 2 3 4" $W/s_aa $W/ia; cp_src "1 2 3 4" $W/s_ba $W/ia; cp_src "1 2 3 4" $W/s_bb $W/ib +$A merge --output $W/M_AA --force --force-presence $W/s_aa/L*.idx 2>$W/m_aa.err; echo "M_AA rc=$?" +$B merge --output $W/M_BA --force --force-presence $W/s_ba/L*.idx 2>$W/m_ba.err; echo "M_BA rc=$? (B merging A-built sources)" +$B merge --output $W/M_BB --force --force-presence $W/s_bb/L*.idx 2>$W/m_bb.err; echo "M_BB rc=$? (B merging B-built sources)" +for X in M_AA M_BA M_BB; do for T in A B; do + eval "BIN=\$$T"; $BIN dump $W/$X 2>/dev/null > $W/d_${X}_$T.csv + echo "dump $X by $T: $(wc -l < $W/d_${X}_$T.csv) lines raw:$(h < $W/d_${X}_$T.csv) sorted:$(LC_ALL=C sort -S 2G $W/d_${X}_$T.csv | h)" +done; done + +echo; echo "== T3 queries" +head -c 60000 $W/fa/L1.fasta > $W/q.fa +python3 - <> $W/q.fa +import random +random.seed(1) +print(">rand") +print("".join(random.choice("ACGT") for _ in range(5000))) +PY +for X in ia/L1.idx M_AA M_BA M_BB; do for T in A B; do + eval "BIN=\$$T"; echo "query $X by $T: $($BIN query $W/$X $W/q.fa 2>/dev/null | h)" +done; done + +echo; echo "== T4 incremental merge (existing A-built merged index + a 5th genome)" +for T in A B; do + eval "BIN=\$$T"; rm -rf $W/inc_$T $W/dst_$T; cp -r $W/M_AA $W/dst_$T; cp -r $W/ia/L5.idx $W/l5_$T.idx + $BIN merge --output $W/inc_$T --force --force-presence $W/dst_$T $W/l5_$T.idx 2>$W/inc_$T.err; echo "inc by $T rc=$?" + $BIN dump $W/inc_$T 2>/dev/null | h +done +echo; echo done +' diff --git a/scripts/docker_debug/compat_ptrhash2.sh b/scripts/docker_debug/compat_ptrhash2.sh new file mode 100755 index 00000000..6bfa56e8 --- /dev/null +++ b/scripts/docker_debug/compat_ptrhash2.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Follow-up to compat_ptrhash.sh: determinism control (A built twice) and the +# incremental merge with its stderr shown. Usage: compat_ptrhash2.sh VOL_A VOL_B OUT +set -u +VOL_A="$1"; VOL_B="$2"; OUT="$3" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; REPO="$(cd "$HERE/../.." && pwd)" +mkdir -p "$OUT" +docker run --rm --platform linux/amd64 -v "$VOL_A:/va:ro" -v "$VOL_B:/vb:ro" \ + -v "$REPO/benchmark/run/genomes:/genomes_ro:ro" -v "$OUT:/out" obikmer-debug:linux-amd64 bash -c ' +set -u +R=x86_64-unknown-linux-musl/release/obikmer +mkdir -p /tmp/bin && cp /va/$R /tmp/bin/A && cp /vb/$R /tmp/bin/B; A=/tmp/bin/A; B=/tmp/bin/B +W=/tmp/w; rm -rf $W; mkdir -p $W/i1 $W/i2 $W/ib $W/fa +exec > >(tee /out/report2.txt) 2>&1 +mapfile -t G < <(ls -S /genomes_ro/*.fna.gz | sed -n 8,12p) +n=0; for gz in "${G[@]}"; do n=$((n+1)); zcat "$gz" > $W/fa/L$n.fasta; done +h() { md5sum | cut -c1-12; } +echo "== C1 determinism control: same binary A, same input, built twice" +$A index -k 21 -m 9 --theta 0 -p 256 --label L1 -o $W/i1/L1.idx $W/fa/L1.fasta >/dev/null 2>&1 +$A index -k 21 -m 9 --theta 0 -p 256 --label L1 -o $W/i2/L1.idx $W/fa/L1.fasta >/dev/null 2>&1 +$B index -k 21 -m 9 --theta 0 -p 256 --label L1 -o $W/ib/L1.idx $W/fa/L1.fasta >/dev/null 2>&1 +echo "A vs A: $(diff -rq $W/i1 $W/i2 | wc -l) / $(find $W/i1 -type f | wc -l) files differ" +echo "A vs B: $(diff -rq $W/i1 $W/ib | wc -l) / $(find $W/i1 -type f | wc -l) files differ" +for f in mphf.bin unitigs.bin evidence.bin unitigs.bin.idx; do + p=$(find $W/i1 -name $f | head -1); rel=${p#$W/i1/} + echo " $f AvsA: $(find $W/i1 -name $f | while read q; do cmp -s $q $W/i2/${q#$W/i1/} || echo x; done | wc -l)/256 differ AvsB: $(find $W/i1 -name $f | while read q; do cmp -s $q $W/ib/${q#$W/i1/} || echo x; done | wc -l)/256 differ" +done +echo; echo "== C2 incremental merge, stderr shown" +for i in 2 3 4 5; do $A index -k 21 -m 9 --theta 0 -p 256 --label L$i -o $W/i1/L$i.idx $W/fa/L$i.fasta >/dev/null 2>&1; done +mkdir -p $W/s; for i in 1 2 3; do cp -r $W/i1/L$i.idx $W/s/; done +$A merge --output $W/M3 --force --force-presence $W/s/L*.idx 2>/dev/null; echo "M3 rc=$?" +for T in A B; do + eval "BIN=\$$T"; rm -rf $W/inc_$T $W/dst_$T $W/l4_$T.idx; cp -r $W/M3 $W/dst_$T; cp -r $W/i1/L4.idx $W/l4_$T.idx + $BIN merge --output $W/inc_$T --force --force-presence $W/dst_$T $W/l4_$T.idx 2>$W/inc_$T.err; rc=$? + echo "inc by $T rc=$rc"; sed "s/\x1b\[[0-9;]*m//g" $W/inc_$T.err | tail -3 + [ $rc = 0 ] && echo " dump sorted: $($BIN dump $W/inc_$T 2>/dev/null | LC_ALL=C sort -S 2G | h)" +done +echo done +' diff --git a/src/Cargo.lock b/src/Cargo.lock index 13668f52..030c244c 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -29,6 +29,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -190,6 +196,16 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-pseudorand" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" +dependencies = [ + "chiapos-chacha8", + "nanorand", +] + [[package]] name = "bpaf" version = "0.9.27" @@ -309,6 +325,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chiapos-chacha8" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" +dependencies = [ + "rayon", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -475,6 +500,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "criterion2" version = "3.0.4" @@ -850,6 +911,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "http" version = "1.4.2" @@ -965,12 +1032,32 @@ dependencies = [ "cfb", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.15.0" @@ -1211,6 +1298,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "nanorand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" + [[package]] name = "ndarray" version = "0.17.2" @@ -1792,6 +1885,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -1865,7 +1986,7 @@ dependencies = [ "epserde-derive", "fastrand", "fxhash", - "itertools", + "itertools 0.15.0", "log", "mem_dbg", "prefetch-index", @@ -2025,13 +2146,17 @@ checksum = "e6724c78e033e1c4155b4d3f76593d09ad384739c6986c1addc2bd2f55b1aefe" [[package]] name = "rdst" -version = "0.20.11" +version = "0.20.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cf18dd479f856ca75267591c062e22b50e2db791157495bc2d533558de7061d" +checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" dependencies = [ "arbitrary-chunks", + "block-pseudorand", + "criterion", "partition", "rayon", + "tikv-jemallocator", + "voracious_radix_sort", ] [[package]] @@ -2530,9 +2655,9 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +version = "0.5.4+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" dependencies = [ "cc", "libc", @@ -2540,14 +2665,24 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.7.0" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" dependencies = [ "libc", "tikv-jemalloc-sys", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.13.2" @@ -2735,6 +2870,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -2805,6 +2949,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index a430fdd6..7e24067c 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -41,7 +41,7 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } # merge_mmap_audit.md). Static musl builds use jemalloc instead; glibc and # macOS builds keep the system allocator. [target.'cfg(target_env = "musl")'.dependencies] -tikv-jemallocator = "0.7.0" +tikv-jemallocator = "0.5.4" [features] default = ["numa"]