Push vpnykuuvnrnx #82

Merged
coissac merged 11 commits from push-vpnykuuvnrnx into main 2026-11-19 06:17:44 +00:00
4 changed files with 283 additions and 8 deletions
Showing only changes of commit 07c0a54e04 - Show all commits

No files matched your search

+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
# Index compatibility test between two obikmer binaries that differ only by
# their ptr_hash version (A = baseline, B = candidate).
#
# Usage: compat_ptrhash.sh VOL_A VOL_B OUT
# VOL_A / VOL_B docker volumes holding a musl release build (CARGO_TARGET_DIR)
# OUT host directory receiving report.txt
#
# What it checks (everything compared by md5 of dump/query output):
# T0 on-disk bytes of an index built by A vs by B (informational)
# T1 A-built single-genome indexes read by A and by B
# T2 merges: A merging A-built sources (reference), B merging A-built
# sources, B merging B-built sources; each result dumped by A and by B
# T3 queries (hits and misses) on indexes read by A and by B
# T4 incremental merge into an existing A-built merged index, by A and B
set -u
VOL_A="$1"; VOL_B="$2"; OUT="$3"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
mkdir -p "$OUT"
docker run --rm --platform linux/amd64 \
-v "$VOL_A:/va:ro" -v "$VOL_B:/vb:ro" \
-v "$REPO/benchmark/run/genomes:/genomes_ro:ro" -v "$OUT:/out" \
obikmer-debug:linux-amd64 bash -c '
set -u
R=x86_64-unknown-linux-musl/release/obikmer
mkdir -p /tmp/bin && cp /va/$R /tmp/bin/A && cp /vb/$R /tmp/bin/B
A=/tmp/bin/A; B=/tmp/bin/B
W=/tmp/w; rm -rf $W; mkdir -p $W/ia $W/ib
exec > >(tee /out/report.txt) 2>&1
echo "A sha: $(sha256sum $A | cut -c1-16) B sha: $(sha256sum $B | cut -c1-16)"
mapfile -t G < <(ls -S /genomes_ro/*.fna.gz | sed -n 8,12p)
echo "genomes: ${#G[@]}"; printf " %s\n" "${G[@]##*/}"
mkdir -p $W/fa; n=0; for gz in "${G[@]}"; do n=$((n+1)); g=$W/fa/L$n.fasta; zcat "$gz" > "$g"
$A index -k 21 -m 9 --theta 0 -p 256 --label L$n -o $W/ia/L$n.idx "$g" 2>$W/ia/L$n.err >/dev/null || { echo "A index failed L$n"; tail -2 $W/ia/L$n.err; }
$B index -k 21 -m 9 --theta 0 -p 256 --label L$n -o $W/ib/L$n.idx "$g" 2>$W/ib/L$n.err >/dev/null || { echo "B index failed L$n"; tail -2 $W/ib/L$n.err; }
done
echo; echo "== T0 on-disk bytes, index built by A vs by B"
nd=$(diff -rq $W/ia $W/ib | wc -l); nt=$(find $W/ia -type f | wc -l)
echo "files differing: $nd / $nt"
diff -rq $W/ia $W/ib | sed "s#$W/##g" | awk "{print \$2}" | sed -E "s#/partitions/.*/#/…/#" | sort | uniq -c | sort -rn | head -8
h() { md5sum | cut -c1-12; }
echo; echo "== T1 A-built single-genome indexes: dump by A vs by B"
for i in 1 2 3 4 5; do
a=$($A dump $W/ia/L$i.idx 2>/dev/null | h); b=$($B dump $W/ia/L$i.idx 2>/dev/null | h)
echo "L$i A:$a B:$b $([ "$a" = "$b" ] && echo OK || echo DIFF)"
done
cp_src() { rm -rf $2; mkdir -p $2; for i in $1; do cp -r $3/L$i.idx $2/; done; }
echo; echo "== T2 merges of 4 genomes (presence)"
cp_src "1 2 3 4" $W/s_aa $W/ia; cp_src "1 2 3 4" $W/s_ba $W/ia; cp_src "1 2 3 4" $W/s_bb $W/ib
$A merge --output $W/M_AA --force --force-presence $W/s_aa/L*.idx 2>$W/m_aa.err; echo "M_AA rc=$?"
$B merge --output $W/M_BA --force --force-presence $W/s_ba/L*.idx 2>$W/m_ba.err; echo "M_BA rc=$? (B merging A-built sources)"
$B merge --output $W/M_BB --force --force-presence $W/s_bb/L*.idx 2>$W/m_bb.err; echo "M_BB rc=$? (B merging B-built sources)"
for X in M_AA M_BA M_BB; do for T in A B; do
eval "BIN=\$$T"; $BIN dump $W/$X 2>/dev/null > $W/d_${X}_$T.csv
echo "dump $X by $T: $(wc -l < $W/d_${X}_$T.csv) lines raw:$(h < $W/d_${X}_$T.csv) sorted:$(LC_ALL=C sort -S 2G $W/d_${X}_$T.csv | h)"
done; done
echo; echo "== T3 queries"
head -c 60000 $W/fa/L1.fasta > $W/q.fa
python3 - <<PY >> $W/q.fa
import random
random.seed(1)
print(">rand")
print("".join(random.choice("ACGT") for _ in range(5000)))
PY
for X in ia/L1.idx M_AA M_BA M_BB; do for T in A B; do
eval "BIN=\$$T"; echo "query $X by $T: $($BIN query $W/$X $W/q.fa 2>/dev/null | h)"
done; done
echo; echo "== T4 incremental merge (existing A-built merged index + a 5th genome)"
for T in A B; do
eval "BIN=\$$T"; rm -rf $W/inc_$T $W/dst_$T; cp -r $W/M_AA $W/dst_$T; cp -r $W/ia/L5.idx $W/l5_$T.idx
$BIN merge --output $W/inc_$T --force --force-presence $W/dst_$T $W/l5_$T.idx 2>$W/inc_$T.err; echo "inc by $T rc=$?"
$BIN dump $W/inc_$T 2>/dev/null | h
done
echo; echo done
'
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# Follow-up to compat_ptrhash.sh: determinism control (A built twice) and the
# incremental merge with its stderr shown. Usage: compat_ptrhash2.sh VOL_A VOL_B OUT
set -u
VOL_A="$1"; VOL_B="$2"; OUT="$3"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; REPO="$(cd "$HERE/../.." && pwd)"
mkdir -p "$OUT"
docker run --rm --platform linux/amd64 -v "$VOL_A:/va:ro" -v "$VOL_B:/vb:ro" \
-v "$REPO/benchmark/run/genomes:/genomes_ro:ro" -v "$OUT:/out" obikmer-debug:linux-amd64 bash -c '
set -u
R=x86_64-unknown-linux-musl/release/obikmer
mkdir -p /tmp/bin && cp /va/$R /tmp/bin/A && cp /vb/$R /tmp/bin/B; A=/tmp/bin/A; B=/tmp/bin/B
W=/tmp/w; rm -rf $W; mkdir -p $W/i1 $W/i2 $W/ib $W/fa
exec > >(tee /out/report2.txt) 2>&1
mapfile -t G < <(ls -S /genomes_ro/*.fna.gz | sed -n 8,12p)
n=0; for gz in "${G[@]}"; do n=$((n+1)); zcat "$gz" > $W/fa/L$n.fasta; done
h() { md5sum | cut -c1-12; }
echo "== C1 determinism control: same binary A, same input, built twice"
$A index -k 21 -m 9 --theta 0 -p 256 --label L1 -o $W/i1/L1.idx $W/fa/L1.fasta >/dev/null 2>&1
$A index -k 21 -m 9 --theta 0 -p 256 --label L1 -o $W/i2/L1.idx $W/fa/L1.fasta >/dev/null 2>&1
$B index -k 21 -m 9 --theta 0 -p 256 --label L1 -o $W/ib/L1.idx $W/fa/L1.fasta >/dev/null 2>&1
echo "A vs A: $(diff -rq $W/i1 $W/i2 | wc -l) / $(find $W/i1 -type f | wc -l) files differ"
echo "A vs B: $(diff -rq $W/i1 $W/ib | wc -l) / $(find $W/i1 -type f | wc -l) files differ"
for f in mphf.bin unitigs.bin evidence.bin unitigs.bin.idx; do
p=$(find $W/i1 -name $f | head -1); rel=${p#$W/i1/}
echo " $f AvsA: $(find $W/i1 -name $f | while read q; do cmp -s $q $W/i2/${q#$W/i1/} || echo x; done | wc -l)/256 differ AvsB: $(find $W/i1 -name $f | while read q; do cmp -s $q $W/ib/${q#$W/i1/} || echo x; done | wc -l)/256 differ"
done
echo; echo "== C2 incremental merge, stderr shown"
for i in 2 3 4 5; do $A index -k 21 -m 9 --theta 0 -p 256 --label L$i -o $W/i1/L$i.idx $W/fa/L$i.fasta >/dev/null 2>&1; done
mkdir -p $W/s; for i in 1 2 3; do cp -r $W/i1/L$i.idx $W/s/; done
$A merge --output $W/M3 --force --force-presence $W/s/L*.idx 2>/dev/null; echo "M3 rc=$?"
for T in A B; do
eval "BIN=\$$T"; rm -rf $W/inc_$T $W/dst_$T $W/l4_$T.idx; cp -r $W/M3 $W/dst_$T; cp -r $W/i1/L4.idx $W/l4_$T.idx
$BIN merge --output $W/inc_$T --force --force-presence $W/dst_$T $W/l4_$T.idx 2>$W/inc_$T.err; rc=$?
echo "inc by $T rc=$rc"; sed "s/\x1b\[[0-9;]*m//g" $W/inc_$T.err | tail -3
[ $rc = 0 ] && echo " dump sorted: $($BIN dump $W/inc_$T 2>/dev/null | LC_ALL=C sort -S 2G | h)"
done
echo done
'
+161 -7
View File
@@ -29,6 +29,12 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstream"
version = "1.0.0"
@@ -190,6 +196,16 @@ dependencies = [
"hybrid-array",
]
[[package]]
name = "block-pseudorand"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6"
dependencies = [
"chiapos-chacha8",
"nanorand",
]
[[package]]
name = "bpaf"
version = "0.9.27"
@@ -309,6 +325,15 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "chiapos-chacha8"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a"
dependencies = [
"rayon",
]
[[package]]
name = "ciborium"
version = "0.2.2"
@@ -475,6 +500,42 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"is-terminal",
"itertools 0.10.5",
"num-traits",
"once_cell",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools 0.10.5",
]
[[package]]
name = "criterion2"
version = "3.0.4"
@@ -850,6 +911,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "http"
version = "1.4.2"
@@ -965,12 +1032,32 @@ dependencies = [
"cfb",
]
[[package]]
name = "is-terminal"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.15.0"
@@ -1211,6 +1298,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "nanorand"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958"
[[package]]
name = "ndarray"
version = "0.17.2"
@@ -1792,6 +1885,34 @@ version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "plotters"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
[[package]]
name = "plotters-svg"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
dependencies = [
"plotters-backend",
]
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -1865,7 +1986,7 @@ dependencies = [
"epserde-derive",
"fastrand",
"fxhash",
"itertools",
"itertools 0.15.0",
"log",
"mem_dbg",
"prefetch-index",
@@ -2025,13 +2146,17 @@ checksum = "e6724c78e033e1c4155b4d3f76593d09ad384739c6986c1addc2bd2f55b1aefe"
[[package]]
name = "rdst"
version = "0.20.11"
version = "0.20.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cf18dd479f856ca75267591c062e22b50e2db791157495bc2d533558de7061d"
checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752"
dependencies = [
"arbitrary-chunks",
"block-pseudorand",
"criterion",
"partition",
"rayon",
"tikv-jemallocator",
"voracious_radix_sort",
]
[[package]]
@@ -2530,9 +2655,9 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-sys"
version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8"
version = "0.5.4+5.3.0-patched"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0"
checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1"
dependencies = [
"cc",
"libc",
@@ -2540,14 +2665,24 @@ dependencies = [
[[package]]
name = "tikv-jemallocator"
version = "0.7.0"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd"
checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca"
dependencies = [
"libc",
"tikv-jemalloc-sys",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.13.2"
@@ -2735,6 +2870,15 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "voracious_radix_sort"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899"
dependencies = [
"rayon",
]
[[package]]
name = "walkdir"
version = "2.5.0"
@@ -2805,6 +2949,16 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "web-sys"
version = "0.3.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
+1 -1
View File
@@ -41,7 +41,7 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
# merge_mmap_audit.md). Static musl builds use jemalloc instead; glibc and
# macOS builds keep the system allocator.
[target.'cfg(target_env = "musl")'.dependencies]
tikv-jemallocator = "0.7.0"
tikv-jemallocator = "0.5.4"
[features]
default = ["numa"]