Eric Coissac
878b63566f
Decouple k-mer counting from the partitioner by introducing a new `Counter` struct. The module exposes a fluent builder API with optional partial file retention, executes partition processing in parallel via Rayon with memory-aware chunk sizing, and integrates thread-safe progress callbacks. Update all callers to use the new counter, simplify test pipelines by removing serialization overhead, and clarify algorithm separation in module documentation.
129 lines
4.6 KiB
Bash
Executable File
129 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# smoke_test_index.sh — end-to-end smoke test of `obikmer index` + `obikmer query`
|
|
#
|
|
# The unit/integration test suite never exercises obikmer's real,
|
|
# file-driven CLI path (every test builds indexes by calling library APIs
|
|
# directly — see DevDocMD/implementation/partition_layer_cache.md, "(3)
|
|
# done"/"(6) done": a green `cargo test --workspace` has twice missed a
|
|
# real bug in this exact path). This script is the fast, repeatable
|
|
# substitute for hand-rolling that check each time.
|
|
#
|
|
# What it does:
|
|
# 1. builds `obikmer` (debug, via `cargo run`)
|
|
# 2. generates a small deterministic random FASTA
|
|
# 3. runs `obikmer index` on it
|
|
# 4. picks a real k-mer from the source sequence, avoiding low-complexity
|
|
# substrings (a homopolymer run tripped up an earlier manual run of
|
|
# this check — it gets rejected by query's own entropy filter, which
|
|
# looks like a bug but isn't one)
|
|
# 5. runs `obikmer query` and checks the k-mer round-trips
|
|
# 6. prints total-kmers-indexed and a clear PASS/FAIL, exit code matches
|
|
#
|
|
# Usage:
|
|
# scripts/smoke_test_index.sh [-k KMER_SIZE] [-m MINIMIZER_SIZE] [-p PARTITIONS] [--keep]
|
|
#
|
|
# --keep leaves the temp directory in place (path printed) instead of
|
|
# deleting it on exit, for manual inspection of a failure.
|
|
|
|
set -euo pipefail
|
|
|
|
K=11
|
|
M=5
|
|
PARTITIONS=4
|
|
KEEP=0
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-k) K="$2"; shift 2 ;;
|
|
-m) M="$2"; shift 2 ;;
|
|
-p) PARTITIONS="$2"; shift 2 ;;
|
|
--keep) KEEP=1; shift ;;
|
|
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
WORK="$(mktemp -d -t obikmer_smoke.XXXXXX)"
|
|
|
|
cleanup() {
|
|
if [ "$KEEP" -eq 1 ]; then
|
|
echo "kept: $WORK"
|
|
else
|
|
rm -rf "$WORK"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
fail() {
|
|
echo "FAIL: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
# ── 1. generate a small deterministic random FASTA ─────────────────────────
|
|
python3 - "$WORK/test.fasta" "$K" <<'EOF'
|
|
import random, sys
|
|
path, k = sys.argv[1], int(sys.argv[2])
|
|
random.seed(1234)
|
|
bases = "ACGT"
|
|
with open(path, "w") as f:
|
|
for i in range(3):
|
|
seq = "".join(random.choice(bases) for _ in range(300))
|
|
f.write(f">seq{i}\n{seq}\n")
|
|
EOF
|
|
|
|
# ── 2. build + run index ────────────────────────────────────────────────────
|
|
cd "$REPO_ROOT/src"
|
|
INDEX_LOG="$WORK/index.log"
|
|
if ! cargo run -q -p obikmer --bin obikmer -- \
|
|
index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
|
|
-o "$WORK/out.idx" "$WORK/test.fasta" > "$INDEX_LOG" 2>&1
|
|
then
|
|
cat "$INDEX_LOG" >&2
|
|
fail "obikmer index exited non-zero"
|
|
fi
|
|
|
|
N_KMERS="$(grep -o '[0-9]* total kmers indexed' "$INDEX_LOG" | grep -o '^[0-9]*' || true)"
|
|
[ -n "$N_KMERS" ] || { cat "$INDEX_LOG" >&2; fail "could not find 'N total kmers indexed' in index log"; }
|
|
[ "$N_KMERS" -gt 0 ] || fail "index reports 0 kmers indexed"
|
|
|
|
# ── 3+4. try candidate k-mers spread across the source sequence until one
|
|
# round-trips. `query` applies its own entropy filter to the query
|
|
# sequence (not exposed as a CLI flag, undocumented threshold) — a
|
|
# window that fails it reports kmer_count:0 even though the k-mer
|
|
# is genuinely in the index (not a bug, just a bad test fixture).
|
|
# Rather than reverse-engineer the filter's formula, just ask the
|
|
# real binary and move to the next candidate on a miss.
|
|
readarray -t CANDIDATES < <(python3 - "$WORK/test.fasta" "$K" <<'EOF'
|
|
import sys
|
|
path, k = sys.argv[1], int(sys.argv[2])
|
|
with open(path) as f:
|
|
seq = "".join(l.strip() for l in f if not l.startswith(">"))
|
|
for start in range(0, len(seq) - k, 17):
|
|
print(seq[start:start+k])
|
|
EOF
|
|
)
|
|
[ "${#CANDIDATES[@]}" -gt 0 ] || fail "could not extract any candidate k-mer from the source FASTA"
|
|
|
|
QUERY_LOG="$WORK/query.log"
|
|
FOUND=0
|
|
for QUERY_KMER in "${CANDIDATES[@]}"; do
|
|
printf ">q1\n%s\n" "$QUERY_KMER" > "$WORK/query.fasta"
|
|
if ! cargo run -q -p obikmer --bin obikmer -- \
|
|
query "$WORK/out.idx" "$WORK/query.fasta" > "$QUERY_LOG" 2>&1
|
|
then
|
|
cat "$QUERY_LOG" >&2
|
|
fail "obikmer query exited non-zero"
|
|
fi
|
|
if grep -q '"kmer_count":1' "$QUERY_LOG"; then
|
|
FOUND=1
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ "$FOUND" -ne 1 ]; then
|
|
cat "$QUERY_LOG" >&2
|
|
fail "no candidate k-mer round-tripped (tried ${#CANDIDATES[@]}) — likely a real regression, not a low-complexity fixture"
|
|
fi
|
|
|
|
echo "PASS: index+query round-trip OK — $N_KMERS kmers indexed, query k-mer '$QUERY_KMER' found"
|