Extract k-mer counting logic into a dedicated counter module
Decouple k-mer counting from the partitioner by introducing a new `Counter` struct. The module exposes a fluent builder API with optional partial file retention, executes partition processing in parallel via Rayon with memory-aware chunk sizing, and integrates thread-safe progress callbacks. Update all callers to use the new counter, simplify test pipelines by removing serialization overhead, and clarify algorithm separation in module documentation.
This commit is contained in:
Eric Coissac
Executable
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env bash
|
||||
# smoke_test_index.sh — end-to-end smoke test of `obikmer index` + `obikmer query`
|
||||
#
|
||||
# The unit/integration test suite never exercises obikmer's real,
|
||||
# file-driven CLI path (every test builds indexes by calling library APIs
|
||||
# directly — see DevDocMD/implementation/partition_layer_cache.md, "(3)
|
||||
# done"/"(6) done": a green `cargo test --workspace` has twice missed a
|
||||
# real bug in this exact path). This script is the fast, repeatable
|
||||
# substitute for hand-rolling that check each time.
|
||||
#
|
||||
# What it does:
|
||||
# 1. builds `obikmer` (debug, via `cargo run`)
|
||||
# 2. generates a small deterministic random FASTA
|
||||
# 3. runs `obikmer index` on it
|
||||
# 4. picks a real k-mer from the source sequence, avoiding low-complexity
|
||||
# substrings (a homopolymer run tripped up an earlier manual run of
|
||||
# this check — it gets rejected by query's own entropy filter, which
|
||||
# looks like a bug but isn't one)
|
||||
# 5. runs `obikmer query` and checks the k-mer round-trips
|
||||
# 6. prints total-kmers-indexed and a clear PASS/FAIL, exit code matches
|
||||
#
|
||||
# Usage:
|
||||
# scripts/smoke_test_index.sh [-k KMER_SIZE] [-m MINIMIZER_SIZE] [-p PARTITIONS] [--keep]
|
||||
#
|
||||
# --keep leaves the temp directory in place (path printed) instead of
|
||||
# deleting it on exit, for manual inspection of a failure.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
K=11
|
||||
M=5
|
||||
PARTITIONS=4
|
||||
KEEP=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-k) K="$2"; shift 2 ;;
|
||||
-m) M="$2"; shift 2 ;;
|
||||
-p) PARTITIONS="$2"; shift 2 ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WORK="$(mktemp -d -t obikmer_smoke.XXXXXX)"
|
||||
|
||||
cleanup() {
|
||||
if [ "$KEEP" -eq 1 ]; then
|
||||
echo "kept: $WORK"
|
||||
else
|
||||
rm -rf "$WORK"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── 1. generate a small deterministic random FASTA ─────────────────────────
|
||||
python3 - "$WORK/test.fasta" "$K" <<'EOF'
|
||||
import random, sys
|
||||
path, k = sys.argv[1], int(sys.argv[2])
|
||||
random.seed(1234)
|
||||
bases = "ACGT"
|
||||
with open(path, "w") as f:
|
||||
for i in range(3):
|
||||
seq = "".join(random.choice(bases) for _ in range(300))
|
||||
f.write(f">seq{i}\n{seq}\n")
|
||||
EOF
|
||||
|
||||
# ── 2. build + run index ────────────────────────────────────────────────────
|
||||
cd "$REPO_ROOT/src"
|
||||
INDEX_LOG="$WORK/index.log"
|
||||
if ! cargo run -q -p obikmer --bin obikmer -- \
|
||||
index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
|
||||
-o "$WORK/out.idx" "$WORK/test.fasta" > "$INDEX_LOG" 2>&1
|
||||
then
|
||||
cat "$INDEX_LOG" >&2
|
||||
fail "obikmer index exited non-zero"
|
||||
fi
|
||||
|
||||
N_KMERS="$(grep -o '[0-9]* total kmers indexed' "$INDEX_LOG" | grep -o '^[0-9]*' || true)"
|
||||
[ -n "$N_KMERS" ] || { cat "$INDEX_LOG" >&2; fail "could not find 'N total kmers indexed' in index log"; }
|
||||
[ "$N_KMERS" -gt 0 ] || fail "index reports 0 kmers indexed"
|
||||
|
||||
# ── 3+4. try candidate k-mers spread across the source sequence until one
|
||||
# round-trips. `query` applies its own entropy filter to the query
|
||||
# sequence (not exposed as a CLI flag, undocumented threshold) — a
|
||||
# window that fails it reports kmer_count:0 even though the k-mer
|
||||
# is genuinely in the index (not a bug, just a bad test fixture).
|
||||
# Rather than reverse-engineer the filter's formula, just ask the
|
||||
# real binary and move to the next candidate on a miss.
|
||||
readarray -t CANDIDATES < <(python3 - "$WORK/test.fasta" "$K" <<'EOF'
|
||||
import sys
|
||||
path, k = sys.argv[1], int(sys.argv[2])
|
||||
with open(path) as f:
|
||||
seq = "".join(l.strip() for l in f if not l.startswith(">"))
|
||||
for start in range(0, len(seq) - k, 17):
|
||||
print(seq[start:start+k])
|
||||
EOF
|
||||
)
|
||||
[ "${#CANDIDATES[@]}" -gt 0 ] || fail "could not extract any candidate k-mer from the source FASTA"
|
||||
|
||||
QUERY_LOG="$WORK/query.log"
|
||||
FOUND=0
|
||||
for QUERY_KMER in "${CANDIDATES[@]}"; do
|
||||
printf ">q1\n%s\n" "$QUERY_KMER" > "$WORK/query.fasta"
|
||||
if ! cargo run -q -p obikmer --bin obikmer -- \
|
||||
query "$WORK/out.idx" "$WORK/query.fasta" > "$QUERY_LOG" 2>&1
|
||||
then
|
||||
cat "$QUERY_LOG" >&2
|
||||
fail "obikmer query exited non-zero"
|
||||
fi
|
||||
if grep -q '"kmer_count":1' "$QUERY_LOG"; then
|
||||
FOUND=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FOUND" -ne 1 ]; then
|
||||
cat "$QUERY_LOG" >&2
|
||||
fail "no candidate k-mer round-tripped (tried ${#CANDIDATES[@]}) — likely a real regression, not a low-complexity fixture"
|
||||
fi
|
||||
|
||||
echo "PASS: index+query round-trip OK — $N_KMERS kmers indexed, query k-mer '$QUERY_KMER' found"
|
||||
Reference in New Issue
Block a user