Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content. - refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check - fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations - feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion - test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior - test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes - chore: remove obsolete cargo build environment variables from `run.sh` script
151 lines
5.7 KiB
Bash
Executable File
151 lines
5.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# smoke_test_index.sh — end-to-end smoke test of `obikmer index` + `obikmer query`
|
|
#
|
|
# The unit/integration test suite never exercises obikmer's real,
|
|
# file-driven CLI path (every test builds indexes by calling library APIs
|
|
# directly — see DevDocMD/implementation/partition_layer_cache.md, "(3)
|
|
# done"/"(6) done": a green `cargo test --workspace` has twice missed a
|
|
# real bug in this exact path). This script is the fast, repeatable
|
|
# substitute for hand-rolling that check each time.
|
|
#
|
|
# What it does:
|
|
# 1. builds `obikmer` (debug, via `cargo run`)
|
|
# 2. generates a small deterministic random FASTA
|
|
# 3. runs `obikmer index` on it
|
|
# 4. picks a real k-mer from the source sequence, avoiding low-complexity
|
|
# substrings (a homopolymer run tripped up an earlier manual run of
|
|
# this check — it gets rejected by query's own entropy filter, which
|
|
# looks like a bug but isn't one)
|
|
# 5. runs `obikmer query` and checks the k-mer round-trips
|
|
# 6. prints total-kmers-indexed and a clear PASS/FAIL, exit code matches
|
|
#
|
|
# Usage:
|
|
# scripts/smoke_test_index.sh [-k KMER_SIZE] [-m MINIMIZER_SIZE] [-p PARTITIONS] [--keep]
|
|
#
|
|
# --keep leaves the temp directory in place (path printed) instead of
|
|
# deleting it on exit, for manual inspection of a failure.
|
|
|
|
set -euo pipefail
|
|
|
|
K=11
|
|
M=5
|
|
PARTITIONS=4
|
|
KEEP=0
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-k) K="$2"; shift 2 ;;
|
|
-m) M="$2"; shift 2 ;;
|
|
-p) PARTITIONS="$2"; shift 2 ;;
|
|
--keep) KEEP=1; shift ;;
|
|
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
WORK="$(mktemp -d -t obikmer_smoke.XXXXXX)"
|
|
|
|
cleanup() {
|
|
if [ "$KEEP" -eq 1 ]; then
|
|
echo "kept: $WORK"
|
|
else
|
|
rm -rf "$WORK"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
fail() {
|
|
echo "FAIL: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
# ── 1. generate a small deterministic random FASTA ─────────────────────────
|
|
python3 - "$WORK/test.fasta" "$K" <<'EOF'
|
|
import random, sys
|
|
path, k = sys.argv[1], int(sys.argv[2])
|
|
random.seed(1234)
|
|
bases = "ACGT"
|
|
with open(path, "w") as f:
|
|
for i in range(3):
|
|
seq = "".join(random.choice(bases) for _ in range(300))
|
|
f.write(f">seq{i}\n{seq}\n")
|
|
EOF
|
|
|
|
# ── 2. build + run index ────────────────────────────────────────────────────
|
|
cd "$REPO_ROOT/src"
|
|
INDEX_LOG="$WORK/index.log"
|
|
if ! cargo run -q -p obikmer --bin obikmer -- \
|
|
index -k "$K" -m "$M" --theta 0 -p "$PARTITIONS" \
|
|
-o "$WORK/out.idx" "$WORK/test.fasta" > "$INDEX_LOG" 2>&1
|
|
then
|
|
cat "$INDEX_LOG" >&2
|
|
fail "obikmer index exited non-zero"
|
|
fi
|
|
|
|
N_KMERS="$(grep -o '[0-9]* total kmers indexed' "$INDEX_LOG" | grep -o '^[0-9]*' || true)"
|
|
[ -n "$N_KMERS" ] || { cat "$INDEX_LOG" >&2; fail "could not find 'N total kmers indexed' in index log"; }
|
|
[ "$N_KMERS" -gt 0 ] || fail "index reports 0 kmers indexed"
|
|
|
|
# ── 3+4. try candidate k-mers spread across the source sequence until one
|
|
# round-trips. `query` applies its own entropy filter to the query
|
|
# sequence (not exposed as a CLI flag, undocumented threshold) — a
|
|
# window that fails it reports kmer_count:0 even though the k-mer
|
|
# is genuinely in the index (not a bug, just a bad test fixture).
|
|
# Rather than reverse-engineer the filter's formula, just ask the
|
|
# real binary and move to the next candidate on a miss.
|
|
readarray -t CANDIDATES < <(python3 - "$WORK/test.fasta" "$K" <<'EOF'
|
|
import sys
|
|
path, k = sys.argv[1], int(sys.argv[2])
|
|
with open(path) as f:
|
|
seq = "".join(l.strip() for l in f if not l.startswith(">"))
|
|
for start in range(0, len(seq) - k, 17):
|
|
print(seq[start:start+k])
|
|
EOF
|
|
)
|
|
[ "${#CANDIDATES[@]}" -gt 0 ] || fail "could not extract any candidate k-mer from the source FASTA"
|
|
|
|
QUERY_LOG="$WORK/query.log"
|
|
FOUND=0
|
|
for QUERY_KMER in "${CANDIDATES[@]}"; do
|
|
printf ">q1\n%s\n" "$QUERY_KMER" > "$WORK/query.fasta"
|
|
if ! cargo run -q -p obikmer --bin obikmer -- \
|
|
query "$WORK/out.idx" "$WORK/query.fasta" > "$QUERY_LOG" 2>&1
|
|
then
|
|
cat "$QUERY_LOG" >&2
|
|
fail "obikmer query exited non-zero"
|
|
fi
|
|
if grep -q '"kmer_count":1' "$QUERY_LOG"; then
|
|
FOUND=1
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ "$FOUND" -ne 1 ]; then
|
|
cat "$QUERY_LOG" >&2
|
|
fail "no candidate k-mer round-tripped (tried ${#CANDIDATES[@]}) — likely a real regression, not a low-complexity fixture"
|
|
fi
|
|
|
|
# ── 5. regressions: (a) many partitions against a tiny input leaves most of
|
|
# them empty (no layer 0 on disk) — must still build; (b) an NCBI-style
|
|
# `.fna.gz` name must be read, not silently skipped (it used to be
|
|
# taken for a directory and ignored, yielding an empty index).
|
|
count_kmers() { # $1 = output index dir, rest = obikmer index args
|
|
local out="$1"; shift
|
|
local log="$WORK/regress.log"
|
|
( cd "$REPO_ROOT/src" && cargo run -q -p obikmer --bin obikmer -- \
|
|
index -k "$K" -m "$M" --theta 0 -o "$out" "$@" ) > "$log" 2>&1 \
|
|
|| { cat "$log" >&2; fail "obikmer index $* exited non-zero"; }
|
|
grep -o '[0-9]* total kmers indexed' "$log" | grep -o '^[0-9]*' || echo 0
|
|
}
|
|
|
|
N_EMPTY="$(count_kmers "$WORK/empty_parts.idx" -p 256 "$WORK/test.fasta")"
|
|
[ "$N_EMPTY" = "$N_KMERS" ] \
|
|
|| fail "-p 256 (mostly empty partitions) indexed $N_EMPTY kmers, expected $N_KMERS"
|
|
|
|
gzip -c "$WORK/test.fasta" > "$WORK/test.fna.gz"
|
|
N_GZ="$(count_kmers "$WORK/gz.idx" -p "$PARTITIONS" "$WORK/test.fna.gz")"
|
|
[ "$N_GZ" = "$N_KMERS" ] \
|
|
|| fail ".fna.gz input indexed $N_GZ kmers, expected $N_KMERS (input skipped?)"
|
|
|
|
echo "PASS: index+query round-trip OK — $N_KMERS kmers indexed, query k-mer '$QUERY_KMER' found; empty-partition and .fna.gz regressions OK"
|