Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content. - refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check - fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations - feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion - test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior - test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes - chore: remove obsolete cargo build environment variables from `run.sh` script
92 lines
4.6 KiB
Bash
Executable File
92 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# run.sh — builds obikmer (release, debug symbols) and merges the benchmark presence
|
|
# indexes in the container while sampling mappings/RSS/commit in parallel.
|
|
#
|
|
# Env:
|
|
# PLATFORM linux/amd64 (default, Rosetta) | linux/arm64
|
|
# GEN if set: build GEN synthetic indexes (gen_fasta.py) instead of using
|
|
# the benchmark ones; GENOME_LEN (default 150000), K (21), M (9), PART (256)
|
|
# REAL if set (with GEN): derive genomes from benchmark/run/genomes (gen_fasta_real.py)
|
|
# SRC_DIR alternative source tree (default: <repo>/src); VARIANT: volume suffix;
|
|
# LOCKED=0 to allow Cargo.lock changes (needed when a dep is added)
|
|
# NSRC number of source indexes to merge (default: all 20)
|
|
# TARGET gnu (default, glibc) | musl (static, same chain as the release CI)
|
|
# TOOL none (default) | heaptrack (heaptrack output goes to OUT/)
|
|
# MAXMAP if set, vm.max_map_count is lowered to this inside the VM for the
|
|
# run and restored afterwards (needs --privileged; affects the whole
|
|
# OrbStack VM while it runs)
|
|
# MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_, MALLOC_TRIM_THRESHOLD_ passed through
|
|
# OUT host output directory (default: ./out/<timestamp>)
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO="$(cd "$HERE/../.." && pwd)"
|
|
PLATFORM="${PLATFORM:-linux/amd64}"
|
|
NSRC="${NSRC:-20}"
|
|
TOOL="${TOOL:-none}"
|
|
TARGET="${TARGET:-gnu}"
|
|
OUT="${OUT:-$HERE/out/$(date +%Y%m%d-%H%M%S)}"
|
|
mkdir -p "$OUT"
|
|
IMG="obikmer-debug:${PLATFORM//\//-}"
|
|
VOL="obikmer-debug-target-${PLATFORM//\//-}-${TARGET}${VARIANT:+-$VARIANT}"
|
|
|
|
docker run --rm --privileged --platform "$PLATFORM" \
|
|
-v "${SRC_DIR:-$REPO/src}:/work/src:$([ -n "${SRC_DIR:-}" ] && echo rw || echo ro)" \
|
|
-v "$REPO/benchmark/run/specimen_index_presence:/idx_ro:ro" \
|
|
-v "$VOL:/target" \
|
|
-v "$OUT:/out" \
|
|
-v "$HERE/monitor.sh:/monitor.sh:ro" \
|
|
-v "$HERE/gen_fasta.py:/gen_fasta.py:ro" \
|
|
-v "$HERE/gen_fasta_real.py:/gen_fasta_real.py:ro" \
|
|
-v "$REPO/benchmark/run/genomes:/genomes_ro:ro" \
|
|
-e CARGO_TARGET_DIR=/target -e CARGO_PROFILE_RELEASE_DEBUG=1 \
|
|
-e GEN="${GEN:-}" -e LOCKED_FLAG="$([ "${LOCKED:-1}" = 0 ] || echo --locked)" -e SEED="${SEED:-20261118}" -e REAL="${REAL:-}" -e GENOME_LEN="${GENOME_LEN:-150000}" -e K="${K:-21}" -e M="${M:-9}" -e PART="${PART:-256}" -e RUST_LOG="${RUST_LOG:-}" -e NSRC="$NSRC" -e TARGET="$TARGET" -e TOOL="$TOOL" -e MAXMAP="${MAXMAP:-}" \
|
|
-e MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-}" \
|
|
-e MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-}" \
|
|
-e MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:-}" \
|
|
"$IMG" bash -c '
|
|
set -euo pipefail
|
|
for v in MALLOC_ARENA_MAX MALLOC_MMAP_THRESHOLD_ MALLOC_TRIM_THRESHOLD_ RUST_LOG; do
|
|
[ -n "${!v}" ] || unset $v
|
|
done
|
|
if [ "$TARGET" = musl ]; then
|
|
PKG_CONFIG_ALLOW_CROSS=1 cargo zigbuild --release $LOCKED_FLAG --target x86_64-unknown-linux-musl \
|
|
--manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
|
|
BIN=/target/x86_64-unknown-linux-musl/release/obikmer
|
|
else
|
|
cargo build --release $LOCKED_FLAG --manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
|
|
BIN=/target/release/obikmer
|
|
fi
|
|
file "$BIN" 2>/dev/null | cut -c1-150 || true
|
|
# merge writes into its sources (lock files…): work on a copy, never on benchmark/
|
|
rm -rf /tmp/idx && mkdir -p /tmp/idx
|
|
if [ -n "$GEN" ]; then
|
|
mkdir -p /tmp/fa
|
|
if [ -n "$REAL" ]; then python3 /gen_fasta_real.py "$GEN" /tmp/fa /genomes_ro "$GENOME_LEN" "$SEED"
|
|
else python3 /gen_fasta.py "$GEN" /tmp/fa "$GENOME_LEN" "$SEED"; fi
|
|
NSRC=$GEN
|
|
echo "indexing $GEN synthetic genomes (k=$K m=$M p=$PART)…"
|
|
ls /tmp/fa/*.fasta | xargs -P 8 -I{} sh -c \
|
|
"b=\$(basename {} .fasta); $BIN index -k $K -m $M --theta 0 -p $PART --label \$b -o /tmp/idx/\$b.idx {} >/dev/null 2>&1 || echo index failed: \$b"
|
|
else
|
|
cp -r /idx_ro/. /tmp/idx/
|
|
fi
|
|
mapfile -t SRC < <(find /tmp/idx -mindepth 1 -maxdepth 1 -type d | sort | head -n "$NSRC")
|
|
echo "sources: ${#SRC[@]} target: $TARGET tool: $TOOL max_map_count: $(cat /proc/sys/vm/max_map_count)"
|
|
if [ -n "$MAXMAP" ]; then
|
|
OLD=$(cat /proc/sys/vm/max_map_count); sysctl -w vm.max_map_count="$MAXMAP" >/dev/null
|
|
trap "sysctl -w vm.max_map_count=$OLD >/dev/null" EXIT
|
|
fi
|
|
rm -rf /tmp/merged
|
|
CMD=("$BIN" merge --output /tmp/merged --force --force-presence "${SRC[@]}")
|
|
if [ "$TOOL" = heaptrack ]; then CMD=(heaptrack -o /out/heaptrack "${CMD[@]}"); fi
|
|
"${CMD[@]}" 2>/out/merge.stderr &
|
|
PID=$!
|
|
/monitor.sh "$PID" /out/monitor.csv 1 &
|
|
MON=$!
|
|
set +e; wait "$PID"; RC=$?; set -e
|
|
wait "$MON" || true
|
|
echo "merge exit code: $RC" | tee /out/exit_code.txt
|
|
tail -5 /out/merge.stderr
|
|
'
|
|
echo "outputs: $OUT"
|