Files
obikmer/scripts/docker_debug/run.sh
T
Eric Coissac 7f829ea557 docs: clarify merge re-merge behavior and input file handling
Update documentation to explain that merged indices can serve as sources if at least one source is unmerged, and that the resulting column order follows the base index first. Clarify that explicitly named files are always read regardless of extension, with format detection based on content.

- refactor: prevent I/O errors when re-merging packed outputs by restricting `choose_base` to valid destinations via new `can_be_destination` check
- fix: handle zero-layer partitions in `build_index_layer` and `remove_build_artifacts` to skip unnecessary operations
- feat: allow `PathIter` to treat unrecognized extensions as explicit files, ensuring they are not silently dropped during directory expansion
- test: add comprehensive tests for `PathIter` including extension recognition, explicit file handling, and stdin sentinel behavior
- test: update smoke tests to verify empty partitions, `.fna.gz` indexing, and refusal of merging only packed indexes
- chore: remove obsolete cargo build environment variables from `run.sh` script
2026-11-19 07:10:58 +01:00

92 lines
4.6 KiB
Bash
Executable File

#!/usr/bin/env bash
# run.sh — builds obikmer (release, debug symbols) and merges the benchmark presence
# indexes in the container while sampling mappings/RSS/commit in parallel.
#
# Env:
# PLATFORM linux/amd64 (default, Rosetta) | linux/arm64
# GEN if set: build GEN synthetic indexes (gen_fasta.py) instead of using
# the benchmark ones; GENOME_LEN (default 150000), K (21), M (9), PART (256)
# REAL if set (with GEN): derive genomes from benchmark/run/genomes (gen_fasta_real.py)
# SRC_DIR alternative source tree (default: <repo>/src); VARIANT: volume suffix;
# LOCKED=0 to allow Cargo.lock changes (needed when a dep is added)
# NSRC number of source indexes to merge (default: all 20)
# TARGET gnu (default, glibc) | musl (static, same chain as the release CI)
# TOOL none (default) | heaptrack (heaptrack output goes to OUT/)
# MAXMAP if set, vm.max_map_count is lowered to this inside the VM for the
# run and restored afterwards (needs --privileged; affects the whole
# OrbStack VM while it runs)
# MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_, MALLOC_TRIM_THRESHOLD_ passed through
# OUT host output directory (default: ./out/<timestamp>)
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
PLATFORM="${PLATFORM:-linux/amd64}"
NSRC="${NSRC:-20}"
TOOL="${TOOL:-none}"
TARGET="${TARGET:-gnu}"
OUT="${OUT:-$HERE/out/$(date +%Y%m%d-%H%M%S)}"
mkdir -p "$OUT"
IMG="obikmer-debug:${PLATFORM//\//-}"
VOL="obikmer-debug-target-${PLATFORM//\//-}-${TARGET}${VARIANT:+-$VARIANT}"
docker run --rm --privileged --platform "$PLATFORM" \
-v "${SRC_DIR:-$REPO/src}:/work/src:$([ -n "${SRC_DIR:-}" ] && echo rw || echo ro)" \
-v "$REPO/benchmark/run/specimen_index_presence:/idx_ro:ro" \
-v "$VOL:/target" \
-v "$OUT:/out" \
-v "$HERE/monitor.sh:/monitor.sh:ro" \
-v "$HERE/gen_fasta.py:/gen_fasta.py:ro" \
-v "$HERE/gen_fasta_real.py:/gen_fasta_real.py:ro" \
-v "$REPO/benchmark/run/genomes:/genomes_ro:ro" \
-e CARGO_TARGET_DIR=/target -e CARGO_PROFILE_RELEASE_DEBUG=1 \
-e GEN="${GEN:-}" -e LOCKED_FLAG="$([ "${LOCKED:-1}" = 0 ] || echo --locked)" -e SEED="${SEED:-20261118}" -e REAL="${REAL:-}" -e GENOME_LEN="${GENOME_LEN:-150000}" -e K="${K:-21}" -e M="${M:-9}" -e PART="${PART:-256}" -e RUST_LOG="${RUST_LOG:-}" -e NSRC="$NSRC" -e TARGET="$TARGET" -e TOOL="$TOOL" -e MAXMAP="${MAXMAP:-}" \
-e MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-}" \
-e MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-}" \
-e MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:-}" \
"$IMG" bash -c '
set -euo pipefail
for v in MALLOC_ARENA_MAX MALLOC_MMAP_THRESHOLD_ MALLOC_TRIM_THRESHOLD_ RUST_LOG; do
[ -n "${!v}" ] || unset $v
done
if [ "$TARGET" = musl ]; then
PKG_CONFIG_ALLOW_CROSS=1 cargo zigbuild --release $LOCKED_FLAG --target x86_64-unknown-linux-musl \
--manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
BIN=/target/x86_64-unknown-linux-musl/release/obikmer
else
cargo build --release $LOCKED_FLAG --manifest-path /work/src/Cargo.toml --bin obikmer 2>&1 | tail -3
BIN=/target/release/obikmer
fi
file "$BIN" 2>/dev/null | cut -c1-150 || true
# merge writes into its sources (lock files…): work on a copy, never on benchmark/
rm -rf /tmp/idx && mkdir -p /tmp/idx
if [ -n "$GEN" ]; then
mkdir -p /tmp/fa
if [ -n "$REAL" ]; then python3 /gen_fasta_real.py "$GEN" /tmp/fa /genomes_ro "$GENOME_LEN" "$SEED"
else python3 /gen_fasta.py "$GEN" /tmp/fa "$GENOME_LEN" "$SEED"; fi
NSRC=$GEN
echo "indexing $GEN synthetic genomes (k=$K m=$M p=$PART)…"
ls /tmp/fa/*.fasta | xargs -P 8 -I{} sh -c \
"b=\$(basename {} .fasta); $BIN index -k $K -m $M --theta 0 -p $PART --label \$b -o /tmp/idx/\$b.idx {} >/dev/null 2>&1 || echo index failed: \$b"
else
cp -r /idx_ro/. /tmp/idx/
fi
mapfile -t SRC < <(find /tmp/idx -mindepth 1 -maxdepth 1 -type d | sort | head -n "$NSRC")
echo "sources: ${#SRC[@]} target: $TARGET tool: $TOOL max_map_count: $(cat /proc/sys/vm/max_map_count)"
if [ -n "$MAXMAP" ]; then
OLD=$(cat /proc/sys/vm/max_map_count); sysctl -w vm.max_map_count="$MAXMAP" >/dev/null
trap "sysctl -w vm.max_map_count=$OLD >/dev/null" EXIT
fi
rm -rf /tmp/merged
CMD=("$BIN" merge --output /tmp/merged --force --force-presence "${SRC[@]}")
if [ "$TOOL" = heaptrack ]; then CMD=(heaptrack -o /out/heaptrack "${CMD[@]}"); fi
"${CMD[@]}" 2>/out/merge.stderr &
PID=$!
/monitor.sh "$PID" /out/monitor.csv 1 &
MON=$!
set +e; wait "$PID"; RC=$?; set -e
wait "$MON" || true
echo "merge exit code: $RC" | tee /out/exit_code.txt
tail -5 /out/merge.stderr
'
echo "outputs: $OUT"