Eric Coissac
52f26809af
ci.yml / build (pull_request) Successful in 3m49s
Restructure the benchmark pipeline to direct all simulated data, indices, statistics, and query outputs into a unified `run/` directory. Update Makefile targets, shell scripts, and Python utilities to resolve paths relative to this new base. Adjust documentation and dependency tracking to match the revised layout, and remove outdated temporary artifacts.
118 lines
4.0 KiB
Bash
Executable File
118 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Usage: pack_dense.sh KIND (KIND = presence | count)
|
|
#
|
|
# Builds global_index_KIND_dense/ from global_index_KIND/ — `merge` packs
|
|
# sparse by default (2026-08-28), so the dense comparison arm needs an
|
|
# explicit rebuild.
|
|
#
|
|
# `obikmer pack --dense` cannot do this: it only converts the raw, freshly-
|
|
# built columnar (per-genome-file, unpacked) matrix into a packed format —
|
|
# `finalize_indexed` already packs (sparse by default) as the last step of
|
|
# every index-building command, so there is no columnar leftover for a
|
|
# second `pack` invocation to work from; it fails ("No such file or
|
|
# directory", `obicompactvec::bitmatrix::packed::pack_bit_matrix` looking
|
|
# for a `meta.json` that packing already cleaned up).
|
|
#
|
|
# `obikmer select` doesn't have that limitation — it always rebuilds its
|
|
# output from scratch via `MatrixBuilder`, reading the source through the
|
|
# format-agnostic `PersistentBitMatrix`/`PersistentIntMatrix` (Sparse
|
|
# included, both content kinds — see `obicompactvec::batch_presence_counts`/
|
|
# `batch_int_group_stats`) — so a full, unaggregated passthrough (`--select`
|
|
# naming every genome, no `--group`) with `--dense` genuinely repacks
|
|
# Sparse → Dense for either kind, and gets the layer-identity hard-linking
|
|
# already implemented in `obikselect::select_layer::copy_layer_files` for
|
|
# free.
|
|
#
|
|
# Outputs:
|
|
# global_index_KIND_dense/index.done (rebuilt via `select`, dense-packed)
|
|
# stats/pack_dense_KIND/current.stats (one CSV data row, no header)
|
|
set -euo pipefail
|
|
|
|
KIND="$1"
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
RUN_DIR="${SCRIPT_DIR}/run"
|
|
BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
|
|
SOURCE="${RUN_DIR}/global_index_${KIND}"
|
|
OUTPUT="${RUN_DIR}/global_index_${KIND}_dense"
|
|
STATS_DIR="${RUN_DIR}/stats/pack_dense_${KIND}"
|
|
STATS_FILE="${STATS_DIR}/current.stats"
|
|
|
|
mkdir -p "${STATS_DIR}"
|
|
|
|
echo "[pack_dense_${KIND}] ${SOURCE} → ${OUTPUT}"
|
|
|
|
LABELS=$("${BINARY}" annotate "${SOURCE}" --dump | tail -n +2 | python3 -c "
|
|
import sys, csv
|
|
r = csv.reader(sys.stdin)
|
|
print(','.join(row[0] for row in r if row))
|
|
")
|
|
|
|
STDERR_LOG=$(mktemp)
|
|
trap 'rm -f "${STDERR_LOG}"' EXIT
|
|
|
|
"${BINARY}" select \
|
|
--output "${OUTPUT}" \
|
|
--force \
|
|
--dense \
|
|
--select "${LABELS}" \
|
|
"${SOURCE}" \
|
|
2>"${STDERR_LOG}"
|
|
|
|
cat "${STDERR_LOG}" >&2
|
|
|
|
python3 - "${STDERR_LOG}" <<'PYEOF' >"${STATS_FILE}"
|
|
import sys, re
|
|
|
|
logfile = sys.argv[1]
|
|
|
|
def strip_ansi(s):
|
|
return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)
|
|
|
|
def parse_wall(s):
|
|
s = s.strip()
|
|
if s.endswith('ms'): return float(s[:-2]) / 1000.0
|
|
if s.endswith('s'): return float(s[:-1])
|
|
return 0.0
|
|
|
|
def parse_rss(s):
|
|
m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
|
|
if not m: return 0
|
|
return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])
|
|
|
|
def is_sep(s):
|
|
return bool(s) and not re.search(r'[A-Za-z0-9]', s)
|
|
|
|
stats = {}
|
|
state = 'scan'
|
|
with open(logfile, errors='replace') as fh:
|
|
for raw in fh:
|
|
line = strip_ansi(raw.rstrip('\n'))
|
|
s = line.strip()
|
|
if state == 'scan':
|
|
if re.search(r'\bstage\b.*\bwall\b', line):
|
|
state = 'in_header'
|
|
elif state == 'in_header':
|
|
if is_sep(s): state = 'rows'
|
|
elif state == 'rows':
|
|
if is_sep(s): state = 'total'
|
|
elif s:
|
|
parts = re.split(r' +', s)
|
|
if len(parts) >= 4:
|
|
stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
|
|
elif state == 'total':
|
|
if s:
|
|
parts = re.split(r' +', s)
|
|
if len(parts) >= 3:
|
|
stats['TOTAL'] = (parse_wall(parts[1]),
|
|
parse_rss(parts[3]) if len(parts) > 3 else 0)
|
|
break
|
|
|
|
w, r = stats.get('select', ('', ''))
|
|
tw, tr = stats.get('TOTAL', ('', ''))
|
|
row = [f'{w:.3f}' if isinstance(w, float) else '', str(r),
|
|
f'{tw:.3f}' if isinstance(tw, float) else '', str(tr)]
|
|
print(','.join(row))
|
|
PYEOF
|
|
|
|
echo "Done → ${OUTPUT}"
|