obikmer/benchmark/pack_dense.sh

#!/usr/bin/env bash
# Usage: pack_dense.sh KIND   (KIND = presence | count)
#
# Builds global_index_KIND_dense/ from global_index_KIND/ — `merge` packs
# sparse by default (2026-08-28), so the dense comparison arm needs an
# explicit rebuild.
#
# `obikmer pack --dense` cannot do this: it only converts the raw, freshly-
# built columnar (per-genome-file, unpacked) matrix into a packed format —
# `finalize_indexed` already packs (sparse by default) as the last step of
# every index-building command, so there is no columnar leftover for a
# second `pack` invocation to work from; it fails ("No such file or
# directory", `obicompactvec::bitmatrix::packed::pack_bit_matrix` looking
# for a `meta.json` that packing already cleaned up).
#
# `obikmer select` doesn't have that limitation — it always rebuilds its
# output from scratch via `MatrixBuilder`, reading the source through the
# format-agnostic `PersistentBitMatrix`/`PersistentIntMatrix` (Sparse
# included, both content kinds — see `obicompactvec::batch_presence_counts`/
# `batch_int_group_stats`) — so a full, unaggregated passthrough (`--select`
# naming every genome, no `--group`) with `--dense` genuinely repacks
# Sparse → Dense for either kind, and gets the layer-identity hard-linking
# already implemented in `obikselect::select_layer::copy_layer_files` for
# free.
#
# Outputs:
#   global_index_KIND_dense/index.done   (rebuilt via `select`, dense-packed)
#   stats/pack_dense_KIND/current.stats  (one CSV data row, no header)
set -euo pipefail

KIND="$1"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BINARY="${SCRIPT_DIR}/../src/target/release/obikmer"
SOURCE="${SCRIPT_DIR}/global_index_${KIND}"
OUTPUT="${SCRIPT_DIR}/global_index_${KIND}_dense"
STATS_DIR="${SCRIPT_DIR}/stats/pack_dense_${KIND}"
STATS_FILE="${STATS_DIR}/current.stats"

mkdir -p "${STATS_DIR}"

echo "[pack_dense_${KIND}] ${SOURCE} → ${OUTPUT}"

LABELS=$("${BINARY}" annotate "${SOURCE}" --dump | tail -n +2 | python3 -c "
import sys, csv
r = csv.reader(sys.stdin)
print(','.join(row[0] for row in r if row))
")

STDERR_LOG=$(mktemp)
trap 'rm -f "${STDERR_LOG}"' EXIT

"${BINARY}" select \
    --output "${OUTPUT}" \
    --force \
    --dense \
    --select "${LABELS}" \
    "${SOURCE}" \
    2>"${STDERR_LOG}"

cat "${STDERR_LOG}" >&2

python3 - "${STDERR_LOG}" <<'PYEOF' >"${STATS_FILE}"
import sys, re

logfile = sys.argv[1]

def strip_ansi(s):
    return re.sub(r'\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]', '', s)

def parse_wall(s):
    s = s.strip()
    if s.endswith('ms'): return float(s[:-2]) / 1000.0
    if s.endswith('s'):  return float(s[:-1])
    return 0.0

def parse_rss(s):
    m = re.match(r'([\d.]+)\s*(GB|MB|KB|B)', s.strip())
    if not m: return 0
    return int(float(m.group(1)) * {'GB': 1<<30, 'MB': 1<<20, 'KB': 1024, 'B': 1}[m.group(2)])

def is_sep(s):
    return bool(s) and not re.search(r'[A-Za-z0-9]', s)

stats = {}
state = 'scan'
with open(logfile, errors='replace') as fh:
    for raw in fh:
        line = strip_ansi(raw.rstrip('\n'))
        s    = line.strip()
        if state == 'scan':
            if re.search(r'\bstage\b.*\bwall\b', line):
                state = 'in_header'
        elif state == 'in_header':
            if is_sep(s): state = 'rows'
        elif state == 'rows':
            if is_sep(s): state = 'total'
            elif s:
                parts = re.split(r'  +', s)
                if len(parts) >= 4:
                    stats[parts[0]] = (parse_wall(parts[1]), parse_rss(parts[3]))
        elif state == 'total':
            if s:
                parts = re.split(r'  +', s)
                if len(parts) >= 3:
                    stats['TOTAL'] = (parse_wall(parts[1]),
                                      parse_rss(parts[3]) if len(parts) > 3 else 0)
            break

w, r = stats.get('select', ('', ''))
tw, tr = stats.get('TOTAL', ('', ''))
row = [f'{w:.3f}' if isinstance(w, float) else '', str(r),
       f'{tw:.3f}' if isinstance(tw, float) else '', str(tr)]
print(','.join(row))
PYEOF

echo "Done → ${OUTPUT}"