Remove in-place flag, optimize presence counting, and handle NUMA panics

The `obikmer select` CLI no longer supports in-place index rewriting; the `--output` flag is now required, with benchmarks updated to use temporary directories for atomic replacement. Added `--dense` and `--force-copy` flags. Introduced `batch_presence_counts` to compute presence counts across multiple column groups in a single pass, eliminating redundant I/O. Refactored the aggregation pipeline to branch on layer content, applying the optimized batched counting for `Presence` layers. Enhanced the NUMA runner to catch worker panics, track them, and re-raise after thread join to prevent indefinite blocking.
This commit is contained in:
Eric Coissac committed 2026-08-28 23:16:37 +02:00
1 parent 7363370065
commit 93fe838f97
9 files changed
+330 -26

No files matched your search

+69 -13
View File
@@ -12,7 +12,7 @@ use std::path::Path;
use obicompactvec::{
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
TempCompactIntVec,
TempBitVecBuilder, TempCompactIntVec, TempCompactIntVecBuilder, batch_presence_counts,
};
use obikindex::layer::{KmerLayer, LayerContent};
use obikindex::{KmerIndex, OKIError, OKIResult};
@@ -93,6 +93,44 @@ fn compute_group(
})
}
/// A bit-matrix source's every `AggOp` is a cheap derivation of one shared
/// presence count (see [`obicompactvec::batch_presence_counts`]'s own
/// docs for why: `sum` = the count itself, `any`/`max` = `count ≥ 1`,
/// `all`/`min` = `count == group size`, `none` = `count == 0`) — no further
/// matrix access needed once `count` is in hand.
fn agg_result_from_count(op: AggOp, group_len: usize, count: TempCompactIntVec) -> io::Result<AggResult> {
let n = count.len();
let group_len = group_len as u32;
Ok(match op {
AggOp::Sum => AggResult::Int(count),
AggOp::Any => {
let mut b = TempBitVecBuilder::new(n)?;
b.or_where(count.view(), |v| v >= 1);
AggResult::Bit(b.freeze()?)
}
AggOp::All => {
let mut b = TempBitVecBuilder::new(n)?;
b.or_where(count.view(), |v| v == group_len);
AggResult::Bit(b.freeze()?)
}
AggOp::None => {
let mut b = TempBitVecBuilder::new(n)?;
b.or_where(count.view(), |v| v == 0);
AggResult::Bit(b.freeze()?)
}
AggOp::Min => {
let mut b = TempCompactIntVecBuilder::new(n)?;
b.inc_predicate_fast(count.view(), |v| v == group_len);
AggResult::Int(b.freeze()?)
}
AggOp::Max => {
let mut b = TempCompactIntVecBuilder::new(n)?;
b.inc_predicate_fast(count.view(), |v| v >= 1);
AggResult::Int(b.freeze()?)
}
})
}
// ── AggResult → MatrixBuilder ─────────────────────────────────────────────────
/// Add one already-aggregated column to `mb` — the only piece `MatrixBuilder`
@@ -161,15 +199,6 @@ pub(crate) fn select_partition(
.to_path_buf();
copy_layer_files(src_layer.dir(), &dst_layer_dir, force_copy).map_err(OKIError::Io)?;
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
LayerContent::Count => {
Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
}
LayerContent::Presence => {
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
}
};
let data_subdir = if output_presence {
"presence"
} else {
@@ -180,10 +209,37 @@ pub(crate) fn select_partition(
let mut builder =
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
for spec in specs {
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?;
match src_layer.content() {
// One shared row-major (Sparse) / deduplicated column-major
// (Columnar/Packed/Implicit) pass covers every spec at once —
// see `batch_presence_counts`'s own docs. Every `AggOp` for a
// bit-matrix source is a cheap derivation of that one count.
LayerContent::Presence => {
let mat = PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?;
let groups: Vec<ColGroup> = specs
.iter()
.map(|s| ColGroup::new(s.label.clone(), s.indices.clone()))
.collect();
let counts = batch_presence_counts(&mat, &groups).map_err(OKIError::Io)?;
for (spec, count) in specs.iter().zip(counts) {
let r = agg_result_from_count(spec.op, spec.indices.len(), count)
.map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?;
}
}
// Count-matrix `sum`/`min`/`max` are genuine per-value
// reductions, not derivable from a single presence count —
// unchanged, one `col_view`-driven pass per spec.
LayerContent::Count => {
let mat = PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?;
for spec in specs {
let r = compute_group(&mat, spec, threshold).map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?;
}
}
}
builder.close().map_err(OKIError::Io)?;
}