Remove in-place flag, optimize presence counting, and handle NUMA panics
The `obikmer select` CLI no longer supports in-place index rewriting; the `--output` flag is now required, with benchmarks updated to use temporary directories for atomic replacement. Added `--dense` and `--force-copy` flags. Introduced `batch_presence_counts` to compute presence counts across multiple column groups in a single pass, eliminating redundant I/O. Refactored the aggregation pipeline to branch on layer content, applying the optimized batched counting for `Presence` layers. Enhanced the NUMA runner to catch worker panics, track them, and re-raise after thread join to prevent indefinite blocking.
This commit is contained in:
1 parent
7363370065
commit
93fe838f97
9 files changed
+330
-26
No files matched your search
@@ -12,7 +12,7 @@ use std::path::Path;
|
||||
|
||||
use obicompactvec::{
|
||||
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
|
||||
TempCompactIntVec,
|
||||
TempBitVecBuilder, TempCompactIntVec, TempCompactIntVecBuilder, batch_presence_counts,
|
||||
};
|
||||
use obikindex::layer::{KmerLayer, LayerContent};
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
@@ -93,6 +93,44 @@ fn compute_group(
|
||||
})
|
||||
}
|
||||
|
||||
/// A bit-matrix source's every `AggOp` is a cheap derivation of one shared
|
||||
/// presence count (see [`obicompactvec::batch_presence_counts`]'s own
|
||||
/// docs for why: `sum` = the count itself, `any`/`max` = `count ≥ 1`,
|
||||
/// `all`/`min` = `count == group size`, `none` = `count == 0`) — no further
|
||||
/// matrix access needed once `count` is in hand.
|
||||
fn agg_result_from_count(op: AggOp, group_len: usize, count: TempCompactIntVec) -> io::Result<AggResult> {
|
||||
let n = count.len();
|
||||
let group_len = group_len as u32;
|
||||
Ok(match op {
|
||||
AggOp::Sum => AggResult::Int(count),
|
||||
AggOp::Any => {
|
||||
let mut b = TempBitVecBuilder::new(n)?;
|
||||
b.or_where(count.view(), |v| v >= 1);
|
||||
AggResult::Bit(b.freeze()?)
|
||||
}
|
||||
AggOp::All => {
|
||||
let mut b = TempBitVecBuilder::new(n)?;
|
||||
b.or_where(count.view(), |v| v == group_len);
|
||||
AggResult::Bit(b.freeze()?)
|
||||
}
|
||||
AggOp::None => {
|
||||
let mut b = TempBitVecBuilder::new(n)?;
|
||||
b.or_where(count.view(), |v| v == 0);
|
||||
AggResult::Bit(b.freeze()?)
|
||||
}
|
||||
AggOp::Min => {
|
||||
let mut b = TempCompactIntVecBuilder::new(n)?;
|
||||
b.inc_predicate_fast(count.view(), |v| v == group_len);
|
||||
AggResult::Int(b.freeze()?)
|
||||
}
|
||||
AggOp::Max => {
|
||||
let mut b = TempCompactIntVecBuilder::new(n)?;
|
||||
b.inc_predicate_fast(count.view(), |v| v >= 1);
|
||||
AggResult::Int(b.freeze()?)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ── AggResult → MatrixBuilder ─────────────────────────────────────────────────
|
||||
|
||||
/// Add one already-aggregated column to `mb` — the only piece `MatrixBuilder`
|
||||
@@ -161,15 +199,6 @@ pub(crate) fn select_partition(
|
||||
.to_path_buf();
|
||||
copy_layer_files(src_layer.dir(), &dst_layer_dir, force_copy).map_err(OKIError::Io)?;
|
||||
|
||||
let group_mat: Box<dyn MatrixGroupOps> = match src_layer.content() {
|
||||
LayerContent::Count => {
|
||||
Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
||||
}
|
||||
LayerContent::Presence => {
|
||||
Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?)
|
||||
}
|
||||
};
|
||||
|
||||
let data_subdir = if output_presence {
|
||||
"presence"
|
||||
} else {
|
||||
@@ -180,10 +209,37 @@ pub(crate) fn select_partition(
|
||||
|
||||
let mut builder =
|
||||
MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?;
|
||||
for spec in specs {
|
||||
let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?;
|
||||
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
||||
|
||||
match src_layer.content() {
|
||||
// One shared row-major (Sparse) / deduplicated column-major
|
||||
// (Columnar/Packed/Implicit) pass covers every spec at once —
|
||||
// see `batch_presence_counts`'s own docs. Every `AggOp` for a
|
||||
// bit-matrix source is a cheap derivation of that one count.
|
||||
LayerContent::Presence => {
|
||||
let mat = PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?;
|
||||
let groups: Vec<ColGroup> = specs
|
||||
.iter()
|
||||
.map(|s| ColGroup::new(s.label.clone(), s.indices.clone()))
|
||||
.collect();
|
||||
let counts = batch_presence_counts(&mat, &groups).map_err(OKIError::Io)?;
|
||||
for (spec, count) in specs.iter().zip(counts) {
|
||||
let r = agg_result_from_count(spec.op, spec.indices.len(), count)
|
||||
.map_err(OKIError::Io)?;
|
||||
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
||||
}
|
||||
}
|
||||
// Count-matrix `sum`/`min`/`max` are genuine per-value
|
||||
// reductions, not derivable from a single presence count —
|
||||
// unchanged, one `col_view`-driven pass per spec.
|
||||
LayerContent::Count => {
|
||||
let mat = PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?;
|
||||
for spec in specs {
|
||||
let r = compute_group(&mat, spec, threshold).map_err(OKIError::Io)?;
|
||||
add_result(&mut builder, r).map_err(OKIError::Io)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
builder.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user