Remove in-place flag, optimize presence counting, and handle NUMA panics
The `obikmer select` CLI no longer supports in-place index rewriting; the `--output` flag is now required, with benchmarks updated to use temporary directories for atomic replacement. Added `--dense` and `--force-copy` flags. Introduced `batch_presence_counts` to compute presence counts across multiple column groups in a single pass, eliminating redundant I/O. Refactored the aggregation pipeline to branch on layer content, applying the optimized batched counting for `Presence` layers. Enhanced the NUMA runner to catch worker panics, track them, and re-raise after thread join to prevent indefinite blocking.
This commit is contained in:
1 parent
7363370065
commit
93fe838f97
9 files changed
+330
-26
No files matched your search
@@ -1,3 +1,4 @@
|
||||
use std::collections::HashMap;
|
||||
use std::io;
|
||||
|
||||
use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps};
|
||||
@@ -6,6 +7,110 @@ use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
|
||||
|
||||
use super::persistent::PersistentBitMatrix;
|
||||
|
||||
// ── batch_presence_counts ───────────────────────────────────────────────────
|
||||
//
|
||||
// One shared pass computing every group's presence count at once, instead
|
||||
// of `select`'s old per-spec loop over `MatrixGroupOps::partial_group_*`
|
||||
// (one independent `col_view`-driven pass per output column). Two reasons
|
||||
// this exists, not just one:
|
||||
//
|
||||
// - `Sparse` has no on-disk column representation at all (`col_view` is a
|
||||
// deliberate `panic!` there — see `PersistentBitMatrix::col_view`'s own
|
||||
// docs) — a real column, however cheap, could only ever be synthesised by
|
||||
// scanning every row, which is exactly the row-major work this function
|
||||
// already does directly, natively, via
|
||||
// `PersistentSparseBitMatrix::for_each_genome_in_row` (touches only the
|
||||
// columns actually present per row — 1 for a singleton row, k̄ for a
|
||||
// multi-genome row).
|
||||
// - Even for `Columnar`/`Packed`, doing this once for *every* group
|
||||
// together is strictly less work than the old per-group loop whenever two
|
||||
// groups share a column (`--aggregate-by`'s auto-groups are disjoint, but
|
||||
// nothing stops `--group` from overlapping) — a column referenced by `g`
|
||||
// groups is now read from disk once, not `g` times.
|
||||
//
|
||||
// For a bit matrix, presence count at threshold 1 is the *only* primitive
|
||||
// needed: `sum` (bit matrix: sum of 0/1 = presence count), `any`
|
||||
// (count ≥ 1), `max` (= any), `min`/`all` (count == group size) are all
|
||||
// cheap derivations from this one vector — see
|
||||
// `obikselect::select_layer::agg_result_from_count`, the caller this
|
||||
// exists for.
|
||||
pub fn batch_presence_counts(
|
||||
mat: &PersistentBitMatrix,
|
||||
groups: &[ColGroup],
|
||||
) -> io::Result<Vec<TempCompactIntVec>> {
|
||||
let n = mat.n();
|
||||
|
||||
// Column -> every group index that references it — built once, so a
|
||||
// shared column is attributed to every one of its groups from a single
|
||||
// read, whichever storage format this turns out to be.
|
||||
let mut col_to_groups: HashMap<usize, Vec<usize>> = HashMap::new();
|
||||
for (gi, g) in groups.iter().enumerate() {
|
||||
for &c in &g.indices {
|
||||
col_to_groups.entry(c).or_default().push(gi);
|
||||
}
|
||||
}
|
||||
|
||||
let mut builders: Vec<TempCompactIntVecBuilder> = groups
|
||||
.iter()
|
||||
.map(|_| TempCompactIntVecBuilder::new(n))
|
||||
.collect::<io::Result<_>>()?;
|
||||
|
||||
let bump = |builders: &mut [TempCompactIntVecBuilder], gi: usize, slot: usize| {
|
||||
let b = &mut builders[gi];
|
||||
let v = b.get(slot);
|
||||
b.set(slot, v + 1);
|
||||
};
|
||||
|
||||
match mat {
|
||||
PersistentBitMatrix::Sparse(m) => {
|
||||
// Native row-major decode — the whole reason this exists.
|
||||
for slot in 0..n {
|
||||
m.for_each_genome_in_row(slot, |col| {
|
||||
if let Some(gs) = col_to_groups.get(&col) {
|
||||
for &gi in gs {
|
||||
bump(&mut builders, gi, slot);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
PersistentBitMatrix::Implicit { n_rows, .. } => {
|
||||
// Single column (index 0), present at every slot.
|
||||
if let Some(gs) = col_to_groups.get(&0) {
|
||||
for &gi in gs {
|
||||
for slot in 0..*n_rows {
|
||||
bump(&mut builders, gi, slot);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PersistentBitMatrix::Columnar(_) | PersistentBitMatrix::Packed(_) => {
|
||||
// One `col_view` per *distinct referenced column*, not per
|
||||
// group — the common case (a column belongs to exactly one
|
||||
// group) reuses the existing fast bulk primitive directly;
|
||||
// only a column shared by several groups falls back to a
|
||||
// per-bit scan.
|
||||
for (&col, gs) in &col_to_groups {
|
||||
let view = mat.col_view(col);
|
||||
match gs.as_slice() {
|
||||
[gi] => builders[*gi].inc_present_fast(view),
|
||||
_ => {
|
||||
for (slot, present) in view.iter().enumerate() {
|
||||
if present {
|
||||
for &gi in gs {
|
||||
bump(&mut builders, gi, slot);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
builders.into_iter().map(|b| b.freeze()).collect()
|
||||
}
|
||||
|
||||
// ── MatrixGroupOps ────────────────────────────────────────────────────────────
|
||||
|
||||
impl MatrixGroupOps for PersistentBitMatrix {
|
||||
|
||||
@@ -20,6 +20,7 @@ mod persistent;
|
||||
mod sparse;
|
||||
|
||||
pub use builder::PersistentBitMatrixBuilder;
|
||||
pub use group_ops::batch_presence_counts;
|
||||
pub use packed::pack_bit_matrix;
|
||||
pub use persistent::PersistentBitMatrix;
|
||||
pub use sparse::{PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_sparse_bit_matrix};
|
||||
|
||||
@@ -21,7 +21,7 @@ mod views;
|
||||
|
||||
pub use bitmatrix::{
|
||||
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
|
||||
PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix,
|
||||
PersistentSparseBitMatrixBuilder, batch_presence_counts, pack_bit_matrix, pack_sparse_bit_matrix,
|
||||
};
|
||||
pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
|
||||
pub use builder::PersistentCompactIntVecBuilder;
|
||||
|
||||
@@ -3,6 +3,7 @@ use tempfile::tempdir;
|
||||
use crate::{
|
||||
ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder,
|
||||
PersistentIntMatrix, PersistentCompactIntMatrixBuilder,
|
||||
PersistentSparseBitMatrixBuilder, batch_presence_counts,
|
||||
};
|
||||
use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder};
|
||||
|
||||
@@ -234,3 +235,66 @@ fn int_presence_count_additive_across_split() {
|
||||
assert_eq!(pb.get(1), 2);
|
||||
assert_eq!(pb.get(2), 1);
|
||||
}
|
||||
|
||||
// ── batch_presence_counts: Columnar vs Sparse, disjoint and overlapping groups ──
|
||||
|
||||
/// `batch_presence_counts` must agree with the naive "count group members
|
||||
/// present at this row" definition, for both a `Columnar` (col_view-driven)
|
||||
/// and a `Sparse` (native row-major) source — and in particular must still
|
||||
/// be correct when two groups share a column (`g1`/`g3` share column 1,
|
||||
/// `g2`/`g3` share column 2), the case that forces the per-bit fallback
|
||||
/// instead of the single-target `inc_present_fast` fast path.
|
||||
#[test]
|
||||
fn batch_presence_counts_matches_naive_for_columnar_and_sparse() {
|
||||
// Column-major: cols[c][row]
|
||||
let cols: [&[bool]; 4] = [
|
||||
&[true, false, true, false, true], // col0
|
||||
&[true, true, false, false, true], // col1
|
||||
&[false, true, true, true, false], // col2
|
||||
&[false, false, false, true, true], // col3
|
||||
];
|
||||
let (_dcol, columnar) = make_bit_matrix(&cols);
|
||||
|
||||
// Same data, row-major, for the sparse builder.
|
||||
let n_rows = 5;
|
||||
let n_cols = 4;
|
||||
let dir = tempdir().unwrap();
|
||||
let sparse_dir = dir.path().join("sparse");
|
||||
let mut b = PersistentSparseBitMatrixBuilder::new(n_rows, n_cols, &sparse_dir).unwrap();
|
||||
let mut genomes = Vec::new();
|
||||
for row in 0..n_rows {
|
||||
genomes.clear();
|
||||
genomes.extend((0..n_cols).filter(|&c| cols[c][row]).map(|c| c as u32));
|
||||
b.push_row(&genomes);
|
||||
}
|
||||
let sparse = PersistentBitMatrix::Sparse(b.finish().unwrap());
|
||||
|
||||
let groups = [
|
||||
ColGroup::new("g1", vec![0, 1]),
|
||||
ColGroup::new("g2", vec![2, 3]),
|
||||
ColGroup::new("g3", vec![1, 2]), // overlaps g1 (col1) and g2 (col2)
|
||||
];
|
||||
|
||||
let expected: [[u32; 5]; 3] = [
|
||||
[2, 1, 1, 0, 2], // g1: col0+col1
|
||||
[0, 1, 1, 2, 1], // g2: col2+col3
|
||||
[1, 2, 1, 1, 1], // g3: col1+col2
|
||||
];
|
||||
|
||||
for (label, mat) in [("columnar", &columnar), ("sparse", &sparse)] {
|
||||
let counts = batch_presence_counts(mat, &groups).unwrap();
|
||||
assert_eq!(counts.len(), groups.len(), "{label}: wrong number of results");
|
||||
for (gi, exp) in expected.iter().enumerate() {
|
||||
for (row, &e) in exp.iter().enumerate() {
|
||||
assert_eq!(counts[gi].get(row), e, "{label}: group {gi} row {row}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_presence_counts_empty_groups_is_empty() {
|
||||
let (_d, m) = make_bit_matrix(&[&[true, false], &[false, true]]);
|
||||
let counts = batch_presence_counts(&m, &[]).unwrap();
|
||||
assert!(counts.is_empty());
|
||||
}
|
||||
Reference in new issue
Block a user