feat: add batched int group stats API and expand benchmark variants

Introduces a `batch_int_group_stats` API for computing presence counts, sums, minimums, and maximums across sparse and dense matrix representations. The selection layer now utilizes this batched approach to optimize aggregation semantics for boolean and numeric operations. Additionally, reorganizes the benchmarking infrastructure to support querying across presence and count index variants in both dense and sparse formats, including new packing scripts and updated statistics aggregation.
This commit is contained in:
Eric Coissac committed 2026-08-28 23:37:16 +02:00
1 parent 579cfd8752
commit 54e4600120
13 files changed
+546 -163

No files matched your search

+51 -26
View File
@@ -1,18 +1,24 @@
//! Per-layer column projection/aggregation — the mechanics
//! [`crate::select::Select`] runs once per partition-layer. Generic over
//! source content (`Count`/`Presence`) via `obicompactvec::MatrixGroupOps`
//! (object-safe: one `&dyn MatrixGroupOps` covers both matrix kinds, so
//! this never branches on source content beyond the single `match` that
//! opens it) and over destination content via [`DstBuilder`] — no
//! duplicated per-op-per-content code path.
//! [`crate::select::Select`] runs once per partition-layer. `Presence` and
//! `Count` sources each get their own batched, row-major-friendly pass
//! (`obicompactvec::batch_presence_counts`/`batch_int_group_stats`)
//! computing every output spec's statistics together in one shared scan,
//! rather than the generic `MatrixGroupOps` per-group dispatch this used to
//! go through (still used by `obikfilter`, whose ad hoc `FilterMask`
//! column-index lists don't fit the same "one shared batch" shape) —
//! `sum`/`min`/`max` mean genuinely different things for the two content
//! kinds (a bit matrix's are cheap derivations of one presence count, a
//! count matrix's are real per-value reductions), so each gets its own
//! `agg_result_from_*` derivation, not a shared `AggOp` dispatch.
use std::fs;
use std::io;
use std::path::Path;
use obicompactvec::{
ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
TempBitVecBuilder, TempCompactIntVec, TempCompactIntVecBuilder, batch_presence_counts,
ColGroup, IntGroupStats, MatrixBuilder, PersistentBitMatrix, PersistentIntMatrix, TempBitVec,
TempBitVecBuilder, TempCompactIntVec, TempCompactIntVecBuilder, batch_int_group_stats,
batch_presence_counts,
};
use obikindex::layer::{KmerLayer, LayerContent};
use obikindex::{KmerIndex, OKIError, OKIResult};
@@ -77,22 +83,6 @@ enum AggResult {
Int(TempCompactIntVec),
}
fn compute_group(
mat: &dyn MatrixGroupOps,
spec: &OutputCol,
threshold: u32,
) -> io::Result<AggResult> {
let g = ColGroup::new(spec.label.clone(), spec.indices.clone());
Ok(match spec.op {
AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?),
AggOp::All => AggResult::Bit(mat.partial_group_all(&g, threshold)?),
AggOp::None => AggResult::Bit(mat.partial_group_none(&g, threshold)?),
AggOp::Sum => AggResult::Int(mat.partial_group_sum(&g)?),
AggOp::Min => AggResult::Int(mat.partial_group_min(&g)?),
AggOp::Max => AggResult::Int(mat.partial_group_max(&g)?),
})
}
/// A bit-matrix source's every `AggOp` is a cheap derivation of one shared
/// presence count (see [`obicompactvec::batch_presence_counts`]'s own
/// docs for why: `sum` = the count itself, `any`/`max` = `count ≥ 1`,
@@ -131,6 +121,35 @@ fn agg_result_from_count(op: AggOp, group_len: usize, count: TempCompactIntVec)
})
}
/// A count-matrix source's `AggOp` from [`IntGroupStats`] — unlike the
/// bit-matrix case, `sum`/`min`/`max` are genuine per-value reductions
/// already computed by `batch_int_group_stats`, not further derivations;
/// only `any`/`all`/`none` still need a threshold-count comparison here.
fn agg_result_from_int_stats(op: AggOp, group_len: usize, stats: IntGroupStats) -> io::Result<AggResult> {
let n = stats.presence_count.len();
let group_len = group_len as u32;
Ok(match op {
AggOp::Sum => AggResult::Int(stats.sum),
AggOp::Min => AggResult::Int(stats.min),
AggOp::Max => AggResult::Int(stats.max),
AggOp::Any => {
let mut b = TempBitVecBuilder::new(n)?;
b.or_where(stats.presence_count.view(), |v| v >= 1);
AggResult::Bit(b.freeze()?)
}
AggOp::All => {
let mut b = TempBitVecBuilder::new(n)?;
b.or_where(stats.presence_count.view(), |v| v == group_len);
AggResult::Bit(b.freeze()?)
}
AggOp::None => {
let mut b = TempBitVecBuilder::new(n)?;
b.or_where(stats.presence_count.view(), |v| v == 0);
AggResult::Bit(b.freeze()?)
}
})
}
// ── AggResult → MatrixBuilder ─────────────────────────────────────────────────
/// Add one already-aggregated column to `mb` — the only piece `MatrixBuilder`
@@ -233,8 +252,14 @@ pub(crate) fn select_partition(
// unchanged, one `col_view`-driven pass per spec.
LayerContent::Count => {
let mat = PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?;
for spec in specs {
let r = compute_group(&mat, spec, threshold).map_err(OKIError::Io)?;
let groups: Vec<ColGroup> = specs
.iter()
.map(|s| ColGroup::new(s.label.clone(), s.indices.clone()))
.collect();
let stats = batch_int_group_stats(&mat, &groups, threshold).map_err(OKIError::Io)?;
for (spec, stat) in specs.iter().zip(stats) {
let r = agg_result_from_int_stats(spec.op, spec.indices.len(), stat)
.map_err(OKIError::Io)?;
add_result(&mut builder, r).map_err(OKIError::Io)?;
}
}