From 6d6d5e5e5332655e648f7762800cff08b063449d Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 19 Nov 2026 16:04:54 +0100 Subject: [PATCH] feat: add count_geq method to calculate column counts above a threshold Introduce `count_geq` across the codebase to return the number of k-mers present in columns with counts greater than or equal to a specified threshold. This unifies the interface for counting distinct k-mers and updates `partial_kmer_counts` to rely on `count_geq(1)`. - Add `count_geq` method to `PersistentBitMatrix` in `src/obicompactvec/src/bitmatrix/persistent.rs`, delegating to `count_ones`. - Implement `ColumnWeights` trait for `PersistentBitMatrix` and add `count_geq` to `PersistentSparseBitMatrix` in `src/obicompactvec/src/bitmatrix/sparse.rs`. - Add `count_geq` methods to `IntMatrix` variants (`Columnar`, `Packed`, `Sparse`) and `PersistentIntMatrix` in `src/obicompactvec/src/intmatrix.rs`, removing `partial_kmer_counts`. - Implement `count_geq` for `SparseCompactIntMatrix` in `src/obicompactvec/src/sparse_intmatrix.rs`, handling threshold 0 explicitly, and remove `partial_kmer_counts`. - Add `count_geq` method to `Reader` struct in `src/obicompactvec/src/reader.rs` with documentation. - Update `ColumnWeights` trait in `src/obicompactvec/src/traits.rs` to include `count_geq` and redefine `partial_kmer_counts` as `count_geq(1)`. - Add `count_geq` method to `Views` struct in `src/obicompactvec/src/views.rs` for distance calculations. - Add `count_geq` tests in `src/obicompactvec/src/tests/sparse_intmatrix.rs` verifying behavior against naive implementations. - Add `count_geq` to `ContentLayer` in `src/obikindex/src/layer/content_layer.rs`, delegating based on layer type. - Add `count_geq` to `TypedLayer` in `src/obikindex/src/layer/typed_layer.rs`, delegating to `obicompactvec::ColumnWeights`. - Add `count_geq` method to distance struct in `src/obikphylo/src/distance.rs` using `self.reduce`. --- src/obicompactvec/src/bitmatrix/persistent.rs | 4 ++++ src/obicompactvec/src/bitmatrix/sparse.rs | 5 +++++ src/obicompactvec/src/intmatrix.rs | 21 +++++++++++++++++-- src/obicompactvec/src/reader.rs | 7 +++++++ src/obicompactvec/src/sparse_intmatrix.rs | 20 ++++++++++-------- .../src/tests/sparse_intmatrix.rs | 17 +++++++++++++++ src/obicompactvec/src/traits.rs | 15 +++++++++---- src/obicompactvec/src/views.rs | 11 ++++++++++ src/obikindex/src/layer/content_layer.rs | 15 +++++++++++++ src/obikindex/src/layer/typed_layer.rs | 13 ++++++++++++ src/obikphylo/src/distance.rs | 8 +++++++ 11 files changed, 121 insertions(+), 15 deletions(-) diff --git a/src/obicompactvec/src/bitmatrix/persistent.rs b/src/obicompactvec/src/bitmatrix/persistent.rs index dacfdf99..59d5025d 100644 --- a/src/obicompactvec/src/bitmatrix/persistent.rs +++ b/src/obicompactvec/src/bitmatrix/persistent.rs @@ -304,6 +304,10 @@ impl PersistentBitMatrix { impl ColumnWeights for PersistentBitMatrix { #[inline] fn col_weights(&self) -> Array1 { self.count_ones() } + /// A presence matrix holds no counts: a set bit is present whatever + /// `threshold` is. + #[inline] + fn count_geq(&self, _threshold: u32) -> Array1 { self.count_ones() } } impl BitPartials for PersistentBitMatrix { diff --git a/src/obicompactvec/src/bitmatrix/sparse.rs b/src/obicompactvec/src/bitmatrix/sparse.rs index 8d19d55a..f6ff3034 100644 --- a/src/obicompactvec/src/bitmatrix/sparse.rs +++ b/src/obicompactvec/src/bitmatrix/sparse.rs @@ -559,6 +559,11 @@ impl ColumnWeights for PersistentSparseBitMatrix { fn col_weights(&self) -> Array1 { self.count_ones() } + /// A presence matrix holds no counts: a set bit is present whatever + /// `threshold` is. + fn count_geq(&self, _threshold: u32) -> Array1 { + self.count_ones() + } } impl BitPartials for PersistentSparseBitMatrix { diff --git a/src/obicompactvec/src/intmatrix.rs b/src/obicompactvec/src/intmatrix.rs index ea9f664b..b27d4c24 100644 --- a/src/obicompactvec/src/intmatrix.rs +++ b/src/obicompactvec/src/intmatrix.rs @@ -71,6 +71,10 @@ impl ColumnarCompactIntMatrix { par_col_reduce(self.n_cols(), |c| self.col(c).count_nonzero()) } + pub(crate) fn count_geq(&self, threshold: u32) -> Array1 { + par_col_reduce(self.n_cols(), |c| self.col(c).count_geq(threshold)) + } + pub(crate) fn partial_bray_dist_matrix(&self) -> Array2 { pairwise_matrix(self.n_cols(), |i, j| { self.col(i).partial_bray_dist(self.col(j)) @@ -236,6 +240,10 @@ impl PackedCompactIntMatrix { par_col_reduce(self.n_cols, |c| self.col_view(c).count_nonzero()) } + pub(crate) fn count_geq(&self, threshold: u32) -> Array1 { + par_col_reduce(self.n_cols, |c| self.col_view(c).count_geq(threshold)) + } + // Every pair formula below delegates to `IntSliceView` (`views.rs`) — // the canonical implementation, also used by `ColumnarCompactIntMatrix` // via `PersistentCompactIntVec`. No formula is re-derived here. @@ -572,6 +580,15 @@ impl PersistentIntMatrix { Self::Sparse(m) => m.count_nonzero(), } } + /// Per-column count of values `>= threshold`. + #[inline] + pub fn count_geq(&self, threshold: u32) -> Array1 { + match self { + Self::Columnar(m) => m.count_geq(threshold), + Self::Packed(m) => m.count_geq(threshold), + Self::Sparse(m) => m.count_geq(threshold), + } + } #[inline] pub fn partial_bray_dist_matrix(&self) -> Array2 { match self { @@ -639,8 +656,8 @@ impl ColumnWeights for PersistentIntMatrix { self.sum() } #[inline] - fn partial_kmer_counts(&self) -> Array1 { - self.count_nonzero() + fn count_geq(&self, threshold: u32) -> Array1 { + PersistentIntMatrix::count_geq(self, threshold) } } diff --git a/src/obicompactvec/src/reader.rs b/src/obicompactvec/src/reader.rs index e69ae0ec..87247a40 100644 --- a/src/obicompactvec/src/reader.rs +++ b/src/obicompactvec/src/reader.rs @@ -124,6 +124,13 @@ impl PersistentCompactIntVec { byte_count_nonzero(primary) } + /// Number of slots whose value is `>= threshold` (see + /// [`IntSliceView::count_geq`]). + #[inline] + pub fn count_geq(&self, threshold: u32) -> u64 { + self.view().count_geq(threshold) + } + /// Lightweight zero-copy view — primary and overflow point into the mmap. #[inline] pub fn view(&self) -> IntSliceView<'_> { diff --git a/src/obicompactvec/src/sparse_intmatrix.rs b/src/obicompactvec/src/sparse_intmatrix.rs index 2061b4f0..39005225 100644 --- a/src/obicompactvec/src/sparse_intmatrix.rs +++ b/src/obicompactvec/src/sparse_intmatrix.rs @@ -213,14 +213,16 @@ impl PersistentSparseCompactIntMatrix { Array1::from(counts) } - /// Per-column count of values `>= threshold` — the marginal - /// `partial_threshold_jaccard` needs both for its `union` formula + /// Per-column count of values `>= threshold` — also the marginal + /// `partial_threshold_jaccard` needs for its `union` formula /// (`count_geq[i] + count_geq[j] - inter[i,j]`) and for its diagonal. - /// `threshold == 1` is `count_nonzero()` exactly (`v >= 1 ⟺ v != 0`), - /// but this is only ever called with `threshold >= 1` by - /// [`partial_threshold_jaccard`](CountPartials::partial_threshold_jaccard) - /// (the `threshold == 0` case is closed-form there, no marginal needed). - fn count_geq(&self, threshold: u32) -> Array1 { + /// `threshold == 1` is `count_nonzero()` exactly (`v >= 1 ⟺ v != 0`). + /// `threshold == 0` counts every slot (`n()` per column), as in the + /// dense matrices; only stored cells are visited otherwise. + pub fn count_geq(&self, threshold: u32) -> Array1 { + if threshold == 0 { + return Array1::from_elem(self.n_cols(), self.n() as u64); + } let mut counts = vec![0u64; self.n_cols()]; for slot in 0..self.n() { self.for_each_cell_in_row(slot, |g, v| { @@ -289,8 +291,8 @@ impl ColumnWeights for PersistentSparseCompactIntMatrix { self.sum() } #[inline] - fn partial_kmer_counts(&self) -> Array1 { - self.count_nonzero() + fn count_geq(&self, threshold: u32) -> Array1 { + PersistentSparseCompactIntMatrix::count_geq(self, threshold) } } diff --git a/src/obicompactvec/src/tests/sparse_intmatrix.rs b/src/obicompactvec/src/tests/sparse_intmatrix.rs index 58a656c2..ac12c9d1 100644 --- a/src/obicompactvec/src/tests/sparse_intmatrix.rs +++ b/src/obicompactvec/src/tests/sparse_intmatrix.rs @@ -154,6 +154,23 @@ fn sum_and_count_nonzero_match_naive() { assert_eq!(m.partial_kmer_counts().to_vec(), expected_count.to_vec()); } +#[test] +fn count_geq_matches_naive_for_every_threshold() { + let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]]; + let (_dir, m) = make_sparse(&rows, 3); + for threshold in 0..=10u32 { + let mut expected = [0u64; 3]; + for row in &rows { + for (c, &v) in row.iter().enumerate() { + if v >= threshold { + expected[c] += 1; + } + } + } + assert_eq!(m.count_geq(threshold).to_vec(), expected.to_vec(), "threshold {threshold}"); + } +} + #[test] fn single_genome_matrix() { // n_cols=1 — every row is necessarily a singleton or empty. diff --git a/src/obicompactvec/src/traits.rs b/src/obicompactvec/src/traits.rs index 934acf22..94a4b3b3 100644 --- a/src/obicompactvec/src/traits.rs +++ b/src/obicompactvec/src/traits.rs @@ -52,14 +52,21 @@ fn jaccard_to_mash(d_jaccard: &Array2, k: usize) -> Array2 { // ── Column-level weight statistic — total count or presence count per column. /// Additive across layers and partitions; used as denominator in normalised distances. /// -/// `partial_kmer_counts` returns the number of **distinct k-mers** present per -/// column (presence = 1 entries; count > 0 entries). For presence matrices this -/// equals `col_weights`; for count matrices it differs (count_nonzero vs sum). +/// `count_geq(threshold)` returns the number of **distinct k-mers** present per +/// column, a k-mer being present when its value reaches `threshold` (so +/// `threshold == 1` is "count > 0"). Presence matrices hold no counts: every +/// set bit is present whatever the threshold, and `count_geq` equals +/// `col_weights`. For count matrices `col_weights` is the sum of the counts +/// and differs. Additive across layers and partitions. pub trait ColumnWeights: Send + Sync { fn col_weights(&self) -> Array1; + fn count_geq(&self, threshold: u32) -> Array1; + + /// Distinct k-mers present per column, with the default presence + /// threshold of 1. fn partial_kmer_counts(&self) -> Array1 { - self.col_weights() + self.count_geq(1) } } diff --git a/src/obicompactvec/src/views.rs b/src/obicompactvec/src/views.rs index c0672b65..a66ebdbf 100644 --- a/src/obicompactvec/src/views.rs +++ b/src/obicompactvec/src/views.rs @@ -248,6 +248,17 @@ impl<'a> IntSliceView<'a> { byte_count_nonzero(self.primary) } + /// Number of slots whose value is `>= threshold`. `threshold == 1` is + /// [`count_nonzero`](Self::count_nonzero) (vectorised byte scan); + /// `threshold == 0` counts every slot. + pub fn count_geq(&self, threshold: u32) -> u64 { + match threshold { + 0 => self.n as u64, + 1 => self.count_nonzero(), + t => self.iter().filter(|&v| v >= t).count() as u64, + } + } + // ── Distance methods ────────────────────────────────────────────────────── // // `partial_bray_dist`, `partial_euclidean_dist` and diff --git a/src/obikindex/src/layer/content_layer.rs b/src/obikindex/src/layer/content_layer.rs index 610fcf36..b6132607 100644 --- a/src/obikindex/src/layer/content_layer.rs +++ b/src/obikindex/src/layer/content_layer.rs @@ -254,6 +254,18 @@ impl KmerLayer { } } + /// Per-genome number of distinct k-mers present, a k-mer being present + /// when its count reaches `threshold` — presence layers hold no counts, + /// so `threshold` has no effect there (see + /// `obicompactvec::ColumnWeights::count_geq`). + pub fn count_geq(&self, threshold: u32) -> ndarray::Array1 { + match self { + KmerLayer::Count { layer, .. } => layer.count_geq(threshold), + KmerLayer::Presence { layer, .. } => layer.count_geq(threshold), + KmerLayer::Empty { .. } => panic!("Layer::count_geq() called on an Empty layer"), + } + } + /// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` = /// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must /// have one entry per genome column, each resized to `slots.len()`. @@ -365,6 +377,9 @@ impl ColumnWeights for KmerLayer { fn col_weights(&self) -> Array1 { self.col_weights() } + fn count_geq(&self, threshold: u32) -> Array1 { + self.count_geq(threshold) + } } impl CountPartials for KmerLayer { diff --git a/src/obikindex/src/layer/typed_layer.rs b/src/obikindex/src/layer/typed_layer.rs index a0bfce12..82fc9bfd 100644 --- a/src/obikindex/src/layer/typed_layer.rs +++ b/src/obikindex/src/layer/typed_layer.rs @@ -420,6 +420,12 @@ impl TypedLayer { obicompactvec::ColumnWeights::col_weights(&self.data) } + /// Per-genome number of k-mers whose count reaches `threshold` (see + /// `obicompactvec::ColumnWeights::count_geq`). + pub fn count_geq(&self, threshold: u32) -> ndarray::Array1 { + obicompactvec::ColumnWeights::count_geq(&self.data, threshold) + } + /// Every nonzero `(idx into slots, col, value)` triple among `slots` — /// delegates to `PersistentCompactIntMatrix::nonzero_iter`. pub fn nonzero_iter<'a>( @@ -502,6 +508,13 @@ impl> + BinaryMatrix + obicompactvec::ColumnWeig pub fn col_weights(&self) -> ndarray::Array1 { obicompactvec::ColumnWeights::col_weights(&self.data) } + + /// Per-genome k-mer count; a presence matrix has no counts, so + /// `threshold` has no effect (see + /// `obicompactvec::ColumnWeights::count_geq`). + pub fn count_geq(&self, threshold: u32) -> ndarray::Array1 { + obicompactvec::ColumnWeights::count_geq(&self.data, threshold) + } } impl TypedLayer { diff --git a/src/obikphylo/src/distance.rs b/src/obikphylo/src/distance.rs index 19e67ac3..d1259634 100644 --- a/src/obikphylo/src/distance.rs +++ b/src/obikphylo/src/distance.rs @@ -90,6 +90,10 @@ impl ColumnWeights for AggregatedCount<'_> { let n = self.n_genomes; self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b) } + fn count_geq(&self, threshold: u32) -> Array1 { + let n = self.n_genomes; + self.reduce(|| Array1::zeros(n), |l| l.count_geq(threshold), |a, b| a + b) + } } impl CountPartials for AggregatedCount<'_> { @@ -149,6 +153,10 @@ impl ColumnWeights for AggregatedBit<'_> { let n = self.n_genomes; self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b) } + fn count_geq(&self, threshold: u32) -> Array1 { + let n = self.n_genomes; + self.reduce(|| Array1::zeros(n), |l| l.count_geq(threshold), |a, b| a + b) + } } impl BitPartials for AggregatedBit<'_> {