Push punmmmpkuymy #83

Merged
coissac merged 4 commits from push-punmmmpkuymy into main 2026-11-20 13:24:07 +00:00
11 changed files with 121 additions and 15 deletions
Showing only changes of commit 6d6d5e5e53 - Show all commits

No files matched your search

@@ -304,6 +304,10 @@ impl PersistentBitMatrix {
impl ColumnWeights for PersistentBitMatrix { impl ColumnWeights for PersistentBitMatrix {
#[inline] #[inline]
fn col_weights(&self) -> Array1<u64> { self.count_ones() } fn col_weights(&self) -> Array1<u64> { self.count_ones() }
/// A presence matrix holds no counts: a set bit is present whatever
/// `threshold` is.
#[inline]
fn count_geq(&self, _threshold: u32) -> Array1<u64> { self.count_ones() }
} }
impl BitPartials for PersistentBitMatrix { impl BitPartials for PersistentBitMatrix {
@@ -559,6 +559,11 @@ impl ColumnWeights for PersistentSparseBitMatrix {
fn col_weights(&self) -> Array1<u64> { fn col_weights(&self) -> Array1<u64> {
self.count_ones() self.count_ones()
} }
/// A presence matrix holds no counts: a set bit is present whatever
/// `threshold` is.
fn count_geq(&self, _threshold: u32) -> Array1<u64> {
self.count_ones()
}
} }
impl BitPartials for PersistentSparseBitMatrix { impl BitPartials for PersistentSparseBitMatrix {
+19 -2
View File
@@ -71,6 +71,10 @@ impl ColumnarCompactIntMatrix {
par_col_reduce(self.n_cols(), |c| self.col(c).count_nonzero()) par_col_reduce(self.n_cols(), |c| self.col(c).count_nonzero())
} }
pub(crate) fn count_geq(&self, threshold: u32) -> Array1<u64> {
par_col_reduce(self.n_cols(), |c| self.col(c).count_geq(threshold))
}
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> { pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
pairwise_matrix(self.n_cols(), |i, j| { pairwise_matrix(self.n_cols(), |i, j| {
self.col(i).partial_bray_dist(self.col(j)) self.col(i).partial_bray_dist(self.col(j))
@@ -236,6 +240,10 @@ impl PackedCompactIntMatrix {
par_col_reduce(self.n_cols, |c| self.col_view(c).count_nonzero()) par_col_reduce(self.n_cols, |c| self.col_view(c).count_nonzero())
} }
pub(crate) fn count_geq(&self, threshold: u32) -> Array1<u64> {
par_col_reduce(self.n_cols, |c| self.col_view(c).count_geq(threshold))
}
// Every pair formula below delegates to `IntSliceView` (`views.rs`) — // Every pair formula below delegates to `IntSliceView` (`views.rs`) —
// the canonical implementation, also used by `ColumnarCompactIntMatrix` // the canonical implementation, also used by `ColumnarCompactIntMatrix`
// via `PersistentCompactIntVec`. No formula is re-derived here. // via `PersistentCompactIntVec`. No formula is re-derived here.
@@ -572,6 +580,15 @@ impl PersistentIntMatrix {
Self::Sparse(m) => m.count_nonzero(), Self::Sparse(m) => m.count_nonzero(),
} }
} }
/// Per-column count of values `>= threshold`.
#[inline]
pub fn count_geq(&self, threshold: u32) -> Array1<u64> {
match self {
Self::Columnar(m) => m.count_geq(threshold),
Self::Packed(m) => m.count_geq(threshold),
Self::Sparse(m) => m.count_geq(threshold),
}
}
#[inline] #[inline]
pub fn partial_bray_dist_matrix(&self) -> Array2<u64> { pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
match self { match self {
@@ -639,8 +656,8 @@ impl ColumnWeights for PersistentIntMatrix {
self.sum() self.sum()
} }
#[inline] #[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { fn count_geq(&self, threshold: u32) -> Array1<u64> {
self.count_nonzero() PersistentIntMatrix::count_geq(self, threshold)
} }
} }
+7
View File
@@ -124,6 +124,13 @@ impl PersistentCompactIntVec {
byte_count_nonzero(primary) byte_count_nonzero(primary)
} }
/// Number of slots whose value is `>= threshold` (see
/// [`IntSliceView::count_geq`]).
#[inline]
pub fn count_geq(&self, threshold: u32) -> u64 {
self.view().count_geq(threshold)
}
/// Lightweight zero-copy view — primary and overflow point into the mmap. /// Lightweight zero-copy view — primary and overflow point into the mmap.
#[inline] #[inline]
pub fn view(&self) -> IntSliceView<'_> { pub fn view(&self) -> IntSliceView<'_> {
+11 -9
View File
@@ -213,14 +213,16 @@ impl PersistentSparseCompactIntMatrix {
Array1::from(counts) Array1::from(counts)
} }
/// Per-column count of values `>= threshold` — the marginal /// Per-column count of values `>= threshold` — also the marginal
/// `partial_threshold_jaccard` needs both for its `union` formula /// `partial_threshold_jaccard` needs for its `union` formula
/// (`count_geq[i] + count_geq[j] - inter[i,j]`) and for its diagonal. /// (`count_geq[i] + count_geq[j] - inter[i,j]`) and for its diagonal.
/// `threshold == 1` is `count_nonzero()` exactly (`v >= 1 ⟺ v != 0`), /// `threshold == 1` is `count_nonzero()` exactly (`v >= 1 ⟺ v != 0`).
/// but this is only ever called with `threshold >= 1` by /// `threshold == 0` counts every slot (`n()` per column), as in the
/// [`partial_threshold_jaccard`](CountPartials::partial_threshold_jaccard) /// dense matrices; only stored cells are visited otherwise.
/// (the `threshold == 0` case is closed-form there, no marginal needed). pub fn count_geq(&self, threshold: u32) -> Array1<u64> {
fn count_geq(&self, threshold: u32) -> Array1<u64> { if threshold == 0 {
return Array1::from_elem(self.n_cols(), self.n() as u64);
}
let mut counts = vec![0u64; self.n_cols()]; let mut counts = vec![0u64; self.n_cols()];
for slot in 0..self.n() { for slot in 0..self.n() {
self.for_each_cell_in_row(slot, |g, v| { self.for_each_cell_in_row(slot, |g, v| {
@@ -289,8 +291,8 @@ impl ColumnWeights for PersistentSparseCompactIntMatrix {
self.sum() self.sum()
} }
#[inline] #[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { fn count_geq(&self, threshold: u32) -> Array1<u64> {
self.count_nonzero() PersistentSparseCompactIntMatrix::count_geq(self, threshold)
} }
} }
@@ -154,6 +154,23 @@ fn sum_and_count_nonzero_match_naive() {
assert_eq!(m.partial_kmer_counts().to_vec(), expected_count.to_vec()); assert_eq!(m.partial_kmer_counts().to_vec(), expected_count.to_vec());
} }
#[test]
fn count_geq_matches_naive_for_every_threshold() {
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
let (_dir, m) = make_sparse(&rows, 3);
for threshold in 0..=10u32 {
let mut expected = [0u64; 3];
for row in &rows {
for (c, &v) in row.iter().enumerate() {
if v >= threshold {
expected[c] += 1;
}
}
}
assert_eq!(m.count_geq(threshold).to_vec(), expected.to_vec(), "threshold {threshold}");
}
}
#[test] #[test]
fn single_genome_matrix() { fn single_genome_matrix() {
// n_cols=1 — every row is necessarily a singleton or empty. // n_cols=1 — every row is necessarily a singleton or empty.
+11 -4
View File
@@ -52,14 +52,21 @@ fn jaccard_to_mash(d_jaccard: &Array2<f64>, k: usize) -> Array2<f64> {
// ── Column-level weight statistic — total count or presence count per column. // ── Column-level weight statistic — total count or presence count per column.
/// Additive across layers and partitions; used as denominator in normalised distances. /// Additive across layers and partitions; used as denominator in normalised distances.
/// ///
/// `partial_kmer_counts` returns the number of **distinct k-mers** present per /// `count_geq(threshold)` returns the number of **distinct k-mers** present per
/// column (presence = 1 entries; count > 0 entries). For presence matrices this /// column, a k-mer being present when its value reaches `threshold` (so
/// equals `col_weights`; for count matrices it differs (count_nonzero vs sum). /// `threshold == 1` is "count > 0"). Presence matrices hold no counts: every
/// set bit is present whatever the threshold, and `count_geq` equals
/// `col_weights`. For count matrices `col_weights` is the sum of the counts
/// and differs. Additive across layers and partitions.
pub trait ColumnWeights: Send + Sync { pub trait ColumnWeights: Send + Sync {
fn col_weights(&self) -> Array1<u64>; fn col_weights(&self) -> Array1<u64>;
fn count_geq(&self, threshold: u32) -> Array1<u64>;
/// Distinct k-mers present per column, with the default presence
/// threshold of 1.
fn partial_kmer_counts(&self) -> Array1<u64> { fn partial_kmer_counts(&self) -> Array1<u64> {
self.col_weights() self.count_geq(1)
} }
} }
+11
View File
@@ -248,6 +248,17 @@ impl<'a> IntSliceView<'a> {
byte_count_nonzero(self.primary) byte_count_nonzero(self.primary)
} }
/// Number of slots whose value is `>= threshold`. `threshold == 1` is
/// [`count_nonzero`](Self::count_nonzero) (vectorised byte scan);
/// `threshold == 0` counts every slot.
pub fn count_geq(&self, threshold: u32) -> u64 {
match threshold {
0 => self.n as u64,
1 => self.count_nonzero(),
t => self.iter().filter(|&v| v >= t).count() as u64,
}
}
// ── Distance methods ────────────────────────────────────────────────────── // ── Distance methods ──────────────────────────────────────────────────────
// //
// `partial_bray_dist`, `partial_euclidean_dist` and // `partial_bray_dist`, `partial_euclidean_dist` and
+15
View File
@@ -254,6 +254,18 @@ impl KmerLayer {
} }
} }
/// Per-genome number of distinct k-mers present, a k-mer being present
/// when its count reaches `threshold` — presence layers hold no counts,
/// so `threshold` has no effect there (see
/// `obicompactvec::ColumnWeights::count_geq`).
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
match self {
KmerLayer::Count { layer, .. } => layer.count_geq(threshold),
KmerLayer::Presence { layer, .. } => layer.count_geq(threshold),
KmerLayer::Empty { .. } => panic!("Layer::count_geq() called on an Empty layer"),
}
}
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` = /// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must /// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
/// have one entry per genome column, each resized to `slots.len()`. /// have one entry per genome column, each resized to `slots.len()`.
@@ -365,6 +377,9 @@ impl ColumnWeights for KmerLayer {
fn col_weights(&self) -> Array1<u64> { fn col_weights(&self) -> Array1<u64> {
self.col_weights() self.col_weights()
} }
fn count_geq(&self, threshold: u32) -> Array1<u64> {
self.count_geq(threshold)
}
} }
impl CountPartials for KmerLayer { impl CountPartials for KmerLayer {
+13
View File
@@ -420,6 +420,12 @@ impl TypedLayer<PersistentIntMatrix> {
obicompactvec::ColumnWeights::col_weights(&self.data) obicompactvec::ColumnWeights::col_weights(&self.data)
} }
/// Per-genome number of k-mers whose count reaches `threshold` (see
/// `obicompactvec::ColumnWeights::count_geq`).
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
obicompactvec::ColumnWeights::count_geq(&self.data, threshold)
}
/// Every nonzero `(idx into slots, col, value)` triple among `slots` — /// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`. /// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
pub fn nonzero_iter<'a>( pub fn nonzero_iter<'a>(
@@ -502,6 +508,13 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
pub fn col_weights(&self) -> ndarray::Array1<u64> { pub fn col_weights(&self) -> ndarray::Array1<u64> {
obicompactvec::ColumnWeights::col_weights(&self.data) obicompactvec::ColumnWeights::col_weights(&self.data)
} }
/// Per-genome k-mer count; a presence matrix has no counts, so
/// `threshold` has no effect (see
/// `obicompactvec::ColumnWeights::count_geq`).
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
obicompactvec::ColumnWeights::count_geq(&self.data, threshold)
}
} }
impl TypedLayer<PersistentBitMatrix> { impl TypedLayer<PersistentBitMatrix> {
+8
View File
@@ -90,6 +90,10 @@ impl ColumnWeights for AggregatedCount<'_> {
let n = self.n_genomes; let n = self.n_genomes;
self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b) self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b)
} }
fn count_geq(&self, threshold: u32) -> Array1<u64> {
let n = self.n_genomes;
self.reduce(|| Array1::zeros(n), |l| l.count_geq(threshold), |a, b| a + b)
}
} }
impl CountPartials for AggregatedCount<'_> { impl CountPartials for AggregatedCount<'_> {
@@ -149,6 +153,10 @@ impl ColumnWeights for AggregatedBit<'_> {
let n = self.n_genomes; let n = self.n_genomes;
self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b) self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b)
} }
fn count_geq(&self, threshold: u32) -> Array1<u64> {
let n = self.n_genomes;
self.reduce(|| Array1::zeros(n), |l| l.count_geq(threshold), |a, b| a + b)
}
} }
impl BitPartials for AggregatedBit<'_> { impl BitPartials for AggregatedBit<'_> {