Push punmmmpkuymy #83
No files matched your search
@@ -304,6 +304,10 @@ impl PersistentBitMatrix {
|
|||||||
impl ColumnWeights for PersistentBitMatrix {
|
impl ColumnWeights for PersistentBitMatrix {
|
||||||
#[inline]
|
#[inline]
|
||||||
fn col_weights(&self) -> Array1<u64> { self.count_ones() }
|
fn col_weights(&self) -> Array1<u64> { self.count_ones() }
|
||||||
|
/// A presence matrix holds no counts: a set bit is present whatever
|
||||||
|
/// `threshold` is.
|
||||||
|
#[inline]
|
||||||
|
fn count_geq(&self, _threshold: u32) -> Array1<u64> { self.count_ones() }
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BitPartials for PersistentBitMatrix {
|
impl BitPartials for PersistentBitMatrix {
|
||||||
|
|||||||
@@ -559,6 +559,11 @@ impl ColumnWeights for PersistentSparseBitMatrix {
|
|||||||
fn col_weights(&self) -> Array1<u64> {
|
fn col_weights(&self) -> Array1<u64> {
|
||||||
self.count_ones()
|
self.count_ones()
|
||||||
}
|
}
|
||||||
|
/// A presence matrix holds no counts: a set bit is present whatever
|
||||||
|
/// `threshold` is.
|
||||||
|
fn count_geq(&self, _threshold: u32) -> Array1<u64> {
|
||||||
|
self.count_ones()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BitPartials for PersistentSparseBitMatrix {
|
impl BitPartials for PersistentSparseBitMatrix {
|
||||||
|
|||||||
@@ -71,6 +71,10 @@ impl ColumnarCompactIntMatrix {
|
|||||||
par_col_reduce(self.n_cols(), |c| self.col(c).count_nonzero())
|
par_col_reduce(self.n_cols(), |c| self.col(c).count_nonzero())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
|
par_col_reduce(self.n_cols(), |c| self.col(c).count_geq(threshold))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
pub(crate) fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
||||||
pairwise_matrix(self.n_cols(), |i, j| {
|
pairwise_matrix(self.n_cols(), |i, j| {
|
||||||
self.col(i).partial_bray_dist(self.col(j))
|
self.col(i).partial_bray_dist(self.col(j))
|
||||||
@@ -236,6 +240,10 @@ impl PackedCompactIntMatrix {
|
|||||||
par_col_reduce(self.n_cols, |c| self.col_view(c).count_nonzero())
|
par_col_reduce(self.n_cols, |c| self.col_view(c).count_nonzero())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
|
par_col_reduce(self.n_cols, |c| self.col_view(c).count_geq(threshold))
|
||||||
|
}
|
||||||
|
|
||||||
// Every pair formula below delegates to `IntSliceView` (`views.rs`) —
|
// Every pair formula below delegates to `IntSliceView` (`views.rs`) —
|
||||||
// the canonical implementation, also used by `ColumnarCompactIntMatrix`
|
// the canonical implementation, also used by `ColumnarCompactIntMatrix`
|
||||||
// via `PersistentCompactIntVec`. No formula is re-derived here.
|
// via `PersistentCompactIntVec`. No formula is re-derived here.
|
||||||
@@ -572,6 +580,15 @@ impl PersistentIntMatrix {
|
|||||||
Self::Sparse(m) => m.count_nonzero(),
|
Self::Sparse(m) => m.count_nonzero(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/// Per-column count of values `>= threshold`.
|
||||||
|
#[inline]
|
||||||
|
pub fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
|
match self {
|
||||||
|
Self::Columnar(m) => m.count_geq(threshold),
|
||||||
|
Self::Packed(m) => m.count_geq(threshold),
|
||||||
|
Self::Sparse(m) => m.count_geq(threshold),
|
||||||
|
}
|
||||||
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
|
||||||
match self {
|
match self {
|
||||||
@@ -639,8 +656,8 @@ impl ColumnWeights for PersistentIntMatrix {
|
|||||||
self.sum()
|
self.sum()
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> {
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
self.count_nonzero()
|
PersistentIntMatrix::count_geq(self, threshold)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -124,6 +124,13 @@ impl PersistentCompactIntVec {
|
|||||||
byte_count_nonzero(primary)
|
byte_count_nonzero(primary)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of slots whose value is `>= threshold` (see
|
||||||
|
/// [`IntSliceView::count_geq`]).
|
||||||
|
#[inline]
|
||||||
|
pub fn count_geq(&self, threshold: u32) -> u64 {
|
||||||
|
self.view().count_geq(threshold)
|
||||||
|
}
|
||||||
|
|
||||||
/// Lightweight zero-copy view — primary and overflow point into the mmap.
|
/// Lightweight zero-copy view — primary and overflow point into the mmap.
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn view(&self) -> IntSliceView<'_> {
|
pub fn view(&self) -> IntSliceView<'_> {
|
||||||
|
|||||||
@@ -213,14 +213,16 @@ impl PersistentSparseCompactIntMatrix {
|
|||||||
Array1::from(counts)
|
Array1::from(counts)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Per-column count of values `>= threshold` — the marginal
|
/// Per-column count of values `>= threshold` — also the marginal
|
||||||
/// `partial_threshold_jaccard` needs both for its `union` formula
|
/// `partial_threshold_jaccard` needs for its `union` formula
|
||||||
/// (`count_geq[i] + count_geq[j] - inter[i,j]`) and for its diagonal.
|
/// (`count_geq[i] + count_geq[j] - inter[i,j]`) and for its diagonal.
|
||||||
/// `threshold == 1` is `count_nonzero()` exactly (`v >= 1 ⟺ v != 0`),
|
/// `threshold == 1` is `count_nonzero()` exactly (`v >= 1 ⟺ v != 0`).
|
||||||
/// but this is only ever called with `threshold >= 1` by
|
/// `threshold == 0` counts every slot (`n()` per column), as in the
|
||||||
/// [`partial_threshold_jaccard`](CountPartials::partial_threshold_jaccard)
|
/// dense matrices; only stored cells are visited otherwise.
|
||||||
/// (the `threshold == 0` case is closed-form there, no marginal needed).
|
pub fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
if threshold == 0 {
|
||||||
|
return Array1::from_elem(self.n_cols(), self.n() as u64);
|
||||||
|
}
|
||||||
let mut counts = vec![0u64; self.n_cols()];
|
let mut counts = vec![0u64; self.n_cols()];
|
||||||
for slot in 0..self.n() {
|
for slot in 0..self.n() {
|
||||||
self.for_each_cell_in_row(slot, |g, v| {
|
self.for_each_cell_in_row(slot, |g, v| {
|
||||||
@@ -289,8 +291,8 @@ impl ColumnWeights for PersistentSparseCompactIntMatrix {
|
|||||||
self.sum()
|
self.sum()
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> {
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
self.count_nonzero()
|
PersistentSparseCompactIntMatrix::count_geq(self, threshold)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -154,6 +154,23 @@ fn sum_and_count_nonzero_match_naive() {
|
|||||||
assert_eq!(m.partial_kmer_counts().to_vec(), expected_count.to_vec());
|
assert_eq!(m.partial_kmer_counts().to_vec(), expected_count.to_vec());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn count_geq_matches_naive_for_every_threshold() {
|
||||||
|
let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]];
|
||||||
|
let (_dir, m) = make_sparse(&rows, 3);
|
||||||
|
for threshold in 0..=10u32 {
|
||||||
|
let mut expected = [0u64; 3];
|
||||||
|
for row in &rows {
|
||||||
|
for (c, &v) in row.iter().enumerate() {
|
||||||
|
if v >= threshold {
|
||||||
|
expected[c] += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(m.count_geq(threshold).to_vec(), expected.to_vec(), "threshold {threshold}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn single_genome_matrix() {
|
fn single_genome_matrix() {
|
||||||
// n_cols=1 — every row is necessarily a singleton or empty.
|
// n_cols=1 — every row is necessarily a singleton or empty.
|
||||||
|
|||||||
@@ -52,14 +52,21 @@ fn jaccard_to_mash(d_jaccard: &Array2<f64>, k: usize) -> Array2<f64> {
|
|||||||
// ── Column-level weight statistic — total count or presence count per column.
|
// ── Column-level weight statistic — total count or presence count per column.
|
||||||
/// Additive across layers and partitions; used as denominator in normalised distances.
|
/// Additive across layers and partitions; used as denominator in normalised distances.
|
||||||
///
|
///
|
||||||
/// `partial_kmer_counts` returns the number of **distinct k-mers** present per
|
/// `count_geq(threshold)` returns the number of **distinct k-mers** present per
|
||||||
/// column (presence = 1 entries; count > 0 entries). For presence matrices this
|
/// column, a k-mer being present when its value reaches `threshold` (so
|
||||||
/// equals `col_weights`; for count matrices it differs (count_nonzero vs sum).
|
/// `threshold == 1` is "count > 0"). Presence matrices hold no counts: every
|
||||||
|
/// set bit is present whatever the threshold, and `count_geq` equals
|
||||||
|
/// `col_weights`. For count matrices `col_weights` is the sum of the counts
|
||||||
|
/// and differs. Additive across layers and partitions.
|
||||||
pub trait ColumnWeights: Send + Sync {
|
pub trait ColumnWeights: Send + Sync {
|
||||||
fn col_weights(&self) -> Array1<u64>;
|
fn col_weights(&self) -> Array1<u64>;
|
||||||
|
|
||||||
|
fn count_geq(&self, threshold: u32) -> Array1<u64>;
|
||||||
|
|
||||||
|
/// Distinct k-mers present per column, with the default presence
|
||||||
|
/// threshold of 1.
|
||||||
fn partial_kmer_counts(&self) -> Array1<u64> {
|
fn partial_kmer_counts(&self) -> Array1<u64> {
|
||||||
self.col_weights()
|
self.count_geq(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -248,6 +248,17 @@ impl<'a> IntSliceView<'a> {
|
|||||||
byte_count_nonzero(self.primary)
|
byte_count_nonzero(self.primary)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of slots whose value is `>= threshold`. `threshold == 1` is
|
||||||
|
/// [`count_nonzero`](Self::count_nonzero) (vectorised byte scan);
|
||||||
|
/// `threshold == 0` counts every slot.
|
||||||
|
pub fn count_geq(&self, threshold: u32) -> u64 {
|
||||||
|
match threshold {
|
||||||
|
0 => self.n as u64,
|
||||||
|
1 => self.count_nonzero(),
|
||||||
|
t => self.iter().filter(|&v| v >= t).count() as u64,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Distance methods ──────────────────────────────────────────────────────
|
// ── Distance methods ──────────────────────────────────────────────────────
|
||||||
//
|
//
|
||||||
// `partial_bray_dist`, `partial_euclidean_dist` and
|
// `partial_bray_dist`, `partial_euclidean_dist` and
|
||||||
|
|||||||
@@ -254,6 +254,18 @@ impl KmerLayer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-genome number of distinct k-mers present, a k-mer being present
|
||||||
|
/// when its count reaches `threshold` — presence layers hold no counts,
|
||||||
|
/// so `threshold` has no effect there (see
|
||||||
|
/// `obicompactvec::ColumnWeights::count_geq`).
|
||||||
|
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
|
||||||
|
match self {
|
||||||
|
KmerLayer::Count { layer, .. } => layer.count_geq(threshold),
|
||||||
|
KmerLayer::Presence { layer, .. } => layer.count_geq(threshold),
|
||||||
|
KmerLayer::Empty { .. } => panic!("Layer::count_geq() called on an Empty layer"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
|
||||||
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
|
||||||
/// have one entry per genome column, each resized to `slots.len()`.
|
/// have one entry per genome column, each resized to `slots.len()`.
|
||||||
@@ -365,6 +377,9 @@ impl ColumnWeights for KmerLayer {
|
|||||||
fn col_weights(&self) -> Array1<u64> {
|
fn col_weights(&self) -> Array1<u64> {
|
||||||
self.col_weights()
|
self.col_weights()
|
||||||
}
|
}
|
||||||
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
|
self.count_geq(threshold)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountPartials for KmerLayer {
|
impl CountPartials for KmerLayer {
|
||||||
|
|||||||
@@ -420,6 +420,12 @@ impl TypedLayer<PersistentIntMatrix> {
|
|||||||
obicompactvec::ColumnWeights::col_weights(&self.data)
|
obicompactvec::ColumnWeights::col_weights(&self.data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-genome number of k-mers whose count reaches `threshold` (see
|
||||||
|
/// `obicompactvec::ColumnWeights::count_geq`).
|
||||||
|
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
|
||||||
|
obicompactvec::ColumnWeights::count_geq(&self.data, threshold)
|
||||||
|
}
|
||||||
|
|
||||||
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
|
||||||
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
|
/// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
|
||||||
pub fn nonzero_iter<'a>(
|
pub fn nonzero_iter<'a>(
|
||||||
@@ -502,6 +508,13 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
|
|||||||
pub fn col_weights(&self) -> ndarray::Array1<u64> {
|
pub fn col_weights(&self) -> ndarray::Array1<u64> {
|
||||||
obicompactvec::ColumnWeights::col_weights(&self.data)
|
obicompactvec::ColumnWeights::col_weights(&self.data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-genome k-mer count; a presence matrix has no counts, so
|
||||||
|
/// `threshold` has no effect (see
|
||||||
|
/// `obicompactvec::ColumnWeights::count_geq`).
|
||||||
|
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
|
||||||
|
obicompactvec::ColumnWeights::count_geq(&self.data, threshold)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TypedLayer<PersistentBitMatrix> {
|
impl TypedLayer<PersistentBitMatrix> {
|
||||||
|
|||||||
@@ -90,6 +90,10 @@ impl ColumnWeights for AggregatedCount<'_> {
|
|||||||
let n = self.n_genomes;
|
let n = self.n_genomes;
|
||||||
self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b)
|
self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b)
|
||||||
}
|
}
|
||||||
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
|
let n = self.n_genomes;
|
||||||
|
self.reduce(|| Array1::zeros(n), |l| l.count_geq(threshold), |a, b| a + b)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountPartials for AggregatedCount<'_> {
|
impl CountPartials for AggregatedCount<'_> {
|
||||||
@@ -149,6 +153,10 @@ impl ColumnWeights for AggregatedBit<'_> {
|
|||||||
let n = self.n_genomes;
|
let n = self.n_genomes;
|
||||||
self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b)
|
self.reduce(|| Array1::zeros(n), |l| l.col_weights(), |a, b| a + b)
|
||||||
}
|
}
|
||||||
|
fn count_geq(&self, threshold: u32) -> Array1<u64> {
|
||||||
|
let n = self.n_genomes;
|
||||||
|
self.reduce(|| Array1::zeros(n), |l| l.count_geq(threshold), |a, b| a + b)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BitPartials for AggregatedBit<'_> {
|
impl BitPartials for AggregatedBit<'_> {
|
||||||
|
|||||||
Reference in new issue
Block a user