diff --git a/src/obicompactvec/src/bitmatrix/sparse.rs b/src/obicompactvec/src/bitmatrix/sparse.rs index 89eeb496..ba95a9b0 100644 --- a/src/obicompactvec/src/bitmatrix/sparse.rs +++ b/src/obicompactvec/src/bitmatrix/sparse.rs @@ -36,7 +36,6 @@ use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBu use super::PersistentBitMatrix; use super::packed::{PackedBitMatrix, pack_bit_matrix}; -use crate::views::nonzero_triples; fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") } fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") } @@ -499,12 +498,12 @@ impl PersistentSparseBitMatrixBuilder { /// existing index. /// /// Processes rows in batches of [`BUILD_FROM_DENSE_BATCH`] instead of - /// one row at a time: `nonzero_triples(..., sorted_slots=true)` walks - /// each column *once per batch*, sequentially — matching how - /// `PersistentBitMatrix::nonzero_iter` already batches slots for - /// queries — rather than `fill_row`'s row-major loop, which re-reads - /// every column with one `get()` per cell and jumps between all - /// `n_cols` column files on every single row. + /// one row at a time: [`BitSliceView::for_each_set_in`] walks each + /// column *once per batch*, word by word, visiting only the set bits — + /// rather than `fill_row`'s row-major loop, which re-reads every column + /// with one `get()` per cell and jumps between all `n_cols` columns on + /// every single row. Columns are visited in ascending order, so each + /// row's genome list comes out sorted, as `push_row` requires. pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result { let n = dense.n(); let n_cols = dense.n_cols(); @@ -513,9 +512,10 @@ impl PersistentSparseBitMatrixBuilder { let mut start = 0; while start < n { let end = (start + BUILD_FROM_DENSE_BATCH).min(n); - let slots: Vec = (start..end).collect(); - for (row, c, _) in nonzero_triples(&slots, n_cols, |c| dense.col_view(c), true) { - row_buffers[row].push(c as u32); + for c in 0..n_cols { + dense.col_view(c).for_each_set_in(start..end, |slot| { + row_buffers[slot - start].push(c as u32); + }); } for buf in row_buffers.iter_mut().take(end - start) { builder.push_row(buf); diff --git a/src/obicompactvec/src/lib.rs b/src/obicompactvec/src/lib.rs index d64e0eb3..58a4ab75 100644 --- a/src/obicompactvec/src/lib.rs +++ b/src/obicompactvec/src/lib.rs @@ -20,6 +20,7 @@ mod tempbitvec; mod tempintvec; pub mod traits; mod views; +mod views_scan; pub use bitmatrix::{ PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, diff --git a/src/obicompactvec/src/tests/mod.rs b/src/obicompactvec/src/tests/mod.rs index d3878925..810ac805 100644 --- a/src/obicompactvec/src/tests/mod.rs +++ b/src/obicompactvec/src/tests/mod.rs @@ -10,6 +10,7 @@ mod mem_builder; mod rankselect; mod sparse; mod sparse_intmatrix; +mod views_scan; use tempfile::tempdir; diff --git a/src/obicompactvec/src/tests/views_scan.rs b/src/obicompactvec/src/tests/views_scan.rs new file mode 100644 index 00000000..56037b77 --- /dev/null +++ b/src/obicompactvec/src/tests/views_scan.rs @@ -0,0 +1,71 @@ +use crate::views::BitSliceView; + +/// Deterministic pseudo-random words (xorshift), thinned so that many words +/// are zero: the case the word-wise scan has to skip correctly. +fn words(n_words: usize, seed: u64, thin: u32) -> Vec { + let mut s = seed | 1; + (0..n_words) + .map(|_| { + let mut w = u64::MAX; + for _ in 0..thin { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + w &= s; + } + w + }) + .collect() +} + +fn naive(v: &BitSliceView<'_>, r: std::ops::Range) -> Vec { + r.filter(|&s| v.get(s)).collect() +} + +fn scanned(v: &BitSliceView<'_>, r: std::ops::Range) -> Vec { + let mut out = Vec::new(); + v.for_each_set_in(r, |s| out.push(s)); + out +} + +#[test] +fn for_each_set_in_matches_get_loop_on_all_boundaries() { + // Lengths around word boundaries; padding bits of the last word are zero. + for n in [0usize, 1, 63, 64, 65, 127, 128, 129, 200, 4096, 4097] { + for (seed, thin) in [(1u64, 0u32), (7, 1), (11, 3), (13, 6)] { + let mut w = words(n.div_ceil(64), seed, thin); + if n % 64 != 0 { + if let Some(last) = w.last_mut() { + *last &= (1u64 << (n % 64)) - 1; + } + } + let v = BitSliceView::new(&w, n); + let cuts = [0, 1, 63, 64, 65, 100, 127, 128, 129, n / 2, n.saturating_sub(1), n]; + for &a in &cuts { + for &b in &cuts { + if a <= n && b <= n { + assert_eq!(scanned(&v, a..b), naive(&v, a..b), "n={n} seed={seed} thin={thin} range={a}..{b}"); + } + } + } + } + } +} + +#[test] +fn for_each_set_in_empty_and_reversed_ranges_do_nothing() { + let w = [u64::MAX; 2]; + let v = BitSliceView::new(&w, 128); + assert!(scanned(&v, 5..5).is_empty()); + #[allow(clippy::reversed_empty_ranges)] + let rev = 9..3; + assert!(scanned(&v, rev).is_empty()); +} + +#[test] +#[should_panic] +fn for_each_set_in_rejects_range_past_the_view() { + let w = [u64::MAX; 2]; + let v = BitSliceView::new(&w, 100); + scanned(&v, 0..101); +} diff --git a/src/obicompactvec/src/views_scan.rs b/src/obicompactvec/src/views_scan.rs new file mode 100644 index 00000000..76248e02 --- /dev/null +++ b/src/obicompactvec/src/views_scan.rs @@ -0,0 +1,48 @@ +//! Word-wise scans over a [`BitSliceView`]. Kept apart from `views.rs`, which +//! is already past the size budget; these are inherent methods of the same type. + +use std::ops::Range; + +use crate::views::BitSliceView; + +impl BitSliceView<'_> { + /// Calls `f(slot)` for every set bit with `slot` in `range`, in ascending + /// order. + /// + /// Walks the underlying `u64` words and jumps from set bit to set bit + /// with `trailing_zeros`, so the cost is proportional to the number of + /// words plus the number of set bits — not to the number of slots, as a + /// `get(slot)` loop would be. Words that are entirely zero cost one load + /// and one test, which is what makes this the right traversal for the + /// sparse presence columns `build_from_dense` reads. + /// + /// Panics if `range.end > self.len()`. + #[inline] + pub fn for_each_set_in(&self, range: Range, mut f: impl FnMut(usize)) { + assert!(range.end <= self.n, "range end {} past view length {}", range.end, self.n); + if range.start >= range.end { + return; + } + let first = range.start >> 6; + let last = (range.end - 1) >> 6; + for w in first..=last { + let mut word = self.words[w]; + if word == 0 { + continue; + } + if w == first { + word &= !0u64 << (range.start & 63); + } + if w == last { + let tail = range.end & 63; + if tail != 0 { + word &= (1u64 << tail) - 1; + } + } + while word != 0 { + f((w << 6) | word.trailing_zeros() as usize); + word &= word - 1; + } + } + } +}