feat: add BitSliceView::for_each_set_in and optimize build_from_dense

Introduce a new `views_scan` module containing `BitSliceView::for_each_set_in`, which iterates over set bits in a specified range using optimized word traversal. Update `build_from_dense` to use this method for batched, ordered column access, ensuring sorted row output. Add comprehensive tests for the new iteration logic and boundary handling.

- Add `src/obicompactvec/src/views_scan.rs` with `BitSliceView::for_each_set_in`, which iterates over a `Range<usize>` calling a closure for each set bit while optimizing sparse traversal via `word.trailing_zeros()`.
- Add `src/obicompactvec/src/tests/views_scan.rs` with tests `words`, `naive`, `scanned`, and helpers verifying that `for_each_set_in` matches naive iteration, handles empty/reversed ranges correctly, and rejects out-of-bounds ranges.
- Modify `build_from_dense` in `src/obicompactvec/src/bitmatrix/sparse.rs` to replace `nonzero_triples` with `dense.col_view(c).for_each_set_in`, ensuring column visits are batched and ordered for sorted row processing.
- Update module structure in `src/obicompactvec/src/lib.rs` and `src/obicompactvec/src/tests/mod.rs` to include the new `views_scan` module.
This commit is contained in:
Eric Coissac committed 2026-11-19 07:08:07 +01:00
1 parent 03eaa4544a
commit 4e98de4b77
5 files changed
+131 -10

No files matched your search

+10 -10
View File
@@ -36,7 +36,6 @@ use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBu
use super::PersistentBitMatrix; use super::PersistentBitMatrix;
use super::packed::{PackedBitMatrix, pack_bit_matrix}; use super::packed::{PackedBitMatrix, pack_bit_matrix};
use crate::views::nonzero_triples;
fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") } fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") } fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") }
@@ -499,12 +498,12 @@ impl PersistentSparseBitMatrixBuilder {
/// existing index. /// existing index.
/// ///
/// Processes rows in batches of [`BUILD_FROM_DENSE_BATCH`] instead of /// Processes rows in batches of [`BUILD_FROM_DENSE_BATCH`] instead of
/// one row at a time: `nonzero_triples(..., sorted_slots=true)` walks /// one row at a time: [`BitSliceView::for_each_set_in`] walks each
/// each column *once per batch*, sequentially — matching how /// column *once per batch*, word by word, visiting only the set bits —
/// `PersistentBitMatrix::nonzero_iter` already batches slots for /// rather than `fill_row`'s row-major loop, which re-reads every column
/// queries — rather than `fill_row`'s row-major loop, which re-reads /// with one `get()` per cell and jumps between all `n_cols` columns on
/// every column with one `get()` per cell and jumps between all /// every single row. Columns are visited in ascending order, so each
/// `n_cols` column files on every single row. /// row's genome list comes out sorted, as `push_row` requires.
pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result<Self> { pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result<Self> {
let n = dense.n(); let n = dense.n();
let n_cols = dense.n_cols(); let n_cols = dense.n_cols();
@@ -513,9 +512,10 @@ impl PersistentSparseBitMatrixBuilder {
let mut start = 0; let mut start = 0;
while start < n { while start < n {
let end = (start + BUILD_FROM_DENSE_BATCH).min(n); let end = (start + BUILD_FROM_DENSE_BATCH).min(n);
let slots: Vec<usize> = (start..end).collect(); for c in 0..n_cols {
for (row, c, _) in nonzero_triples(&slots, n_cols, |c| dense.col_view(c), true) { dense.col_view(c).for_each_set_in(start..end, |slot| {
row_buffers[row].push(c as u32); row_buffers[slot - start].push(c as u32);
});
} }
for buf in row_buffers.iter_mut().take(end - start) { for buf in row_buffers.iter_mut().take(end - start) {
builder.push_row(buf); builder.push_row(buf);
+1
View File
@@ -20,6 +20,7 @@ mod tempbitvec;
mod tempintvec; mod tempintvec;
pub mod traits; pub mod traits;
mod views; mod views;
mod views_scan;
pub use bitmatrix::{ pub use bitmatrix::{
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
+1
View File
@@ -10,6 +10,7 @@ mod mem_builder;
mod rankselect; mod rankselect;
mod sparse; mod sparse;
mod sparse_intmatrix; mod sparse_intmatrix;
mod views_scan;
use tempfile::tempdir; use tempfile::tempdir;
+71
View File
@@ -0,0 +1,71 @@
use crate::views::BitSliceView;
/// Deterministic pseudo-random words (xorshift), thinned so that many words
/// are zero: the case the word-wise scan has to skip correctly.
fn words(n_words: usize, seed: u64, thin: u32) -> Vec<u64> {
let mut s = seed | 1;
(0..n_words)
.map(|_| {
let mut w = u64::MAX;
for _ in 0..thin {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
w &= s;
}
w
})
.collect()
}
fn naive(v: &BitSliceView<'_>, r: std::ops::Range<usize>) -> Vec<usize> {
r.filter(|&s| v.get(s)).collect()
}
fn scanned(v: &BitSliceView<'_>, r: std::ops::Range<usize>) -> Vec<usize> {
let mut out = Vec::new();
v.for_each_set_in(r, |s| out.push(s));
out
}
#[test]
fn for_each_set_in_matches_get_loop_on_all_boundaries() {
// Lengths around word boundaries; padding bits of the last word are zero.
for n in [0usize, 1, 63, 64, 65, 127, 128, 129, 200, 4096, 4097] {
for (seed, thin) in [(1u64, 0u32), (7, 1), (11, 3), (13, 6)] {
let mut w = words(n.div_ceil(64), seed, thin);
if n % 64 != 0 {
if let Some(last) = w.last_mut() {
*last &= (1u64 << (n % 64)) - 1;
}
}
let v = BitSliceView::new(&w, n);
let cuts = [0, 1, 63, 64, 65, 100, 127, 128, 129, n / 2, n.saturating_sub(1), n];
for &a in &cuts {
for &b in &cuts {
if a <= n && b <= n {
assert_eq!(scanned(&v, a..b), naive(&v, a..b), "n={n} seed={seed} thin={thin} range={a}..{b}");
}
}
}
}
}
}
#[test]
fn for_each_set_in_empty_and_reversed_ranges_do_nothing() {
let w = [u64::MAX; 2];
let v = BitSliceView::new(&w, 128);
assert!(scanned(&v, 5..5).is_empty());
#[allow(clippy::reversed_empty_ranges)]
let rev = 9..3;
assert!(scanned(&v, rev).is_empty());
}
#[test]
#[should_panic]
fn for_each_set_in_rejects_range_past_the_view() {
let w = [u64::MAX; 2];
let v = BitSliceView::new(&w, 100);
scanned(&v, 0..101);
}
+48
View File
@@ -0,0 +1,48 @@
//! Word-wise scans over a [`BitSliceView`]. Kept apart from `views.rs`, which
//! is already past the size budget; these are inherent methods of the same type.
use std::ops::Range;
use crate::views::BitSliceView;
impl BitSliceView<'_> {
/// Calls `f(slot)` for every set bit with `slot` in `range`, in ascending
/// order.
///
/// Walks the underlying `u64` words and jumps from set bit to set bit
/// with `trailing_zeros`, so the cost is proportional to the number of
/// words plus the number of set bits — not to the number of slots, as a
/// `get(slot)` loop would be. Words that are entirely zero cost one load
/// and one test, which is what makes this the right traversal for the
/// sparse presence columns `build_from_dense` reads.
///
/// Panics if `range.end > self.len()`.
#[inline]
pub fn for_each_set_in(&self, range: Range<usize>, mut f: impl FnMut(usize)) {
assert!(range.end <= self.n, "range end {} past view length {}", range.end, self.n);
if range.start >= range.end {
return;
}
let first = range.start >> 6;
let last = (range.end - 1) >> 6;
for w in first..=last {
let mut word = self.words[w];
if word == 0 {
continue;
}
if w == first {
word &= !0u64 << (range.start & 63);
}
if w == last {
let tail = range.end & 63;
if tail != 0 {
word &= (1u64 << tail) - 1;
}
}
while word != 0 {
f((w << 6) | word.trailing_zeros() as usize);
word &= word - 1;
}
}
}
}