Push vpnykuuvnrnx #82

Merged
coissac merged 11 commits from push-vpnykuuvnrnx into main 2026-11-19 06:17:44 +00:00
5 changed files with 131 additions and 10 deletions
Showing only changes of commit 4e98de4b77 - Show all commits

No files matched your search

+10 -10
View File
@@ -36,7 +36,6 @@ use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBu
use super::PersistentBitMatrix;
use super::packed::{PackedBitMatrix, pack_bit_matrix};
use crate::views::nonzero_triples;
fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") }
@@ -499,12 +498,12 @@ impl PersistentSparseBitMatrixBuilder {
/// existing index.
///
/// Processes rows in batches of [`BUILD_FROM_DENSE_BATCH`] instead of
/// one row at a time: `nonzero_triples(..., sorted_slots=true)` walks
/// each column *once per batch*, sequentially — matching how
/// `PersistentBitMatrix::nonzero_iter` already batches slots for
/// queries — rather than `fill_row`'s row-major loop, which re-reads
/// every column with one `get()` per cell and jumps between all
/// `n_cols` column files on every single row.
/// one row at a time: [`BitSliceView::for_each_set_in`] walks each
/// column *once per batch*, word by word, visiting only the set bits —
/// rather than `fill_row`'s row-major loop, which re-reads every column
/// with one `get()` per cell and jumps between all `n_cols` columns on
/// every single row. Columns are visited in ascending order, so each
/// row's genome list comes out sorted, as `push_row` requires.
pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result<Self> {
let n = dense.n();
let n_cols = dense.n_cols();
@@ -513,9 +512,10 @@ impl PersistentSparseBitMatrixBuilder {
let mut start = 0;
while start < n {
let end = (start + BUILD_FROM_DENSE_BATCH).min(n);
let slots: Vec<usize> = (start..end).collect();
for (row, c, _) in nonzero_triples(&slots, n_cols, |c| dense.col_view(c), true) {
row_buffers[row].push(c as u32);
for c in 0..n_cols {
dense.col_view(c).for_each_set_in(start..end, |slot| {
row_buffers[slot - start].push(c as u32);
});
}
for buf in row_buffers.iter_mut().take(end - start) {
builder.push_row(buf);
+1
View File
@@ -20,6 +20,7 @@ mod tempbitvec;
mod tempintvec;
pub mod traits;
mod views;
mod views_scan;
pub use bitmatrix::{
PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix,
+1
View File
@@ -10,6 +10,7 @@ mod mem_builder;
mod rankselect;
mod sparse;
mod sparse_intmatrix;
mod views_scan;
use tempfile::tempdir;
+71
View File
@@ -0,0 +1,71 @@
use crate::views::BitSliceView;
/// Deterministic pseudo-random words (xorshift), thinned so that many words
/// are zero: the case the word-wise scan has to skip correctly.
fn words(n_words: usize, seed: u64, thin: u32) -> Vec<u64> {
let mut s = seed | 1;
(0..n_words)
.map(|_| {
let mut w = u64::MAX;
for _ in 0..thin {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
w &= s;
}
w
})
.collect()
}
fn naive(v: &BitSliceView<'_>, r: std::ops::Range<usize>) -> Vec<usize> {
r.filter(|&s| v.get(s)).collect()
}
fn scanned(v: &BitSliceView<'_>, r: std::ops::Range<usize>) -> Vec<usize> {
let mut out = Vec::new();
v.for_each_set_in(r, |s| out.push(s));
out
}
#[test]
fn for_each_set_in_matches_get_loop_on_all_boundaries() {
// Lengths around word boundaries; padding bits of the last word are zero.
for n in [0usize, 1, 63, 64, 65, 127, 128, 129, 200, 4096, 4097] {
for (seed, thin) in [(1u64, 0u32), (7, 1), (11, 3), (13, 6)] {
let mut w = words(n.div_ceil(64), seed, thin);
if n % 64 != 0 {
if let Some(last) = w.last_mut() {
*last &= (1u64 << (n % 64)) - 1;
}
}
let v = BitSliceView::new(&w, n);
let cuts = [0, 1, 63, 64, 65, 100, 127, 128, 129, n / 2, n.saturating_sub(1), n];
for &a in &cuts {
for &b in &cuts {
if a <= n && b <= n {
assert_eq!(scanned(&v, a..b), naive(&v, a..b), "n={n} seed={seed} thin={thin} range={a}..{b}");
}
}
}
}
}
}
#[test]
fn for_each_set_in_empty_and_reversed_ranges_do_nothing() {
let w = [u64::MAX; 2];
let v = BitSliceView::new(&w, 128);
assert!(scanned(&v, 5..5).is_empty());
#[allow(clippy::reversed_empty_ranges)]
let rev = 9..3;
assert!(scanned(&v, rev).is_empty());
}
#[test]
#[should_panic]
fn for_each_set_in_rejects_range_past_the_view() {
let w = [u64::MAX; 2];
let v = BitSliceView::new(&w, 100);
scanned(&v, 0..101);
}
+48
View File
@@ -0,0 +1,48 @@
//! Word-wise scans over a [`BitSliceView`]. Kept apart from `views.rs`, which
//! is already past the size budget; these are inherent methods of the same type.
use std::ops::Range;
use crate::views::BitSliceView;
impl BitSliceView<'_> {
/// Calls `f(slot)` for every set bit with `slot` in `range`, in ascending
/// order.
///
/// Walks the underlying `u64` words and jumps from set bit to set bit
/// with `trailing_zeros`, so the cost is proportional to the number of
/// words plus the number of set bits — not to the number of slots, as a
/// `get(slot)` loop would be. Words that are entirely zero cost one load
/// and one test, which is what makes this the right traversal for the
/// sparse presence columns `build_from_dense` reads.
///
/// Panics if `range.end > self.len()`.
#[inline]
pub fn for_each_set_in(&self, range: Range<usize>, mut f: impl FnMut(usize)) {
assert!(range.end <= self.n, "range end {} past view length {}", range.end, self.n);
if range.start >= range.end {
return;
}
let first = range.start >> 6;
let last = (range.end - 1) >> 6;
for w in first..=last {
let mut word = self.words[w];
if word == 0 {
continue;
}
if w == first {
word &= !0u64 << (range.start & 63);
}
if w == last {
let tail = range.end & 63;
if tail != 0 {
word &= (1u64 << tail) - 1;
}
}
while word != 0 {
f((w << 6) | word.trailing_zeros() as usize);
word &= word - 1;
}
}
}
}