perf: add #[inline] attributes to obicompactvec methods

Adds compiler inlining hints to accessors, iterators, bitwise operations, and distance functions across multiple modules. This optimization aims to reduce call overhead for frequently invoked methods without modifying runtime behavior, API contracts, or data models.
This commit is contained in:
Eric Coissac
2026-08-16 21:18:58 +02:00
parent 693c18bfa7
commit 151493526c
10 changed files with 151 additions and 0 deletions
@@ -27,14 +27,19 @@ impl ColumnarBitMatrix {
Ok(Self { cols, n: meta.n }) Ok(Self { cols, n: meta.n })
} }
#[inline]
pub(crate) fn n(&self) -> usize { self.n } pub(crate) fn n(&self) -> usize { self.n }
#[inline]
pub(crate) fn n_cols(&self) -> usize { self.cols.len() } pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
#[inline]
pub(crate) fn col(&self, c: usize) -> &PersistentBitVec { &self.cols[c] } pub(crate) fn col(&self, c: usize) -> &PersistentBitVec { &self.cols[c] }
#[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[bool]> { pub(crate) fn row(&self, slot: usize) -> Box<[bool]> {
self.cols.iter().map(|c| c.get(slot)).collect() self.cols.iter().map(|c| c.get(slot)).collect()
} }
#[inline]
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
for (c, col) in self.cols.iter().enumerate() { for (c, col) in self.cols.iter().enumerate() {
buf[c] = col.get(slot) as u32; buf[c] = col.get(slot) as u32;
@@ -58,17 +58,20 @@ impl PackedBitMatrix {
} }
} }
#[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[bool]> { pub(crate) fn row(&self, slot: usize) -> Box<[bool]> {
(0..self.n_cols).map(|c| { (0..self.n_cols).map(|c| {
(self.mmap[self.data_offsets[c] + (slot >> 3)] >> (slot & 7)) & 1 != 0 (self.mmap[self.data_offsets[c] + (slot >> 3)] >> (slot & 7)) & 1 != 0
}).collect() }).collect()
} }
#[inline]
fn col_bytes(&self, c: usize) -> &[u8] { fn col_bytes(&self, c: usize) -> &[u8] {
let start = self.data_offsets[c]; let start = self.data_offsets[c];
&self.mmap[start..start + self.n_rows.div_ceil(8)] &self.mmap[start..start + self.n_rows.div_ceil(8)]
} }
#[inline]
fn col_words(&self, c: usize) -> &[u64] { fn col_words(&self, c: usize) -> &[u64] {
let nw = self.n_rows.div_ceil(64); let nw = self.n_rows.div_ceil(64);
// SAFETY: data_offsets[c] is always 8-byte aligned. // SAFETY: data_offsets[c] is always 8-byte aligned.
@@ -78,6 +81,7 @@ impl PackedBitMatrix {
unsafe { std::slice::from_raw_parts(ptr, nw) } unsafe { std::slice::from_raw_parts(ptr, nw) }
} }
#[inline]
pub(crate) fn col_slice(&self, c: usize) -> BitSliceView<'_> { pub(crate) fn col_slice(&self, c: usize) -> BitSliceView<'_> {
BitSliceView::new(self.col_words(c), self.n_rows) BitSliceView::new(self.col_words(c), self.n_rows)
} }
@@ -55,6 +55,7 @@ impl PersistentBitMatrix {
Ok(Self::Implicit { n_rows: meta.n, n_cols: 1 }) Ok(Self::Implicit { n_rows: meta.n, n_cols: 1 })
} }
#[inline]
pub fn n(&self) -> usize { pub fn n(&self) -> usize {
match self { match self {
Self::Columnar(m) => m.n(), Self::Columnar(m) => m.n(),
@@ -63,6 +64,7 @@ impl PersistentBitMatrix {
} }
} }
#[inline]
pub fn n_cols(&self) -> usize { pub fn n_cols(&self) -> usize {
match self { match self {
Self::Columnar(m) => m.n_cols(), Self::Columnar(m) => m.n_cols(),
@@ -71,6 +73,7 @@ impl PersistentBitMatrix {
} }
} }
#[inline]
pub fn col(&self, c: usize) -> &PersistentBitVec { pub fn col(&self, c: usize) -> &PersistentBitVec {
match self { match self {
Self::Columnar(m) => m.col(c), Self::Columnar(m) => m.col(c),
@@ -78,6 +81,7 @@ impl PersistentBitMatrix {
} }
} }
#[inline]
pub fn col_view(&self, c: usize) -> BitSliceView<'_> { pub fn col_view(&self, c: usize) -> BitSliceView<'_> {
match self { match self {
Self::Columnar(m) => m.col(c).view(), Self::Columnar(m) => m.col(c).view(),
@@ -91,6 +95,7 @@ impl PersistentBitMatrix {
/// Unlike [`col_view`](Self::col_view), this never panics on `Implicit` /// Unlike [`col_view`](Self::col_view), this never panics on `Implicit`
/// (every column reads as present, per the mono-genome fast path) — safe /// (every column reads as present, per the mono-genome fast path) — safe
/// to call for any `c < self.n_cols()`. /// to call for any `c < self.n_cols()`.
#[inline]
pub fn get(&self, c: usize, slot: usize) -> u32 { pub fn get(&self, c: usize, slot: usize) -> u32 {
match self { match self {
Self::Columnar(m) => m.col(c).get(slot) as u32, Self::Columnar(m) => m.col(c).get(slot) as u32,
@@ -109,6 +114,7 @@ impl PersistentBitMatrix {
} }
} }
#[inline]
pub fn row(&self, slot: usize) -> Box<[bool]> { pub fn row(&self, slot: usize) -> Box<[bool]> {
match self { match self {
Self::Columnar(m) => m.row(slot), Self::Columnar(m) => m.row(slot),
@@ -118,6 +124,7 @@ impl PersistentBitMatrix {
} }
/// Fill `buf[i]` with `col_i[slot]` as 0/1 u32, without allocating. /// Fill `buf[i]` with `col_i[slot]` as 0/1 u32, without allocating.
#[inline]
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
match self { match self {
Self::Columnar(m) => m.fill_row(slot, buf), Self::Columnar(m) => m.fill_row(slot, buf),
@@ -177,6 +184,7 @@ impl PersistentBitMatrix {
} }
} }
#[inline]
pub fn count_ones(&self) -> Array1<u64> { pub fn count_ones(&self) -> Array1<u64> {
match self { match self {
Self::Columnar(m) => m.count_ones(), Self::Columnar(m) => m.count_ones(),
@@ -219,13 +227,16 @@ impl PersistentBitMatrix {
// ── Trait impls ─────────────────────────────────────────────────────────────── // ── Trait impls ───────────────────────────────────────────────────────────────
impl ColumnWeights for PersistentBitMatrix { impl ColumnWeights for PersistentBitMatrix {
#[inline]
fn col_weights(&self) -> Array1<u64> { self.count_ones() } fn col_weights(&self) -> Array1<u64> { self.count_ones() }
} }
impl BitPartials for PersistentBitMatrix { impl BitPartials for PersistentBitMatrix {
#[inline]
fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) { fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) {
self.partial_jaccard_dist_matrix() self.partial_jaccard_dist_matrix()
} }
#[inline]
fn partial_hamming(&self) -> Array2<u64> { fn partial_hamming(&self) -> Array2<u64> {
self.partial_hamming_dist_matrix() self.partial_hamming_dist_matrix()
} }
+25
View File
@@ -51,16 +51,20 @@ impl PersistentBitVec {
}) })
} }
#[inline]
pub fn path(&self) -> &Path { pub fn path(&self) -> &Path {
&self.path &self.path
} }
#[inline]
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.n self.n
} }
#[inline]
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.n == 0 self.n == 0
} }
#[inline]
pub fn get(&self, slot: usize) -> bool { pub fn get(&self, slot: usize) -> bool {
(self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0 (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
} }
@@ -104,37 +108,46 @@ impl PersistentBitVec {
} }
// SAFETY: mmap is page-aligned, HEADER_SIZE=16 divisible by 8 → u64-aligned. // SAFETY: mmap is page-aligned, HEADER_SIZE=16 divisible by 8 → u64-aligned.
#[inline]
fn data_words(&self) -> &[u64] { fn data_words(&self) -> &[u64] {
let nw = n_words(self.n); let nw = n_words(self.n);
let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64; let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
unsafe { std::slice::from_raw_parts(ptr, nw) } unsafe { std::slice::from_raw_parts(ptr, nw) }
} }
#[inline]
pub fn view(&self) -> BitSliceView<'_> { pub fn view(&self) -> BitSliceView<'_> {
BitSliceView::new(self.data_words(), self.n) BitSliceView::new(self.data_words(), self.n)
} }
#[inline]
pub fn words(&self) -> &[u64] { pub fn words(&self) -> &[u64] {
self.data_words() self.data_words()
} }
#[inline]
pub fn count_ones(&self) -> u64 { pub fn count_ones(&self) -> u64 {
self.view().count_ones() self.view().count_ones()
} }
#[inline]
pub fn count_zeros(&self) -> u64 { pub fn count_zeros(&self) -> u64 {
self.view().count_zeros() self.view().count_zeros()
} }
#[inline]
pub fn partial_jaccard_dist(&self, other: &PersistentBitVec) -> (u64, u64) { pub fn partial_jaccard_dist(&self, other: &PersistentBitVec) -> (u64, u64) {
self.view().partial_jaccard_dist(other.view()) self.view().partial_jaccard_dist(other.view())
} }
#[inline]
pub fn jaccard_dist(&self, other: &PersistentBitVec) -> f64 { pub fn jaccard_dist(&self, other: &PersistentBitVec) -> f64 {
self.view().jaccard_dist(other.view()) self.view().jaccard_dist(other.view())
} }
#[inline]
pub fn hamming_dist(&self, other: &PersistentBitVec) -> u64 { pub fn hamming_dist(&self, other: &PersistentBitVec) -> u64 {
self.view().hamming_dist(other.view()) self.view().hamming_dist(other.view())
} }
#[inline]
pub fn iter(&self) -> BitIter<'_> { pub fn iter(&self) -> BitIter<'_> {
BitIter { BitIter {
words: self.data_words(), words: self.data_words(),
@@ -147,6 +160,7 @@ impl PersistentBitVec {
impl<'a> IntoIterator for &'a PersistentBitVec { impl<'a> IntoIterator for &'a PersistentBitVec {
type Item = bool; type Item = bool;
type IntoIter = BitIter<'a>; type IntoIter = BitIter<'a>;
#[inline]
fn into_iter(self) -> BitIter<'a> { fn into_iter(self) -> BitIter<'a> {
self.iter() self.iter()
} }
@@ -164,6 +178,7 @@ impl ExactSizeIterator for BitIter<'_> {}
impl Iterator for BitIter<'_> { impl Iterator for BitIter<'_> {
type Item = bool; type Item = bool;
#[inline]
fn next(&mut self) -> Option<bool> { fn next(&mut self) -> Option<bool> {
if self.slot >= self.n { if self.slot >= self.n {
return None; return None;
@@ -172,6 +187,7 @@ impl Iterator for BitIter<'_> {
self.slot += 1; self.slot += 1;
Some(v) Some(v)
} }
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) { fn size_hint(&self) -> (usize, Option<usize>) {
let rem = self.n - self.slot; let rem = self.n - self.slot;
(rem, Some(rem)) (rem, Some(rem))
@@ -314,17 +330,21 @@ impl PersistentBitVecBuilder {
Self::build_from_counts(source, 1, path) Self::build_from_counts(source, 1, path)
} }
#[inline]
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.n self.n
} }
#[inline]
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.n == 0 self.n == 0
} }
#[inline]
pub fn get(&self, slot: usize) -> bool { pub fn get(&self, slot: usize) -> bool {
(self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0 (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
} }
#[inline]
pub fn set(&mut self, slot: usize, value: bool) { pub fn set(&mut self, slot: usize, value: bool) {
let bit = 1u64 << (slot & 63); let bit = 1u64 << (slot & 63);
if value { if value {
@@ -334,6 +354,7 @@ impl PersistentBitVecBuilder {
} }
} }
#[inline]
fn data_words(&self) -> &[u64] { fn data_words(&self) -> &[u64] {
let nw = n_words(self.n); let nw = n_words(self.n);
let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64; let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
@@ -341,16 +362,19 @@ impl PersistentBitVecBuilder {
} }
// SAFETY: same alignment argument as PersistentBitVec::data_words. // SAFETY: same alignment argument as PersistentBitVec::data_words.
#[inline]
fn data_words_mut(&mut self) -> &mut [u64] { fn data_words_mut(&mut self) -> &mut [u64] {
let nw = n_words(self.n); let nw = n_words(self.n);
let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64; let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
unsafe { std::slice::from_raw_parts_mut(ptr, nw) } unsafe { std::slice::from_raw_parts_mut(ptr, nw) }
} }
#[inline]
pub fn view(&self) -> BitSliceView<'_> { pub fn view(&self) -> BitSliceView<'_> {
BitSliceView::new(self.data_words(), self.n) BitSliceView::new(self.data_words(), self.n)
} }
#[inline]
pub fn words(&self) -> &[u64] { pub fn words(&self) -> &[u64] {
self.data_words() self.data_words()
} }
@@ -472,6 +496,7 @@ impl PersistentBitVecBuilder {
} }
} }
#[inline]
pub fn iter(&self) -> BitSliceIter<'_> { pub fn iter(&self) -> BitSliceIter<'_> {
self.view().iter() self.view().iter()
} }
+12
View File
@@ -52,6 +52,7 @@ impl PersistentCompactIntVecBuilder {
Ok(Self { path: path.to_path_buf(), mmap, n, overflow }) Ok(Self { path: path.to_path_buf(), mmap, n, overflow })
} }
#[inline]
pub fn get(&self, slot: usize) -> u32 { pub fn get(&self, slot: usize) -> u32 {
match self.mmap[HEADER_SIZE + slot] { match self.mmap[HEADER_SIZE + slot] {
255 => *self.overflow.get(&slot).expect("sentinel without overflow entry"), 255 => *self.overflow.get(&slot).expect("sentinel without overflow entry"),
@@ -59,6 +60,7 @@ impl PersistentCompactIntVecBuilder {
} }
} }
#[inline]
pub fn set(&mut self, slot: usize, value: u32) { pub fn set(&mut self, slot: usize, value: u32) {
if value < 255 { if value < 255 {
self.mmap[HEADER_SIZE + slot] = value as u8; self.mmap[HEADER_SIZE + slot] = value as u8;
@@ -69,20 +71,28 @@ impl PersistentCompactIntVecBuilder {
} }
} }
#[inline]
pub fn len(&self) -> usize { self.n } pub fn len(&self) -> usize { self.n }
#[inline]
pub fn is_empty(&self) -> bool { self.n == 0 } pub fn is_empty(&self) -> bool { self.n == 0 }
#[inline]
pub fn primary_bytes(&self) -> &[u8] { &self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] } pub fn primary_bytes(&self) -> &[u8] { &self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
#[inline]
pub fn primary_bytes_mut(&mut self) -> &mut [u8] { &mut self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] } pub fn primary_bytes_mut(&mut self) -> &mut [u8] { &mut self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
#[inline]
pub fn clear_overflow(&mut self) { self.overflow.clear(); } pub fn clear_overflow(&mut self) { self.overflow.clear(); }
#[inline]
pub fn sum(&self) -> u64 { pub fn sum(&self) -> u64 {
byte_sum(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n], self.overflow.values().copied()) byte_sum(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n], self.overflow.values().copied())
} }
#[inline]
pub fn count_nonzero(&self) -> u64 { pub fn count_nonzero(&self) -> u64 {
byte_count_nonzero(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n]) byte_count_nonzero(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n])
} }
#[inline]
pub fn view(&self) -> IntSliceView<'_> { pub fn view(&self) -> IntSliceView<'_> {
// Builder overflow is a HashMap, not sorted raw bytes — convert on the fly // Builder overflow is a HashMap, not sorted raw bytes — convert on the fly
// by collecting into a sorted vec and storing in a thread-local buffer. // by collecting into a sorted vec and storing in a thread-local buffer.
@@ -94,10 +104,12 @@ impl PersistentCompactIntVecBuilder {
IntSliceView::new(primary, &[], 0, self.n) IntSliceView::new(primary, &[], 0, self.n)
} }
#[inline]
pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ { pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
self.overflow.iter().map(|(&k, &v)| (k, v)) self.overflow.iter().map(|(&k, &v)| (k, v))
} }
#[inline]
pub fn inc(&mut self, slot: usize) { pub fn inc(&mut self, slot: usize) {
let v = self.get(slot); let v = self.get(slot);
self.set(slot, v.saturating_add(1)); self.set(slot, v.saturating_add(1));
+33
View File
@@ -36,14 +36,19 @@ impl ColumnarCompactIntMatrix {
Ok(Self { cols, n: meta.n }) Ok(Self { cols, n: meta.n })
} }
#[inline]
pub(crate) fn n(&self) -> usize { self.n } pub(crate) fn n(&self) -> usize { self.n }
#[inline]
pub(crate) fn n_cols(&self) -> usize { self.cols.len() } pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
#[inline]
pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] } pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
#[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> { pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
self.cols.iter().map(|c| c.get(slot)).collect() self.cols.iter().map(|c| c.get(slot)).collect()
} }
#[inline]
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
for (c, col) in self.cols.iter().enumerate() { buf[c] = col.get(slot); } for (c, col) in self.cols.iter().enumerate() { buf[c] = col.get(slot); }
} }
@@ -142,6 +147,7 @@ impl PackedCompactIntMatrix {
Ok(Self { mmap, n_rows, n_cols, columns }) Ok(Self { mmap, n_rows, n_cols, columns })
} }
#[inline]
pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> { pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
let ci = &self.columns[c]; let ci = &self.columns[c];
let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows]; let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
@@ -158,10 +164,12 @@ impl PackedCompactIntMatrix {
#[inline] #[inline]
pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) } pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
#[inline]
pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
for c in 0..self.n_cols { buf[c] = self.get(c, slot); } for c in 0..self.n_cols { buf[c] = self.get(c, slot); }
} }
#[inline]
pub(crate) fn row(&self, slot: usize) -> Box<[u32]> { pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
(0..self.n_cols).map(|c| self.get(c, slot)).collect() (0..self.n_cols).map(|c| self.get(c, slot)).collect()
} }
@@ -312,13 +320,16 @@ impl PersistentCompactIntMatrix {
)) ))
} }
#[inline]
pub fn n(&self) -> usize { pub fn n(&self) -> usize {
match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows } match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows }
} }
#[inline]
pub fn n_cols(&self) -> usize { pub fn n_cols(&self) -> usize {
match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols } match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols }
} }
#[inline]
pub fn col(&self, c: usize) -> &PersistentCompactIntVec { pub fn col(&self, c: usize) -> &PersistentCompactIntVec {
match self { match self {
Self::Columnar(m) => m.col(c), Self::Columnar(m) => m.col(c),
@@ -326,6 +337,7 @@ impl PersistentCompactIntMatrix {
} }
} }
#[inline]
pub fn col_view(&self, c: usize) -> IntSliceView<'_> { pub fn col_view(&self, c: usize) -> IntSliceView<'_> {
match self { match self {
Self::Columnar(m) => m.col(c).view(), Self::Columnar(m) => m.col(c).view(),
@@ -340,9 +352,11 @@ impl PersistentCompactIntMatrix {
} }
} }
#[inline]
pub fn row(&self, slot: usize) -> Box<[u32]> { pub fn row(&self, slot: usize) -> Box<[u32]> {
match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot) } match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot) }
} }
#[inline]
pub fn fill_row(&self, slot: usize, buf: &mut [u32]) { pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
match self { Self::Columnar(m) => m.fill_row(slot, buf), Self::Packed(m) => m.fill_row(slot, buf) } match self { Self::Columnar(m) => m.fill_row(slot, buf), Self::Packed(m) => m.fill_row(slot, buf) }
} }
@@ -390,30 +404,39 @@ impl PersistentCompactIntMatrix {
} }
} }
#[inline]
pub fn sum(&self) -> Array1<u64> { pub fn sum(&self) -> Array1<u64> {
match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum() } match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum() }
} }
#[inline]
pub fn count_nonzero(&self) -> Array1<u64> { pub fn count_nonzero(&self) -> Array1<u64> {
match self { Self::Columnar(m) => m.count_nonzero(), Self::Packed(m) => m.count_nonzero() } match self { Self::Columnar(m) => m.count_nonzero(), Self::Packed(m) => m.count_nonzero() }
} }
#[inline]
pub fn partial_bray_dist_matrix(&self) -> Array2<u64> { pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
match self { Self::Columnar(m) => m.partial_bray_dist_matrix(), Self::Packed(m) => m.partial_bray_dist_matrix() } match self { Self::Columnar(m) => m.partial_bray_dist_matrix(), Self::Packed(m) => m.partial_bray_dist_matrix() }
} }
#[inline]
pub fn partial_euclidean_dist_matrix(&self) -> Array2<f64> { pub fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
match self { Self::Columnar(m) => m.partial_euclidean_dist_matrix(), Self::Packed(m) => m.partial_euclidean_dist_matrix() } match self { Self::Columnar(m) => m.partial_euclidean_dist_matrix(), Self::Packed(m) => m.partial_euclidean_dist_matrix() }
} }
#[inline]
pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) { pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
match self { Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold), Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold) } match self { Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold), Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold) }
} }
#[inline]
pub fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self { Self::Columnar(m) => m.partial_relfreq_bray_dist_matrix(col_sums), Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums) } match self { Self::Columnar(m) => m.partial_relfreq_bray_dist_matrix(col_sums), Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums) }
} }
#[inline]
pub fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self { Self::Columnar(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums) } match self { Self::Columnar(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums) }
} }
#[inline]
pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> { pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
match self { Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums) } match self { Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums) }
} }
#[inline]
pub fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> { pub fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
ColumnarCompactIntMatrix::append_column(dir, value_of) ColumnarCompactIntMatrix::append_column(dir, value_of)
} }
@@ -424,16 +447,24 @@ impl PersistentCompactIntMatrix {
use crate::traits::{ColumnWeights, CountPartials}; use crate::traits::{ColumnWeights, CountPartials};
impl ColumnWeights for PersistentCompactIntMatrix { impl ColumnWeights for PersistentCompactIntMatrix {
#[inline]
fn col_weights(&self) -> Array1<u64> { self.sum() } fn col_weights(&self) -> Array1<u64> { self.sum() }
#[inline]
fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() } fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
} }
impl CountPartials for PersistentCompactIntMatrix { impl CountPartials for PersistentCompactIntMatrix {
#[inline]
fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() } fn partial_bray(&self) -> Array2<u64> { self.partial_bray_dist_matrix() }
#[inline]
fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() } fn partial_euclidean(&self) -> Array2<f64> { self.partial_euclidean_dist_matrix() }
#[inline]
fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) } fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
#[inline]
fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) } fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_bray_dist_matrix(g) }
#[inline]
fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) } fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_relfreq_euclidean_dist_matrix(g) }
#[inline]
fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) } fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64> { self.partial_hellinger_euclidean_dist_matrix(g) }
} }
@@ -450,7 +481,9 @@ impl PersistentCompactIntMatrixBuilder {
fs::create_dir_all(dir)?; fs::create_dir_all(dir)?;
Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 }) Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
} }
#[inline]
pub fn n(&self) -> usize { self.n } pub fn n(&self) -> usize { self.n }
#[inline]
pub fn n_cols(&self) -> usize { self.n_cols } pub fn n_cols(&self) -> usize { self.n_cols }
pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> { pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
let path = col_path(&self.dir, self.n_cols); let path = col_path(&self.dir, self.n_cols);
+11
View File
@@ -46,10 +46,14 @@ impl PersistentCompactIntVec {
Ok(Self { mmap, n, n_overflow, step, index, primary_offset, data_offset, path: path.to_path_buf() }) Ok(Self { mmap, n, n_overflow, step, index, primary_offset, data_offset, path: path.to_path_buf() })
} }
#[inline]
pub fn path(&self) -> &Path { &self.path } pub fn path(&self) -> &Path { &self.path }
#[inline]
pub fn len(&self) -> usize { self.n } pub fn len(&self) -> usize { self.n }
#[inline]
pub fn is_empty(&self) -> bool { self.n == 0 } pub fn is_empty(&self) -> bool { self.n == 0 }
#[inline]
pub fn get(&self, slot: usize) -> u32 { pub fn get(&self, slot: usize) -> u32 {
match self.mmap[self.primary_offset + slot] { match self.mmap[self.primary_offset + slot] {
255 => self.overflow_get(slot), 255 => self.overflow_get(slot),
@@ -127,23 +131,27 @@ impl PersistentCompactIntVec {
u32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap()) u32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap())
} }
#[inline]
pub fn sum(&self) -> u64 { pub fn sum(&self) -> u64 {
let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n]; let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
byte_sum(primary, (0..self.n_overflow).map(|i| self.data_value(i))) byte_sum(primary, (0..self.n_overflow).map(|i| self.data_value(i)))
} }
#[inline]
pub fn count_nonzero(&self) -> u64 { pub fn count_nonzero(&self) -> u64 {
let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n]; let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
byte_count_nonzero(primary) byte_count_nonzero(primary)
} }
/// Lightweight zero-copy view — primary and overflow point into the mmap. /// Lightweight zero-copy view — primary and overflow point into the mmap.
#[inline]
pub fn view(&self) -> IntSliceView<'_> { pub fn view(&self) -> IntSliceView<'_> {
let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n]; let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
let overflow_raw = &self.mmap[self.data_offset..self.data_offset + self.n_overflow * OVERFLOW_ENTRY_SIZE]; let overflow_raw = &self.mmap[self.data_offset..self.data_offset + self.n_overflow * OVERFLOW_ENTRY_SIZE];
IntSliceView::new(primary, overflow_raw, self.n_overflow, self.n) IntSliceView::new(primary, overflow_raw, self.n_overflow, self.n)
} }
#[inline]
pub fn iter(&self) -> Iter<'_> { pub fn iter(&self) -> Iter<'_> {
Iter { pciv: self, slot: 0, overflow_pos: 0 } Iter { pciv: self, slot: 0, overflow_pos: 0 }
} }
@@ -256,6 +264,7 @@ impl PersistentCompactIntVec {
impl<'a> IntoIterator for &'a PersistentCompactIntVec { impl<'a> IntoIterator for &'a PersistentCompactIntVec {
type Item = u32; type Item = u32;
type IntoIter = Iter<'a>; type IntoIter = Iter<'a>;
#[inline]
fn into_iter(self) -> Iter<'a> { self.iter() } fn into_iter(self) -> Iter<'a> { self.iter() }
} }
@@ -270,6 +279,7 @@ impl ExactSizeIterator for Iter<'_> {}
impl Iterator for Iter<'_> { impl Iterator for Iter<'_> {
type Item = u32; type Item = u32;
#[inline]
fn next(&mut self) -> Option<u32> { fn next(&mut self) -> Option<u32> {
if self.slot >= self.pciv.n { return None; } if self.slot >= self.pciv.n { return None; }
let v = self.pciv.mmap[self.pciv.primary_offset + self.slot]; let v = self.pciv.mmap[self.pciv.primary_offset + self.slot];
@@ -283,6 +293,7 @@ impl Iterator for Iter<'_> {
} }
} }
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) { fn size_hint(&self) -> (usize, Option<usize>) {
let remaining = self.pciv.n - self.slot; let remaining = self.pciv.n - self.slot;
(remaining, Some(remaining)) (remaining, Some(remaining))
+16
View File
@@ -21,21 +21,27 @@ impl TempBitVec {
PersistentBitVec::open(path) PersistentBitVec::open(path)
} }
#[inline]
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.vec.len() self.vec.len()
} }
#[inline]
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.vec.is_empty() self.vec.is_empty()
} }
#[inline]
pub fn get(&self, slot: usize) -> bool { pub fn get(&self, slot: usize) -> bool {
self.vec.get(slot) self.vec.get(slot)
} }
#[inline]
pub fn count_ones(&self) -> u64 { pub fn count_ones(&self) -> u64 {
self.vec.count_ones() self.vec.count_ones()
} }
#[inline]
pub fn view(&self) -> BitSliceView<'_> { pub fn view(&self) -> BitSliceView<'_> {
self.vec.view() self.vec.view()
} }
#[inline]
pub fn iter(&self) -> BitSliceIter<'_> { pub fn iter(&self) -> BitSliceIter<'_> {
self.view().iter() self.view().iter()
} }
@@ -69,42 +75,52 @@ impl TempBitVecBuilder {
Ok(TempBitVec { vec, _temp: temp }) Ok(TempBitVec { vec, _temp: temp })
} }
#[inline]
pub fn set(&mut self, slot: usize, value: bool) { pub fn set(&mut self, slot: usize, value: bool) {
self.builder.set(slot, value); self.builder.set(slot, value);
} }
#[inline]
pub fn view(&self) -> BitSliceView<'_> { pub fn view(&self) -> BitSliceView<'_> {
self.builder.view() self.builder.view()
} }
#[inline]
pub fn or(&mut self, other: BitSliceView<'_>) { pub fn or(&mut self, other: BitSliceView<'_>) {
self.builder.or(other); self.builder.or(other);
} }
#[inline]
pub fn and(&mut self, other: BitSliceView<'_>) { pub fn and(&mut self, other: BitSliceView<'_>) {
self.builder.and(other); self.builder.and(other);
} }
#[inline]
pub fn xor(&mut self, other: BitSliceView<'_>) { pub fn xor(&mut self, other: BitSliceView<'_>) {
self.builder.xor(other); self.builder.xor(other);
} }
#[inline]
pub fn not(&mut self) { pub fn not(&mut self) {
self.builder.not(); self.builder.not();
} }
#[inline]
pub fn copy_from(&mut self, src: BitSliceView<'_>) { pub fn copy_from(&mut self, src: BitSliceView<'_>) {
self.builder.copy_from(src); self.builder.copy_from(src);
} }
#[inline]
pub fn or_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) { pub fn or_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
self.builder.or_where(col, pred); self.builder.or_where(col, pred);
} }
#[inline]
pub fn and_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) { pub fn and_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
self.builder.and_where(col, pred); self.builder.and_where(col, pred);
} }
#[inline]
pub fn xor_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) { pub fn xor_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
self.builder.xor_where(col, pred); self.builder.xor_where(col, pred);
} }
+20
View File
@@ -22,11 +22,17 @@ impl TempCompactIntVec {
PersistentCompactIntVec::open(path) PersistentCompactIntVec::open(path)
} }
#[inline]
pub fn len(&self) -> usize { self.vec.len() } pub fn len(&self) -> usize { self.vec.len() }
#[inline]
pub fn is_empty(&self) -> bool { self.vec.is_empty() } pub fn is_empty(&self) -> bool { self.vec.is_empty() }
#[inline]
pub fn get(&self, slot: usize) -> u32 { self.vec.get(slot) } pub fn get(&self, slot: usize) -> u32 { self.vec.get(slot) }
#[inline]
pub fn sum(&self) -> u64 { self.vec.sum() } pub fn sum(&self) -> u64 { self.vec.sum() }
#[inline]
pub fn view(&self) -> IntSliceView<'_> { self.vec.view() } pub fn view(&self) -> IntSliceView<'_> { self.vec.view() }
#[inline]
pub fn iter(&self) -> crate::reader::Iter<'_> { self.vec.iter() } pub fn iter(&self) -> crate::reader::Iter<'_> { self.vec.iter() }
} }
@@ -51,39 +57,53 @@ impl TempCompactIntVecBuilder {
Ok(TempCompactIntVec { vec, _temp: temp }) Ok(TempCompactIntVec { vec, _temp: temp })
} }
#[inline]
pub fn n(&self) -> usize { self.builder.len() } pub fn n(&self) -> usize { self.builder.len() }
#[inline]
pub fn set(&mut self, slot: usize, value: u32) { self.builder.set(slot, value); } pub fn set(&mut self, slot: usize, value: u32) { self.builder.set(slot, value); }
#[inline]
pub fn get(&self, slot: usize) -> u32 { self.builder.get(slot) } pub fn get(&self, slot: usize) -> u32 { self.builder.get(slot) }
#[inline]
pub fn primary_bytes(&self) -> &[u8] { self.builder.primary_bytes() } pub fn primary_bytes(&self) -> &[u8] { self.builder.primary_bytes() }
#[inline]
pub fn primary_bytes_mut(&mut self) -> &mut [u8] { self.builder.primary_bytes_mut() } pub fn primary_bytes_mut(&mut self) -> &mut [u8] { self.builder.primary_bytes_mut() }
#[inline]
pub fn inc_present(&mut self, col: BitSliceView<'_>) { pub fn inc_present(&mut self, col: BitSliceView<'_>) {
self.builder.inc_present(col); self.builder.inc_present(col);
} }
#[inline]
pub fn inc_present_fast(&mut self, col: BitSliceView<'_>) { pub fn inc_present_fast(&mut self, col: BitSliceView<'_>) {
self.builder.inc_present_fast(col); self.builder.inc_present_fast(col);
} }
#[inline]
pub fn inc_predicate(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) { pub fn inc_predicate(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
self.builder.inc_predicate(col, pred); self.builder.inc_predicate(col, pred);
} }
#[inline]
pub fn inc_predicate_fast(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) { pub fn inc_predicate_fast(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
self.builder.inc_predicate_fast(col, pred); self.builder.inc_predicate_fast(col, pred);
} }
#[inline]
pub fn add(&mut self, other: IntSliceView<'_>) { pub fn add(&mut self, other: IntSliceView<'_>) {
self.builder.add(other); self.builder.add(other);
} }
#[inline]
pub fn mask_with(&mut self, mask: BitSliceView<'_>) { pub fn mask_with(&mut self, mask: BitSliceView<'_>) {
self.builder.mask_with(mask); self.builder.mask_with(mask);
} }
#[inline]
pub fn min(&mut self, other: IntSliceView<'_>) { self.builder.min(other); } pub fn min(&mut self, other: IntSliceView<'_>) { self.builder.min(other); }
#[inline]
pub fn max(&mut self, other: IntSliceView<'_>) { self.builder.max(other); } pub fn max(&mut self, other: IntSliceView<'_>) { self.builder.max(other); }
#[inline]
pub fn diff(&mut self, other: IntSliceView<'_>) { self.builder.diff(other); } pub fn diff(&mut self, other: IntSliceView<'_>) { self.builder.diff(other); }
} }
+14
View File
@@ -14,8 +14,11 @@ impl<'a> BitSliceView<'a> {
#[inline] #[inline]
pub fn new(words: &'a [u64], n: usize) -> Self { Self { words, n } } pub fn new(words: &'a [u64], n: usize) -> Self { Self { words, n } }
#[inline]
pub fn len(&self) -> usize { self.n } pub fn len(&self) -> usize { self.n }
#[inline]
pub fn is_empty(&self) -> bool { self.n == 0 } pub fn is_empty(&self) -> bool { self.n == 0 }
#[inline]
pub fn words(&self) -> &'a [u64] { self.words } pub fn words(&self) -> &'a [u64] { self.words }
#[inline] #[inline]
@@ -60,6 +63,7 @@ impl<'a> BitSliceView<'a> {
} }
pub fn count_zeros(&self) -> u64 { self.n as u64 - self.count_ones() } pub fn count_zeros(&self) -> u64 { self.n as u64 - self.count_ones() }
#[inline]
pub fn iter(&self) -> BitSliceIter<'a> { pub fn iter(&self) -> BitSliceIter<'a> {
BitSliceIter { words: self.words, slot: 0, n: self.n } BitSliceIter { words: self.words, slot: 0, n: self.n }
} }
@@ -95,12 +99,14 @@ pub struct BitSliceIter<'a> {
impl Iterator for BitSliceIter<'_> { impl Iterator for BitSliceIter<'_> {
type Item = bool; type Item = bool;
#[inline]
fn next(&mut self) -> Option<bool> { fn next(&mut self) -> Option<bool> {
if self.slot >= self.n { return None; } if self.slot >= self.n { return None; }
let v = (self.words[self.slot >> 6] >> (self.slot & 63)) & 1 != 0; let v = (self.words[self.slot >> 6] >> (self.slot & 63)) & 1 != 0;
self.slot += 1; self.slot += 1;
Some(v) Some(v)
} }
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) { fn size_hint(&self) -> (usize, Option<usize>) {
let rem = self.n - self.slot; let rem = self.n - self.slot;
(rem, Some(rem)) (rem, Some(rem))
@@ -126,11 +132,16 @@ impl<'a> IntSliceView<'a> {
Self { primary, overflow_raw, n_overflow, n } Self { primary, overflow_raw, n_overflow, n }
} }
#[inline]
pub fn len(&self) -> usize { self.n } pub fn len(&self) -> usize { self.n }
#[inline]
pub fn is_empty(&self) -> bool { self.n == 0 } pub fn is_empty(&self) -> bool { self.n == 0 }
#[inline]
pub fn primary_bytes(&self) -> &'a [u8] { self.primary } pub fn primary_bytes(&self) -> &'a [u8] { self.primary }
#[inline]
pub fn n_overflow(&self) -> usize { self.n_overflow } pub fn n_overflow(&self) -> usize { self.n_overflow }
#[inline]
pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + 'a { pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + 'a {
let raw = self.overflow_raw; let raw = self.overflow_raw;
let n_ov = self.n_overflow; let n_ov = self.n_overflow;
@@ -190,6 +201,7 @@ impl<'a> IntSliceView<'a> {
} }
/// Sequential merge scan: yields all n values in slot order. /// Sequential merge scan: yields all n values in slot order.
#[inline]
pub fn iter(&self) -> IntSliceViewIter<'a> { pub fn iter(&self) -> IntSliceViewIter<'a> {
IntSliceViewIter { IntSliceViewIter {
primary: self.primary, primary: self.primary,
@@ -324,6 +336,7 @@ pub struct IntSliceViewIter<'a> {
impl Iterator for IntSliceViewIter<'_> { impl Iterator for IntSliceViewIter<'_> {
type Item = u32; type Item = u32;
#[inline]
fn next(&mut self) -> Option<u32> { fn next(&mut self) -> Option<u32> {
if self.slot >= self.n { return None; } if self.slot >= self.n { return None; }
let v = self.primary[self.slot]; let v = self.primary[self.slot];
@@ -336,6 +349,7 @@ impl Iterator for IntSliceViewIter<'_> {
Some(val) Some(val)
} }
} }
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) { fn size_hint(&self) -> (usize, Option<usize>) {
let rem = self.n - self.slot; let rem = self.n - self.slot;
(rem, Some(rem)) (rem, Some(rem))