diff --git a/src/obicompactvec/examples/compare_sparse_dense_count.rs b/src/obicompactvec/examples/compare_sparse_dense_count.rs index 36040be7..15ce0cf0 100644 --- a/src/obicompactvec/examples/compare_sparse_dense_count.rs +++ b/src/obicompactvec/examples/compare_sparse_dense_count.rs @@ -13,20 +13,30 @@ use std::error::Error; use std::path::Path; use std::time::Instant; -use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder}; +use obicompactvec::{ + PersistentIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, +}; fn main() -> Result<(), Box> { - let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count "); + let layer_dir = std::env::args() + .nth(1) + .expect("usage: compare_sparse_dense_count "); let layer_dir = Path::new(&layer_dir); let t0 = Instant::now(); - let dense = PersistentCompactIntMatrix::open(layer_dir)?; - println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols()); + let dense = PersistentIntMatrix::open(layer_dir)?; + println!( + "dense ouverte en {:?} ({} lignes x {} colonnes)", + t0.elapsed(), + dense.n(), + dense.n_cols() + ); let out_dir = tempfile::tempdir()?; let t0 = Instant::now(); - let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?; + let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())? + .finish()?; println!("build_from_dense: {:?}", t0.elapsed()); compare(&dense, &sparse)?; @@ -36,7 +46,10 @@ fn main() -> Result<(), Box> { Ok(()) } -fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box> { +fn compare( + dense: &PersistentIntMatrix, + sparse: &PersistentSparseCompactIntMatrix, +) -> Result<(), Box> { let n = dense.n(); let n_cols = dense.n_cols(); assert_eq!(n, sparse.n(), "n mismatch"); @@ -63,7 +76,10 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI } } } - println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed()); + println!( + "comparaison case-a-case: {:?} ({total_cells} cellules)", + t0.elapsed() + ); if let Some((slot, col, d, s)) = first_mismatch { eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}"); @@ -78,7 +94,7 @@ fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactI /// for what's actually stored) — sparsity, singleton-row fraction, and the /// value-magnitude buckets this crate's overflow encoding is tuned around /// (< 127, < 255, >= 255). -fn content_stats(dense: &PersistentCompactIntMatrix) { +fn content_stats(dense: &PersistentIntMatrix) { let n = dense.n(); let n_cols = dense.n_cols(); let total_cells = n as u64 * n_cols as u64; @@ -128,7 +144,10 @@ fn content_stats(dense: &PersistentCompactIntMatrix) { 100.0 * singleton_rows as f64 / n as f64 ); if nonzero_cells > 0 { - println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64); + println!( + "valeur moyenne (non-nulles): {:.2}", + sum as f64 / nonzero_cells as f64 + ); println!("valeur max: {max_value}"); println!( "valeurs < 127: {under_127} ({:.3}% des non-nulles)", @@ -162,7 +181,7 @@ fn dir_size(dir: &Path) -> std::io::Result { fn compaction_stats( dense_layer_dir: &Path, sparse_dir: &Path, - dense: &PersistentCompactIntMatrix, + dense: &PersistentIntMatrix, _sparse: &PersistentSparseCompactIntMatrix, ) -> Result<(), Box> { let dense_size = dir_size(&dense_layer_dir.join("counts"))?; diff --git a/src/obicompactvec/src/intmatrix.rs b/src/obicompactvec/src/intmatrix.rs index 7686bfc9..b30f239e 100644 --- a/src/obicompactvec/src/intmatrix.rs +++ b/src/obicompactvec/src/intmatrix.rs @@ -5,17 +5,19 @@ use std::path::{Path, PathBuf}; use memmap2::Mmap; use ndarray::{Array1, Array2}; -use crate::bitmatrix::{fill_row_generic, par_col_reduce, pairwise2_matrix, pairwise_matrix, row_generic}; +use crate::bitmatrix::{ + fill_row_generic, pairwise_matrix, pairwise2_matrix, par_col_reduce, row_generic, +}; use crate::builder::PersistentCompactIntVecBuilder; -use crate::colgroup::{chunked_presence_count, ColGroup, MatrixGroupOps}; +use crate::colgroup::{ColGroup, MatrixGroupOps, chunked_presence_count}; use crate::format::{HEADER_SIZE, OVERFLOW_ENTRY_SIZE}; use crate::meta::MatrixMeta; use crate::reader::PersistentCompactIntVec; -use crate::sparse_intmatrix::{is_present as sparse_is_present, PersistentSparseCompactIntMatrix}; +use crate::sparse_intmatrix::{PersistentSparseCompactIntMatrix, is_present as sparse_is_present}; use crate::storage_kind::StorageKind; use crate::tempbitvec::{TempBitVec, TempBitVecBuilder}; use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder}; -use crate::views::{nonzero_triples, IntSliceView}; +use crate::views::{IntSliceView, nonzero_triples}; pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf { dir.join(format!("col_{col:06}.pciv")) @@ -25,7 +27,7 @@ pub(crate) fn col_path(dir: &Path, col: usize) -> PathBuf { pub struct ColumnarCompactIntMatrix { cols: Vec, - n: usize, + n: usize, } impl ColumnarCompactIntMatrix { @@ -38,11 +40,17 @@ impl ColumnarCompactIntMatrix { } #[inline] - pub(crate) fn n(&self) -> usize { self.n } + pub(crate) fn n(&self) -> usize { + self.n + } #[inline] - pub(crate) fn n_cols(&self) -> usize { self.cols.len() } + pub(crate) fn n_cols(&self) -> usize { + self.cols.len() + } #[inline] - pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] } + pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { + &self.cols[c] + } #[inline] pub(crate) fn row(&self, slot: usize) -> Box<[u32]> { @@ -63,34 +71,64 @@ impl ColumnarCompactIntMatrix { } pub(crate) fn partial_bray_dist_matrix(&self) -> Array2 { - pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_bray_dist(self.col(j))) + pairwise_matrix(self.n_cols(), |i, j| { + self.col(i).partial_bray_dist(self.col(j)) + }) } pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2 { - pairwise_matrix(self.n_cols(), |i, j| self.col(i).partial_euclidean_dist(self.col(j))) + pairwise_matrix(self.n_cols(), |i, j| { + self.col(i).partial_euclidean_dist(self.col(j)) + }) } - pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2, Array2) { - pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_threshold_jaccard_dist(self.col(j), threshold)) + pub(crate) fn partial_threshold_jaccard_dist_matrix( + &self, + threshold: u32, + ) -> (Array2, Array2) { + pairwise2_matrix(self.n_cols(), |i, j| { + self.col(i) + .partial_threshold_jaccard_dist(self.col(j), threshold) + }) } pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1) -> Array2 { pairwise_matrix(self.n_cols(), |i, j| { - self.col(i).partial_relfreq_bray_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64) + self.col(i).partial_relfreq_bray_dist( + self.col(j), + col_sums[i] as f64, + col_sums[j] as f64, + ) }) } - pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1) -> Array2 { + pub(crate) fn partial_relfreq_euclidean_dist_matrix( + &self, + col_sums: &Array1, + ) -> Array2 { pairwise_matrix(self.n_cols(), |i, j| { - self.col(i).partial_relfreq_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64) + self.col(i).partial_relfreq_euclidean_dist( + self.col(j), + col_sums[i] as f64, + col_sums[j] as f64, + ) }) } - pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1) -> Array2 { + pub(crate) fn partial_hellinger_euclidean_dist_matrix( + &self, + col_sums: &Array1, + ) -> Array2 { pairwise_matrix(self.n_cols(), |i, j| { - self.col(i).partial_hellinger_euclidean_dist(self.col(j), col_sums[i] as f64, col_sums[j] as f64) + self.col(i).partial_hellinger_euclidean_dist( + self.col(j), + col_sums[i] as f64, + col_sums[j] as f64, + ) }) } pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> { let mut meta = MatrixMeta::load(dir)?; let mut b = PersistentCompactIntVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?; - for slot in 0..meta.n { b.set(slot, value_of(slot)); } + for slot in 0..meta.n { + b.set(slot, value_of(slot)); + } b.close()?; meta.n_cols += 1; meta.save(dir) @@ -99,19 +137,19 @@ impl ColumnarCompactIntMatrix { // ── PackedCompactIntMatrix ──────────────────────────────────────────────────── -const PCMX_MAGIC: [u8; 4] = *b"PCMX"; -const PCMX_HEADER: usize = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8) +const PCMX_MAGIC: [u8; 4] = *b"PCMX"; +const PCMX_HEADER: usize = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8) struct ColInfo { primary_start: usize, - data_offset: usize, - n_overflow: usize, + data_offset: usize, + n_overflow: usize, } pub struct PackedCompactIntMatrix { - mmap: Mmap, - n_rows: usize, - n_cols: usize, + mmap: Mmap, + n_rows: usize, + n_cols: usize, columns: Vec, } @@ -119,43 +157,65 @@ impl PackedCompactIntMatrix { pub(crate) fn open(path: &Path) -> io::Result { let mmap = unsafe { Mmap::map(&File::open(path)?)? }; if mmap.len() < PCMX_HEADER { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PCMX file too short")); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "PCMX file too short", + )); } if &mmap[0..4] != &PCMX_MAGIC { return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PCMX magic")); } - let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; + let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; let n_cols = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize; let mut columns = Vec::with_capacity(n_cols); for c in 0..n_cols { - let off_pos = PCMX_HEADER + c * 8; - let col_base = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize; - let n_ov = u64::from_le_bytes(mmap[col_base+16..col_base+24].try_into().unwrap()) as usize; - let n_pciv = u64::from_le_bytes(mmap[col_base+8..col_base+16].try_into().unwrap()) as usize; + let off_pos = PCMX_HEADER + c * 8; + let col_base = + u64::from_le_bytes(mmap[off_pos..off_pos + 8].try_into().unwrap()) as usize; + let n_ov = + u64::from_le_bytes(mmap[col_base + 16..col_base + 24].try_into().unwrap()) as usize; + let n_pciv = + u64::from_le_bytes(mmap[col_base + 8..col_base + 16].try_into().unwrap()) as usize; let primary_start = col_base + HEADER_SIZE; - let data_offset = primary_start + n_pciv; - columns.push(ColInfo { primary_start, data_offset, n_overflow: n_ov }); + let data_offset = primary_start + n_pciv; + columns.push(ColInfo { + primary_start, + data_offset, + n_overflow: n_ov, + }); } - Ok(Self { mmap, n_rows, n_cols, columns }) + Ok(Self { + mmap, + n_rows, + n_cols, + columns, + }) } #[inline] pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> { let ci = &self.columns[c]; - let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows]; - let overflow_raw = &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE]; + let primary = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows]; + let overflow_raw = + &self.mmap[ci.data_offset..ci.data_offset + ci.n_overflow * OVERFLOW_ENTRY_SIZE]; IntSliceView::new(primary, overflow_raw, ci.n_overflow, self.n_rows) } - pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result { + pub(crate) fn col_persist( + &self, + c: usize, + path: &Path, + ) -> io::Result { let view = self.col_view(c); let overflow: std::collections::HashMap = view.overflow_entries().collect(); PersistentCompactIntVecBuilder::from_raw_primary(view.primary_bytes(), overflow, path) } #[inline] - pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) } + pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { + self.col_view(col).get(slot) + } #[inline] pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) { @@ -180,27 +240,55 @@ impl PackedCompactIntMatrix { // via `PersistentCompactIntVec`. No formula is re-derived here. pub(crate) fn partial_bray_dist_matrix(&self) -> Array2 { - pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_bray_dist(self.col_view(j))) + pairwise_matrix(self.n_cols, |i, j| { + self.col_view(i).partial_bray_dist(self.col_view(j)) + }) } pub(crate) fn partial_euclidean_dist_matrix(&self) -> Array2 { - pairwise_matrix(self.n_cols, |i, j| self.col_view(i).partial_euclidean_dist(self.col_view(j))) + pairwise_matrix(self.n_cols, |i, j| { + self.col_view(i).partial_euclidean_dist(self.col_view(j)) + }) } - pub(crate) fn partial_threshold_jaccard_dist_matrix(&self, t: u32) -> (Array2, Array2) { - pairwise2_matrix(self.n_cols, |i, j| self.col_view(i).partial_threshold_jaccard_dist(self.col_view(j), t)) + pub(crate) fn partial_threshold_jaccard_dist_matrix( + &self, + t: u32, + ) -> (Array2, Array2) { + pairwise2_matrix(self.n_cols, |i, j| { + self.col_view(i) + .partial_threshold_jaccard_dist(self.col_view(j), t) + }) } pub(crate) fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1) -> Array2 { pairwise_matrix(self.n_cols, |i, j| { - self.col_view(i).partial_relfreq_bray_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64) + self.col_view(i).partial_relfreq_bray_dist( + self.col_view(j), + col_sums[i] as f64, + col_sums[j] as f64, + ) }) } - pub(crate) fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1) -> Array2 { + pub(crate) fn partial_relfreq_euclidean_dist_matrix( + &self, + col_sums: &Array1, + ) -> Array2 { pairwise_matrix(self.n_cols, |i, j| { - self.col_view(i).partial_relfreq_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64) + self.col_view(i).partial_relfreq_euclidean_dist( + self.col_view(j), + col_sums[i] as f64, + col_sums[j] as f64, + ) }) } - pub(crate) fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1) -> Array2 { + pub(crate) fn partial_hellinger_euclidean_dist_matrix( + &self, + col_sums: &Array1, + ) -> Array2 { pairwise_matrix(self.n_cols, |i, j| { - self.col_view(i).partial_hellinger_euclidean_dist(self.col_view(j), col_sums[i] as f64, col_sums[j] as f64) + self.col_view(i).partial_hellinger_euclidean_dist( + self.col_view(j), + col_sums[i] as f64, + col_sums[j] as f64, + ) }) } } @@ -238,7 +326,9 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> { // the columnar files are newer and must be (re-)packed, overwriting the // stale one — never silently discarded as "leftover cleanup". if packed_int_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) { - for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); } + for c in 0..meta.n_cols { + let _ = fs::remove_file(col_path(dir, c)); + } let _ = fs::remove_file(dir.join("meta.json")); return Ok(()); } @@ -250,32 +340,41 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> { let header_size = (PCMX_HEADER + n_cols * 8) as u64; let mut col_offset = header_size; let mut offsets = Vec::with_capacity(n_cols); - for &size in &col_sizes { offsets.push(col_offset); col_offset += size; } + for &size in &col_sizes { + offsets.push(col_offset); + col_offset += size; + } let tmp_path = dir.join("matrix.pcmx.tmp"); let mut out = BufWriter::new(File::create(&tmp_path)?); out.write_all(&PCMX_MAGIC)?; out.write_all(&[0u8; 4])?; out.write_all(&(meta.n as u64).to_le_bytes())?; out.write_all(&(n_cols as u64).to_le_bytes())?; - for &off in &offsets { out.write_all(&off.to_le_bytes())?; } - for c in 0..n_cols { io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; } + for &off in &offsets { + out.write_all(&off.to_le_bytes())?; + } + for c in 0..n_cols { + io::copy(&mut File::open(col_path(dir, c))?, &mut out)?; + } out.flush()?; drop(out); fs::rename(&tmp_path, &packed_path)?; - for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; } + for c in 0..n_cols { + fs::remove_file(col_path(dir, c))?; + } fs::remove_file(dir.join("meta.json"))?; Ok(()) } // ── PersistentCompactIntMatrix — public enum ────────────────────────────────── -pub enum PersistentCompactIntMatrix { +pub enum PersistentIntMatrix { Columnar(ColumnarCompactIntMatrix), Packed(PackedCompactIntMatrix), Sparse(PersistentSparseCompactIntMatrix), } -impl PersistentCompactIntMatrix { +impl PersistentIntMatrix { /// Checks (in order): `counts/matrix.pcmx` → Packed; `counts/meta.json` /// → Columnar; `counts/singleton_values.pciv` → Sparse. Mirrors /// `PersistentBitMatrix::open`'s own Packed → Columnar → Sparse @@ -283,17 +382,24 @@ impl PersistentCompactIntMatrix { pub fn open(layer_dir: &Path) -> io::Result { let counts_dir = layer_dir.join("counts"); if counts_dir.join("matrix.pcmx").exists() { - return Ok(Self::Packed(PackedCompactIntMatrix::open(&counts_dir.join("matrix.pcmx"))?)); + return Ok(Self::Packed(PackedCompactIntMatrix::open( + &counts_dir.join("matrix.pcmx"), + )?)); } if MatrixMeta::load(&counts_dir).is_ok() { return Ok(Self::Columnar(ColumnarCompactIntMatrix::open(&counts_dir)?)); } if sparse_is_present(&counts_dir) { - return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open(&counts_dir)?)); + return Ok(Self::Sparse(PersistentSparseCompactIntMatrix::open( + &counts_dir, + )?)); } Err(io::Error::new( io::ErrorKind::NotFound, - format!("no count matrix found in {} — run 'obikmer upgrade'", layer_dir.display()), + format!( + "no count matrix found in {} — run 'obikmer upgrade'", + layer_dir.display() + ), )) } @@ -303,8 +409,8 @@ impl PersistentCompactIntMatrix { pub fn storage_kind(&self) -> StorageKind { match self { Self::Columnar(_) => StorageKind::Columnar, - Self::Packed(_) => StorageKind::Packed, - Self::Sparse(_) => StorageKind::Sparse, + Self::Packed(_) => StorageKind::Packed, + Self::Sparse(_) => StorageKind::Sparse, } } @@ -330,11 +436,19 @@ impl PersistentCompactIntMatrix { #[inline] pub fn n(&self) -> usize { - match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows, Self::Sparse(m) => m.n() } + match self { + Self::Columnar(m) => m.n(), + Self::Packed(m) => m.n_rows, + Self::Sparse(m) => m.n(), + } } #[inline] pub fn n_cols(&self) -> usize { - match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols, Self::Sparse(m) => m.n_cols() } + match self { + Self::Columnar(m) => m.n_cols(), + Self::Packed(m) => m.n_cols, + Self::Sparse(m) => m.n_cols(), + } } #[inline] @@ -349,30 +463,38 @@ impl PersistentCompactIntMatrix { pub fn col_view(&self, c: usize) -> IntSliceView<'_> { match self { Self::Columnar(m) => m.col(c).view(), - Self::Packed(m) => m.col_view(c), - Self::Sparse(_) => panic!("col_view() not available on Sparse PersistentCompactIntMatrix"), + Self::Packed(m) => m.col_view(c), + Self::Sparse(_) => { + panic!("col_view() not available on Sparse PersistentCompactIntMatrix") + } } } pub fn col_persist(&self, c: usize, path: &Path) -> io::Result { match self { Self::Columnar(m) => PersistentCompactIntVecBuilder::build_from(m.col(c), path), - Self::Packed(m) => m.col_persist(c, path), - Self::Sparse(_) => Err(io::Error::new(io::ErrorKind::Unsupported, - "col_persist not available on Sparse PersistentCompactIntMatrix")), + Self::Packed(m) => m.col_persist(c, path), + Self::Sparse(_) => Err(io::Error::new( + io::ErrorKind::Unsupported, + "col_persist not available on Sparse PersistentCompactIntMatrix", + )), } } #[inline] pub fn row(&self, slot: usize) -> Box<[u32]> { - match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot), Self::Sparse(m) => m.row(slot) } + match self { + Self::Columnar(m) => m.row(slot), + Self::Packed(m) => m.row(slot), + Self::Sparse(m) => m.row(slot), + } } #[inline] pub fn fill_row(&self, slot: usize, buf: &mut [u32]) { match self { Self::Columnar(m) => m.fill_row(slot, buf), - Self::Packed(m) => m.fill_row(slot, buf), - Self::Sparse(m) => m.fill_row(slot, buf), + Self::Packed(m) => m.fill_row(slot, buf), + Self::Sparse(m) => m.fill_row(slot, buf), } } @@ -418,73 +540,86 @@ impl PersistentCompactIntMatrix { /// `Columnar`/`Packed` reuse the shared sorted-slot batching in /// `views::nonzero_triples`. Boxed, not `impl Iterator`, because the /// match arms are genuinely different concrete types. - pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box + 'a> { + pub fn nonzero_iter<'a>( + &'a self, + slots: &'a [usize], + ) -> Box + 'a> { match self { Self::Sparse(m) => Box::new(m.nonzero_iter(slots)), - Self::Columnar(_) | Self::Packed(_) => { - Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false)) - } + Self::Columnar(_) | Self::Packed(_) => Box::new(nonzero_triples( + slots, + self.n_cols(), + |c| self.col_view(c), + false, + )), } } #[inline] pub fn sum(&self) -> Array1 { - match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum(), Self::Sparse(m) => m.sum() } + match self { + Self::Columnar(m) => m.sum(), + Self::Packed(m) => m.sum(), + Self::Sparse(m) => m.sum(), + } } #[inline] pub fn count_nonzero(&self) -> Array1 { match self { Self::Columnar(m) => m.count_nonzero(), - Self::Packed(m) => m.count_nonzero(), - Self::Sparse(m) => m.count_nonzero(), + Self::Packed(m) => m.count_nonzero(), + Self::Sparse(m) => m.count_nonzero(), } } #[inline] pub fn partial_bray_dist_matrix(&self) -> Array2 { match self { Self::Columnar(m) => m.partial_bray_dist_matrix(), - Self::Packed(m) => m.partial_bray_dist_matrix(), - Self::Sparse(m) => CountPartials::partial_bray(m), + Self::Packed(m) => m.partial_bray_dist_matrix(), + Self::Sparse(m) => CountPartials::partial_bray(m), } } #[inline] pub fn partial_euclidean_dist_matrix(&self) -> Array2 { match self { Self::Columnar(m) => m.partial_euclidean_dist_matrix(), - Self::Packed(m) => m.partial_euclidean_dist_matrix(), - Self::Sparse(m) => CountPartials::partial_euclidean(m), + Self::Packed(m) => m.partial_euclidean_dist_matrix(), + Self::Sparse(m) => CountPartials::partial_euclidean(m), } } #[inline] - pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2, Array2) { + pub fn partial_threshold_jaccard_dist_matrix( + &self, + threshold: u32, + ) -> (Array2, Array2) { match self { Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold), - Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold), - Self::Sparse(m) => CountPartials::partial_threshold_jaccard(m, threshold), + Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold), + Self::Sparse(m) => CountPartials::partial_threshold_jaccard(m, threshold), } } #[inline] pub fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1) -> Array2 { match self { Self::Columnar(m) => m.partial_relfreq_bray_dist_matrix(col_sums), - Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums), - Self::Sparse(m) => CountPartials::partial_relfreq_bray(m, col_sums), + Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums), + Self::Sparse(m) => CountPartials::partial_relfreq_bray(m, col_sums), } } #[inline] pub fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1) -> Array2 { match self { Self::Columnar(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums), - Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums), - Self::Sparse(m) => CountPartials::partial_relfreq_euclidean(m, col_sums), + Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums), + Self::Sparse(m) => CountPartials::partial_relfreq_euclidean(m, col_sums), } } #[inline] pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1) -> Array2 { match self { Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), - Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), - Self::Sparse(m) => CountPartials::partial_hellinger(m, col_sums), + Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), + Self::Sparse(m) => CountPartials::partial_hellinger(m, col_sums), } } #[inline] @@ -497,40 +632,60 @@ impl PersistentCompactIntMatrix { use crate::traits::{ColumnWeights, CountPartials}; -impl ColumnWeights for PersistentCompactIntMatrix { +impl ColumnWeights for PersistentIntMatrix { #[inline] - fn col_weights(&self) -> Array1 { self.sum() } + fn col_weights(&self) -> Array1 { + self.sum() + } #[inline] - fn partial_kmer_counts(&self) -> Array1 { self.count_nonzero() } + fn partial_kmer_counts(&self) -> Array1 { + self.count_nonzero() + } } -impl CountPartials for PersistentCompactIntMatrix { +impl CountPartials for PersistentIntMatrix { #[inline] - fn partial_bray(&self) -> Array2 { self.partial_bray_dist_matrix() } + fn partial_bray(&self) -> Array2 { + self.partial_bray_dist_matrix() + } #[inline] - fn partial_euclidean(&self) -> Array2 { self.partial_euclidean_dist_matrix() } + fn partial_euclidean(&self) -> Array2 { + self.partial_euclidean_dist_matrix() + } #[inline] - fn partial_threshold_jaccard(&self, t: u32) -> (Array2, Array2) { self.partial_threshold_jaccard_dist_matrix(t) } + fn partial_threshold_jaccard(&self, t: u32) -> (Array2, Array2) { + self.partial_threshold_jaccard_dist_matrix(t) + } #[inline] - fn partial_relfreq_bray(&self, g: &Array1) -> Array2 { self.partial_relfreq_bray_dist_matrix(g) } + fn partial_relfreq_bray(&self, g: &Array1) -> Array2 { + self.partial_relfreq_bray_dist_matrix(g) + } #[inline] - fn partial_relfreq_euclidean(&self, g: &Array1) -> Array2 { self.partial_relfreq_euclidean_dist_matrix(g) } + fn partial_relfreq_euclidean(&self, g: &Array1) -> Array2 { + self.partial_relfreq_euclidean_dist_matrix(g) + } #[inline] - fn partial_hellinger(&self, g: &Array1) -> Array2 { self.partial_hellinger_euclidean_dist_matrix(g) } + fn partial_hellinger(&self, g: &Array1) -> Array2 { + self.partial_hellinger_euclidean_dist_matrix(g) + } } // ── Builder ─────────────────────────────────────────────────────────────────── pub struct PersistentCompactIntMatrixBuilder { - dir: PathBuf, - n: usize, + dir: PathBuf, + n: usize, n_cols: usize, } impl PersistentCompactIntMatrixBuilder { pub fn new(n: usize, dir: &Path) -> io::Result { fs::create_dir_all(dir)?; - Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 }) + Ok(Self { + dir: dir.to_path_buf(), + n, + n_cols: 0, + }) } /// Resume appending columns to a Columnar matrix directory already @@ -541,13 +696,21 @@ impl PersistentCompactIntMatrixBuilder { /// on-disk column naming or `MatrixMeta` themselves. pub fn resume(dir: &Path) -> io::Result { let meta = MatrixMeta::load(dir)?; - Ok(Self { dir: dir.to_path_buf(), n: meta.n, n_cols: meta.n_cols }) + Ok(Self { + dir: dir.to_path_buf(), + n: meta.n, + n_cols: meta.n_cols, + }) } #[inline] - pub fn n(&self) -> usize { self.n } + pub fn n(&self) -> usize { + self.n + } #[inline] - pub fn n_cols(&self) -> usize { self.n_cols } + pub fn n_cols(&self) -> usize { + self.n_cols + } pub fn add_col(&mut self) -> io::Result { let path = col_path(&self.dir, self.n_cols); self.n_cols += 1; @@ -569,14 +732,22 @@ impl PersistentCompactIntMatrixBuilder { } pub fn close(self) -> io::Result<()> { - MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir) + MatrixMeta { + n: self.n, + n_cols: self.n_cols, + } + .save(&self.dir) } } // ── MatrixGroupOps ──────────────────────────────────────────────────────────── -impl MatrixGroupOps for PersistentCompactIntMatrix { - fn partial_group_presence_count(&self, g: &ColGroup, threshold: u32) -> io::Result { +impl MatrixGroupOps for PersistentIntMatrix { + fn partial_group_presence_count( + &self, + g: &ColGroup, + threshold: u32, + ) -> io::Result { chunked_presence_count(self.n(), &g.indices, |b, c| { b.inc_predicate_fast(self.col_view(c), |v| v >= threshold) }) @@ -585,7 +756,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix { fn partial_group_sum(&self, g: &ColGroup) -> io::Result { let n = self.n(); let mut result = TempCompactIntVecBuilder::new(n)?; - for &c in &g.indices { result.add(self.col_view(c)); } + for &c in &g.indices { + result.add(self.col_view(c)); + } result.freeze() } @@ -603,7 +776,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix { let mut result = TempCompactIntVecBuilder::new(n)?; if let Some((&first, rest)) = g.indices.split_first() { result.add(self.col_view(first)); - for &c in rest { result.min(self.col_view(c)); } + for &c in rest { + result.min(self.col_view(c)); + } } result.freeze() } @@ -611,7 +786,9 @@ impl MatrixGroupOps for PersistentCompactIntMatrix { fn partial_group_max(&self, g: &ColGroup) -> io::Result { let n = self.n(); let mut result = TempCompactIntVecBuilder::new(n)?; - for &c in &g.indices { result.max(self.col_view(c)); } + for &c in &g.indices { + result.max(self.col_view(c)); + } result.freeze() } } diff --git a/src/obicompactvec/src/lib.rs b/src/obicompactvec/src/lib.rs index bb7782b2..a965ff39 100644 --- a/src/obicompactvec/src/lib.rs +++ b/src/obicompactvec/src/lib.rs @@ -1,41 +1,50 @@ -mod bitvec; mod bitmatrix; +mod bitvec; mod builder; mod colgroup; mod eliasfano; mod fixedintvec; mod format; -mod rankselect; mod intmatrix; mod layer_meta; mod matrix_builder; mod meta; mod mmap_file; +mod rankselect; mod reader; mod sparse_intmatrix; mod storage_kind; mod tempbitvec; mod tempintvec; -mod views; pub mod traits; +mod views; +pub use bitmatrix::{ + PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, + PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix, +}; pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder}; -pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range}; -pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder}; -pub use eliasfano::{EliasFano, EliasFanoBuilder}; -pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix}; pub use builder::PersistentCompactIntVecBuilder; pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask}; -pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix}; +pub use eliasfano::{EliasFano, EliasFanoBuilder}; +pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range}; +pub use intmatrix::{ + PersistentCompactIntMatrixBuilder, PersistentIntMatrix, pack_compact_int_matrix, +}; + pub use layer_meta::LayerMeta; pub use matrix_builder::{ColBuilder, MatrixBuilder}; -pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter}; -pub use sparse_intmatrix::{PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, pack_sparse_compact_int_matrix}; +pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder}; +pub use reader::{Iter as CompactIntVecIter, PersistentCompactIntVec}; +pub use sparse_intmatrix::{ + PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, + pack_sparse_compact_int_matrix, +}; pub use storage_kind::StorageKind; pub use tempbitvec::{TempBitVec, TempBitVecBuilder}; pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder}; pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials}; -pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter}; +pub use views::{BitSliceIter, BitSliceView, IntSliceView, IntSliceViewIter}; #[cfg(test)] #[path = "tests/mod.rs"] diff --git a/src/obicompactvec/src/sparse_intmatrix.rs b/src/obicompactvec/src/sparse_intmatrix.rs index 7f58d2c5..2d596f48 100644 --- a/src/obicompactvec/src/sparse_intmatrix.rs +++ b/src/obicompactvec/src/sparse_intmatrix.rs @@ -35,11 +35,17 @@ use crate::builder::PersistentCompactIntVecBuilder; use crate::eliasfano::{EliasFano, EliasFanoBuilder}; use crate::reader::PersistentCompactIntVec; use crate::traits::{BitPartials, ColumnWeights, CountPartials}; -use crate::{PersistentCompactIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder}; +use crate::{PersistentIntMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder}; -fn singleton_values_path(dir: &Path) -> PathBuf { dir.join("singleton_values.pciv") } -fn multi_values_path(dir: &Path) -> PathBuf { dir.join("multi_values.pciv") } -fn multi_offsets_base(dir: &Path) -> PathBuf { dir.join("multi_offsets") } +fn singleton_values_path(dir: &Path) -> PathBuf { + dir.join("singleton_values.pciv") +} +fn multi_values_path(dir: &Path) -> PathBuf { + dir.join("multi_values.pciv") +} +fn multi_offsets_base(dir: &Path) -> PathBuf { + dir.join("multi_offsets") +} /// Lightweight disk probe: true iff a sparse count matrix has already been /// written at `dir` — the marker [`PersistentCompactIntMatrix::open`]/ @@ -73,9 +79,13 @@ impl PersistentSparseCompactIntMatrix { } #[inline] - pub fn n(&self) -> usize { self.support.n() } + pub fn n(&self) -> usize { + self.support.n() + } #[inline] - pub fn n_cols(&self) -> usize { self.support.n_cols() } + pub fn n_cols(&self) -> usize { + self.support.n_cols() + } /// Calls `f(col, value)` once per non-zero column at `slot`, in /// ascending column order — the shared decode branch behind every @@ -104,7 +114,11 @@ impl PersistentSparseCompactIntMatrix { /// absent). pub fn get(&self, c: usize, slot: usize) -> u32 { let mut found = 0u32; - self.for_each_cell_in_row(slot, |g, v| if g == c { found = v; }); + self.for_each_cell_in_row(slot, |g, v| { + if g == c { + found = v; + } + }); found } @@ -138,27 +152,32 @@ impl PersistentSparseCompactIntMatrix { /// Yields every `(idx into slots, col, value)` triple, row by row, in /// `slots` order — mirrors /// [`PersistentSparseBitMatrix::nonzero_iter`]. - pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator + 'a { + pub fn nonzero_iter<'a>( + &'a self, + slots: &'a [usize], + ) -> impl Iterator + 'a { let mut next_slot = 0usize; let mut cur_idx = 0usize; let mut buf: Vec<(usize, u32)> = Vec::new(); let mut buf_pos = 0usize; - std::iter::from_fn(move || loop { - if buf_pos < buf.len() { - let (g, v) = buf[buf_pos]; - buf_pos += 1; - return Some((cur_idx, g, v)); + std::iter::from_fn(move || { + loop { + if buf_pos < buf.len() { + let (g, v) = buf[buf_pos]; + buf_pos += 1; + return Some((cur_idx, g, v)); + } + if next_slot >= slots.len() { + return None; + } + cur_idx = next_slot; + let slot = slots[next_slot]; + next_slot += 1; + buf.clear(); + self.for_each_cell_in_row(slot, |g, v| buf.push((g, v))); + buf_pos = 0; } - if next_slot >= slots.len() { - return None; - } - cur_idx = next_slot; - let slot = slots[next_slot]; - next_slot += 1; - buf.clear(); - self.for_each_cell_in_row(slot, |g, v| buf.push((g, v))); - buf_pos = 0; }) } @@ -203,7 +222,11 @@ impl PersistentSparseCompactIntMatrix { fn count_geq(&self, threshold: u32) -> Array1 { let mut counts = vec![0u64; self.n_cols()]; for slot in 0..self.n() { - self.for_each_cell_in_row(slot, |g, v| if v >= threshold { counts[g] += 1; }); + self.for_each_cell_in_row(slot, |g, v| { + if v >= threshold { + counts[g] += 1; + } + }); } Array1::from(counts) } @@ -261,9 +284,13 @@ impl PersistentSparseCompactIntMatrix { impl ColumnWeights for PersistentSparseCompactIntMatrix { #[inline] - fn col_weights(&self) -> Array1 { self.sum() } + fn col_weights(&self) -> Array1 { + self.sum() + } #[inline] - fn partial_kmer_counts(&self) -> Array1 { self.count_nonzero() } + fn partial_kmer_counts(&self) -> Array1 { + self.count_nonzero() + } } impl CountPartials for PersistentSparseCompactIntMatrix { @@ -278,7 +305,9 @@ impl CountPartials for PersistentSparseCompactIntMatrix { fn partial_bray(&self) -> Array2 { let mut m = self.row_major_pairwise(|_, a, _, b| (a as u64).min(b as u64)); let sum = self.sum(); - for i in 0..self.n_cols() { m[[i, i]] = sum[i]; } + for i in 0..self.n_cols() { + m[[i, i]] = sum[i]; + } m } @@ -329,7 +358,8 @@ impl CountPartials for PersistentSparseCompactIntMatrix { // left to patch. return BitPartials::partial_jaccard(&self.support); } - let mut inter = self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64); + let mut inter = + self.row_major_pairwise(|_, a, _, b| (a >= threshold && b >= threshold) as u64); let mut union = Array2::::zeros((n, n)); for i in 0..n { for j in 0..n { @@ -363,7 +393,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix { }); let sum = self.sum(); for i in 0..self.n_cols() { - m[[i, i]] = if global[i] > 0 { sum[i] as f64 / global[i] as f64 } else { 0.0 }; + m[[i, i]] = if global[i] > 0 { + sum[i] as f64 / global[i] as f64 + } else { + 0.0 + }; } m } @@ -382,9 +416,21 @@ impl CountPartials for PersistentSparseCompactIntMatrix { if i != j { let sa = global[i] as f64; let sb = global[j] as f64; - let sq_a = if sa > 0.0 { sq[i] as f64 / (sa * sa) } else { 0.0 }; - let sq_b = if sb > 0.0 { sq[j] as f64 / (sb * sb) } else { 0.0 }; - let cross = if sa > 0.0 && sb > 0.0 { dot[[i, j]] / (sa * sb) } else { 0.0 }; + let sq_a = if sa > 0.0 { + sq[i] as f64 / (sa * sa) + } else { + 0.0 + }; + let sq_b = if sb > 0.0 { + sq[j] as f64 / (sb * sb) + } else { + 0.0 + }; + let cross = if sa > 0.0 && sb > 0.0 { + dot[[i, j]] / (sa * sb) + } else { + 0.0 + }; m[[i, j]] = sq_a + sq_b - 2.0 * cross; } } @@ -409,7 +455,11 @@ impl CountPartials for PersistentSparseCompactIntMatrix { let sb = global[j] as f64; let pa_sum = if sa > 0.0 { sum[i] as f64 / sa } else { 0.0 }; let pb_sum = if sb > 0.0 { sum[j] as f64 / sb } else { 0.0 }; - let cross = if sa > 0.0 && sb > 0.0 { sqrt_dot[[i, j]] / (sa * sb).sqrt() } else { 0.0 }; + let cross = if sa > 0.0 && sb > 0.0 { + sqrt_dot[[i, j]] / (sa * sb).sqrt() + } else { + 0.0 + }; m[[i, j]] = pa_sum + pb_sum - 2.0 * cross; } } @@ -461,7 +511,6 @@ impl PersistentSparseCompactIntMatrixBuilder { self.support.push_row(cols); } - /// Builds a sparse matrix from an already-built dense /// [`PersistentCompactIntMatrix`] (`Columnar` or `Packed`) — same /// batched-transpose shape as @@ -475,7 +524,7 @@ impl PersistentSparseCompactIntMatrixBuilder { /// pairs are appended to its own buffer in strictly ascending column /// order "for free", matching `push_row`'s ascending-`cols` contract /// without an explicit sort. - pub fn build_from_dense(dense: &PersistentCompactIntMatrix, dir: &Path) -> io::Result { + pub fn build_from_dense(dense: &PersistentIntMatrix, dir: &Path) -> io::Result { let n = dense.n(); let n_cols = dense.n_cols(); let mut builder = Self::new(n, n_cols, dir)?; @@ -500,23 +549,34 @@ impl PersistentSparseCompactIntMatrixBuilder { } pub fn close(self) -> io::Result<()> { - let Self { support, dir, singleton_values, multi_values, multi_row_offsets } = self; + let Self { + support, + dir, + singleton_values, + multi_values, + multi_row_offsets, + } = self; support.close()?; - let mut sv = PersistentCompactIntVecBuilder::new(singleton_values.len(), &singleton_values_path(&dir))?; + let mut sv = PersistentCompactIntVecBuilder::new( + singleton_values.len(), + &singleton_values_path(&dir), + )?; for (i, &v) in singleton_values.iter().enumerate() { sv.set(i, v); } sv.close()?; - let mut mv = PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?; + let mut mv = + PersistentCompactIntVecBuilder::new(multi_values.len(), &multi_values_path(&dir))?; for (i, &v) in multi_values.iter().enumerate() { mv.set(i, v); } mv.close()?; let universe = multi_values.len() as u64 + 1; - let mut mo = EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?; + let mut mo = + EliasFanoBuilder::new(multi_row_offsets.len(), universe, &multi_offsets_base(&dir))?; for &o in &multi_row_offsets { mo.push(o); } @@ -544,7 +604,7 @@ impl PersistentSparseCompactIntMatrixBuilder { /// half-written one). Idempotent — a no-op if `singleton_values.pciv` /// already exists. pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> { - use crate::intmatrix::{col_path, ColumnarCompactIntMatrix, PackedCompactIntMatrix}; + use crate::intmatrix::{ColumnarCompactIntMatrix, PackedCompactIntMatrix, col_path}; if is_present(dir) { return Ok(()); @@ -552,12 +612,12 @@ pub fn pack_sparse_compact_int_matrix(dir: &Path) -> io::Result<()> { let packed_path = dir.join("matrix.pcmx"); if packed_path.exists() { - let dense = PersistentCompactIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?); + let dense = PersistentIntMatrix::Packed(PackedCompactIntMatrix::open(&packed_path)?); PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; drop(dense); fs::remove_file(&packed_path)?; } else { - let dense = PersistentCompactIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?); + let dense = PersistentIntMatrix::Columnar(ColumnarCompactIntMatrix::open(dir)?); let n_cols = dense.n_cols(); PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, dir)?.close()?; drop(dense); diff --git a/src/obicompactvec/src/tests/colgroup.rs b/src/obicompactvec/src/tests/colgroup.rs index d1c7cf1f..b11ba39f 100644 --- a/src/obicompactvec/src/tests/colgroup.rs +++ b/src/obicompactvec/src/tests/colgroup.rs @@ -1,25 +1,26 @@ use tempfile::tempdir; use crate::{ - ColGroup, MatrixGroupOps, - PersistentBitMatrix, PersistentBitMatrixBuilder, - PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, + ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder, + PersistentIntMatrix, PersistentCompactIntMatrixBuilder, }; use crate::{PersistentBitVecBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder}; // ── helpers ─────────────────────────────────────────────────────────────────── -fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) { +fn make_int_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) { let n = cols.first().map_or(0, |c| c.len()); let dir = tempdir().unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap(); for &col in cols { let mut cb = b.add_col().unwrap(); - for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); } + for (slot, &v) in col.iter().enumerate() { + cb.set(slot, v); + } cb.close().unwrap(); } b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); (dir, m) } @@ -30,7 +31,9 @@ fn make_bit_matrix(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix) let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap(); for &col in cols { let mut cb = b.add_col().unwrap(); - for (slot, &v) in col.iter().enumerate() { cb.set(slot, v); } + for (slot, &v) in col.iter().enumerate() { + cb.set(slot, v); + } cb.close().unwrap(); } b.close().unwrap(); @@ -115,9 +118,13 @@ fn mask_with_zeros_selected_slots() { // count vec [10, 20, 30, 40], mask [T, F, T, F] → [10, 0, 30, 0] let dir = tempdir().unwrap(); let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap(); - v.set(0, 10); v.set(1, 20); v.set(2, 30); v.set(3, 40); + v.set(0, 10); + v.set(1, 20); + v.set(2, 30); + v.set(3, 40); let mut mask = PersistentBitVecBuilder::new(4, &dir.path().join("m.pbiv")).unwrap(); - mask.set(0, true); mask.set(2, true); + mask.set(0, true); + mask.set(2, true); v.mask_with(mask.view()); v.close().unwrap(); let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap(); @@ -132,9 +139,12 @@ fn mask_with_overflow_slot_zeroed() { // overflow slot (value 500) masked out → removed from overflow, primary=0 let dir = tempdir().unwrap(); let mut v = PersistentCompactIntVecBuilder::new(3, &dir.path().join("v.pciv")).unwrap(); - v.set(0, 10); v.set(1, 500); v.set(2, 5); + v.set(0, 10); + v.set(1, 500); + v.set(2, 5); let mut mask = PersistentBitVecBuilder::new(3, &dir.path().join("m.pbiv")).unwrap(); - mask.set(0, true); mask.set(2, true); // slot 1 masked out + mask.set(0, true); + mask.set(2, true); // slot 1 masked out v.mask_with(mask.view()); v.close().unwrap(); let r = PersistentCompactIntVec::open(&dir.path().join("v.pciv")).unwrap(); @@ -142,14 +152,20 @@ fn mask_with_overflow_slot_zeroed() { assert_eq!(r.get(1), 0); assert_eq!(r.get(2), 5); let ov: Vec<_> = r.view().overflow_entries().collect(); - assert!(ov.is_empty(), "overflow entry for masked-out slot should be gone"); + assert!( + ov.is_empty(), + "overflow entry for masked-out slot should be gone" + ); } #[test] fn mask_with_all_ones_is_noop() { let dir = tempdir().unwrap(); let mut v = PersistentCompactIntVecBuilder::new(4, &dir.path().join("v.pciv")).unwrap(); - v.set(0, 300); v.set(1, 1); v.set(2, 0); v.set(3, 42); + v.set(0, 300); + v.set(1, 1); + v.set(2, 0); + v.set(3, 42); let mask = PersistentBitVecBuilder::new_ones(4, &dir.path().join("m.pbiv")).unwrap(); v.mask_with(mask.view()); v.close().unwrap(); @@ -167,9 +183,9 @@ fn bit_partial_group_presence_count() { // col0=[T,F,T,F], col1=[T,T,F,F], col2=[F,T,T,F] // group {0,1,2}: counts = [2, 2, 2, 0] let (_d, m) = make_bit_matrix(&[ - &[true, false, true, false], - &[true, true, false, false], - &[false,true, true, false], + &[true, false, true, false], + &[true, true, false, false], + &[false, true, true, false], ]); let g = ColGroup::new("g", vec![0, 1, 2]); let result = m.partial_group_presence_count(&g, 1).unwrap(); @@ -184,10 +200,7 @@ fn bit_partial_group_presence_count() { #[test] fn bit_partial_group_any() { // col0=[T,F,F], col1=[F,F,T], group {0,1}: any = [T, F, T] - let (_d, m) = make_bit_matrix(&[ - &[true, false, false], - &[false, false, true], - ]); + let (_d, m) = make_bit_matrix(&[&[true, false, false], &[false, false, true]]); let g = ColGroup::new("g", vec![0, 1]); let result = m.partial_group_any(&g, 1).unwrap(); assert_eq!(result.get(0), true); diff --git a/src/obicompactvec/src/tests/intmatrix.rs b/src/obicompactvec/src/tests/intmatrix.rs index 1e9cb573..51aecb36 100644 --- a/src/obicompactvec/src/tests/intmatrix.rs +++ b/src/obicompactvec/src/tests/intmatrix.rs @@ -1,9 +1,12 @@ use tempfile::tempdir; -use crate::{pack_compact_int_matrix, pack_sparse_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, StorageKind}; use crate::traits::CountPartials; +use crate::{ + PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, + PersistentIntMatrix, StorageKind, pack_compact_int_matrix, pack_sparse_compact_int_matrix, +}; -fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) { +fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) { let n = cols.first().map_or(0, |c| c.len()); let dir = tempdir().unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(n, &dir.path().join("counts")).unwrap(); @@ -15,7 +18,7 @@ fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatri cb.close().unwrap(); } b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); (dir, m) } @@ -31,7 +34,7 @@ fn single_col_roundtrip() { col.close().unwrap(); b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.n_cols(), 1); assert_eq!(m.n(), 4); assert_eq!(&*m.row(0), &[10u32]); @@ -45,14 +48,18 @@ fn two_cols_roundtrip() { let dir = tempdir().unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(3, &dir.path().join("counts")).unwrap(); let mut col0 = b.add_col().unwrap(); - col0.set(0, 1); col0.set(1, 2); col0.set(2, 3); + col0.set(0, 1); + col0.set(1, 2); + col0.set(2, 3); col0.close().unwrap(); let mut col1 = b.add_col().unwrap(); - col1.set(0, 10); col1.set(1, 20); col1.set(2, 30); + col1.set(0, 10); + col1.set(1, 20); + col1.set(2, 30); col1.close().unwrap(); b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.n_cols(), 2); assert_eq!(&*m.row(0), &[1u32, 10]); assert_eq!(&*m.row(1), &[2u32, 20]); @@ -66,7 +73,9 @@ fn resume_continues_n_cols_and_appends_columns() { let mut b = PersistentCompactIntMatrixBuilder::new(3, &counts).unwrap(); let mut col0 = b.add_col().unwrap(); - col0.set(0, 1); col0.set(1, 2); col0.set(2, 3); + col0.set(0, 1); + col0.set(1, 2); + col0.set(2, 3); col0.close().unwrap(); b.close().unwrap(); @@ -74,11 +83,13 @@ fn resume_continues_n_cols_and_appends_columns() { assert_eq!(resumed.n(), 3); assert_eq!(resumed.n_cols(), 1); let mut col1 = resumed.add_col().unwrap(); - col1.set(0, 10); col1.set(1, 20); col1.set(2, 30); + col1.set(0, 10); + col1.set(1, 20); + col1.set(2, 30); col1.close().unwrap(); resumed.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.n_cols(), 2); assert_eq!(&*m.row(0), &[1u32, 10]); assert_eq!(&*m.row(1), &[2u32, 20]); @@ -90,17 +101,21 @@ fn resume_twice_keeps_appending() { let dir = tempdir().unwrap(); let counts = dir.path().join("counts"); - PersistentCompactIntMatrixBuilder::new(2, &counts).unwrap().close().unwrap(); + PersistentCompactIntMatrixBuilder::new(2, &counts) + .unwrap() + .close() + .unwrap(); for v in [1u32, 2, 3] { let mut b = PersistentCompactIntMatrixBuilder::resume(&counts).unwrap(); let mut col = b.add_col().unwrap(); - col.set(0, v); col.set(1, v * 10); + col.set(0, v); + col.set(1, v * 10); col.close().unwrap(); b.close().unwrap(); } - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.n_cols(), 3); assert_eq!(&*m.row(0), &[1u32, 2, 3]); assert_eq!(&*m.row(1), &[10u32, 20, 30]); @@ -111,11 +126,12 @@ fn col_accessor() { let dir = tempdir().unwrap(); let mut b = PersistentCompactIntMatrixBuilder::new(2, &dir.path().join("counts")).unwrap(); let mut col0 = b.add_col().unwrap(); - col0.set(0, 5); col0.set(1, 7); + col0.set(0, 5); + col0.set(1, 7); col0.close().unwrap(); b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.col(0).get(0), 5); assert_eq!(m.col(0).get(1), 7); } @@ -126,7 +142,7 @@ fn zero_cols_roundtrip() { let b = PersistentCompactIntMatrixBuilder::new(10, &dir.path().join("counts")).unwrap(); b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.n_cols(), 0); assert_eq!(m.n(), 10); } @@ -139,7 +155,9 @@ fn bray_dist_matrix_symmetry_and_diagonal() { let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]); let dm = m.bray_dist_matrix(); let n = m.n_cols(); - for i in 0..n { assert_eq!(dm[[i, i]], 0.0, "diagonal"); } + for i in 0..n { + assert_eq!(dm[[i, i]], 0.0, "diagonal"); + } for i in 0..n { for j in 0..n { assert!((dm[[i, j]] - dm[[j, i]]).abs() < 1e-12, "symmetry"); @@ -174,7 +192,7 @@ fn jaccard_dist_matrix_values_match_pairwise() { #[test] fn partial_bray_dist_matrix_consistent() { let (_d, m) = make_matrix(&[&[1, 0, 1], &[1, 1, 0], &[0, 1, 1]]); - let sum_min = m.partial_bray_dist_matrix(); + let sum_min = m.partial_bray_dist_matrix(); let col_sums = m.sum(); let n = m.n_cols(); @@ -189,7 +207,11 @@ fn partial_bray_dist_matrix_consistent() { for i in 0..n { for j in i + 1..n { let denom = col_sums[i] + col_sums[j]; - let dist = if denom == 0 { 0.0 } else { 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 }; + let dist = if denom == 0 { + 0.0 + } else { + 1.0 - 2.0 * sum_min[[i, j]] as f64 / denom as f64 + }; let expected = m.col(i).bray_dist(m.col(j)); assert!((dist - expected).abs() < 1e-12, "[{i},{j}]"); } @@ -249,13 +271,18 @@ fn partial_relfreq_bray_matches_full() { let (_d, m) = make_matrix(&[&[1, 0, 2], &[0, 1, 1], &[1, 1, 0]]); let col_sums = m.sum(); let partial = m.partial_relfreq_bray_dist_matrix(&col_sums); - let full = m.relfreq_bray_dist_matrix(); + let full = m.relfreq_bray_dist_matrix(); let n = m.n_cols(); // partial[i,j] = sum_min_relfreq; full[i,j] = 1 - sum_min_relfreq (off-diagonal only) for i in 0..n { for j in 0..n { - if i == j { continue; } - assert!((partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, "[{i},{j}]"); + if i == j { + continue; + } + assert!( + (partial[[i, j]] - (1.0 - full[[i, j]])).abs() < 1e-12, + "[{i},{j}]" + ); } } } @@ -265,12 +292,15 @@ fn partial_relfreq_euclidean_matches_full() { let (_d, m) = make_matrix(&[&[3, 0], &[0, 4], &[1, 1]]); let col_sums = m.sum(); let partial = m.partial_relfreq_euclidean_dist_matrix(&col_sums); - let full = m.relfreq_euclidean_dist_matrix(); + let full = m.relfreq_euclidean_dist_matrix(); let n = m.n_cols(); for i in 0..n { for j in 0..n { // partial = squared euclidean; full = sqrt(partial) - assert!((partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, "[{i},{j}]"); + assert!( + (partial[[i, j]].sqrt() - full[[i, j]]).abs() < 1e-12, + "[{i},{j}]" + ); } } } @@ -280,12 +310,15 @@ fn partial_hellinger_matches_full() { let (_d, m) = make_matrix(&[&[3, 0], &[0, 4], &[1, 1]]); let col_sums = m.sum(); let partial = m.partial_hellinger_euclidean_dist_matrix(&col_sums); - let full = m.hellinger_dist_matrix(); + let full = m.hellinger_dist_matrix(); let n = m.n_cols(); for i in 0..n { for j in 0..n { // partial / sqrt(2) gives the Hellinger distance - assert!((partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, "[{i},{j}]"); + assert!( + (partial[[i, j]].sqrt() / std::f64::consts::SQRT_2 - full[[i, j]]).abs() < 1e-12, + "[{i},{j}]" + ); } } } @@ -295,7 +328,7 @@ fn col_view_packed_values() { // Build Columnar with overflow values (≥ 255), pack, reopen as Packed, exercise col_view(). let (dir, _col) = make_matrix(&[&[10, 300, 500], &[200, 50, 1000]]); pack_compact_int_matrix(&dir.path().join("counts")).unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); // col 0: [10, 300, 500] — two overflow slots let v0 = m.col_view(0); @@ -326,10 +359,10 @@ fn col_view_packed_matches_columnar() { // Re-build in a separate dir so we can pack without touching m_col's files. let (dir_pack, _) = make_matrix(data); pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap(); - let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap(); + let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap(); for c in 0..data.len() { - let col_ref = m_col.col(c); + let col_ref = m_col.col(c); let col_view = m_pack.col_view(c); assert_eq!(col_view.len(), col_ref.len()); for s in 0..col_ref.len() { @@ -337,7 +370,7 @@ fn col_view_packed_matches_columnar() { } assert_eq!(col_view.sum(), col_ref.sum(), "col={c} sum"); let mut ov_view: Vec<(usize, u32)> = col_view.overflow_entries().collect(); - let mut ov_ref: Vec<(usize, u32)> = col_ref.view().overflow_entries().collect(); + let mut ov_ref: Vec<(usize, u32)> = col_ref.view().overflow_entries().collect(); ov_view.sort_unstable_by_key(|&(s, _)| s); ov_ref.sort_unstable_by_key(|&(s, _)| s); assert_eq!(ov_view, ov_ref, "col={c} overflow_entries"); @@ -355,7 +388,7 @@ fn nonzero_iter_matches_row() { let (dir_col, m_col) = make_matrix(data); let (dir_pack, _) = make_matrix(data); pack_compact_int_matrix(&dir_pack.path().join("counts")).unwrap(); - let m_pack = PersistentCompactIntMatrix::open(dir_pack.path()).unwrap(); + let m_pack = PersistentIntMatrix::open(dir_pack.path()).unwrap(); let slots = [4usize, 0, 3, 1]; let mut expected: Vec<(usize, usize, u32)> = Vec::new(); @@ -390,7 +423,7 @@ fn sparse_roundtrip_matches_columnar() { let (dir_col, m_col) = make_matrix(data); let (dir_sparse, _) = make_matrix(data); pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); - let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap(); + let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap(); assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse); assert_eq!(m_sparse.n(), m_col.n()); @@ -441,7 +474,7 @@ fn sparse_count_partials_match_dense() { let (dir_col, m_col) = make_matrix(data); let (dir_sparse, _) = make_matrix(data); pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); - let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap(); + let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap(); assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse); let n = m_col.n_cols(); @@ -461,7 +494,11 @@ fn sparse_count_partials_match_dense() { let eucl_sparse = m_sparse.partial_euclidean_dist_matrix(); for i in 0..n { for j in 0..n { - close(eucl_col[[i, j]], eucl_sparse[[i, j]], &format!("partial_euclidean[{i},{j}]")); + close( + eucl_col[[i, j]], + eucl_sparse[[i, j]], + &format!("partial_euclidean[{i},{j}]"), + ); } } @@ -469,9 +506,16 @@ fn sparse_count_partials_match_dense() { // (support shortcut), and >1 (general row-major path). for threshold in [0u32, 1, 2, 3] { let (inter_col, union_col) = m_col.partial_threshold_jaccard_dist_matrix(threshold); - let (inter_sparse, union_sparse) = m_sparse.partial_threshold_jaccard_dist_matrix(threshold); - assert_eq!(inter_col, inter_sparse, "partial_threshold_jaccard inter, threshold={threshold}"); - assert_eq!(union_col, union_sparse, "partial_threshold_jaccard union, threshold={threshold}"); + let (inter_sparse, union_sparse) = + m_sparse.partial_threshold_jaccard_dist_matrix(threshold); + assert_eq!( + inter_col, inter_sparse, + "partial_threshold_jaccard inter, threshold={threshold}" + ); + assert_eq!( + union_col, union_sparse, + "partial_threshold_jaccard union, threshold={threshold}" + ); } // partial_relfreq_bray @@ -479,7 +523,11 @@ fn sparse_count_partials_match_dense() { let rfb_sparse = m_sparse.partial_relfreq_bray_dist_matrix(&global); for i in 0..n { for j in 0..n { - close(rfb_col[[i, j]], rfb_sparse[[i, j]], &format!("partial_relfreq_bray[{i},{j}]")); + close( + rfb_col[[i, j]], + rfb_sparse[[i, j]], + &format!("partial_relfreq_bray[{i},{j}]"), + ); } } @@ -488,7 +536,11 @@ fn sparse_count_partials_match_dense() { let rfe_sparse = m_sparse.partial_relfreq_euclidean_dist_matrix(&global); for i in 0..n { for j in 0..n { - close(rfe_col[[i, j]], rfe_sparse[[i, j]], &format!("partial_relfreq_euclidean[{i},{j}]")); + close( + rfe_col[[i, j]], + rfe_sparse[[i, j]], + &format!("partial_relfreq_euclidean[{i},{j}]"), + ); } } @@ -497,7 +549,11 @@ fn sparse_count_partials_match_dense() { let hel_sparse = m_sparse.partial_hellinger_euclidean_dist_matrix(&global); for i in 0..n { for j in 0..n { - close(hel_col[[i, j]], hel_sparse[[i, j]], &format!("partial_hellinger[{i},{j}]")); + close( + hel_col[[i, j]], + hel_sparse[[i, j]], + &format!("partial_hellinger[{i},{j}]"), + ); } } @@ -514,7 +570,7 @@ fn sparse_roundtrip_from_packed() { let (dir_sparse, _) = make_matrix(data); pack_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); pack_sparse_compact_int_matrix(&dir_sparse.path().join("counts")).unwrap(); - let m_sparse = PersistentCompactIntMatrix::open(dir_sparse.path()).unwrap(); + let m_sparse = PersistentIntMatrix::open(dir_sparse.path()).unwrap(); assert_eq!(m_sparse.storage_kind(), StorageKind::Sparse); for slot in 0..m_col.n() { assert_eq!(&*m_sparse.row(slot), &*m_col.row(slot), "slot={slot}"); @@ -540,7 +596,10 @@ fn partial_relfreq_bray_additive_across_split() { let n = m_full.n_cols(); for i in 0..n { for j in 0..n { - assert!((combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, "[{i},{j}]"); + assert!( + (combined[[i, j]] - full_partial[[i, j]]).abs() < 1e-12, + "[{i},{j}]" + ); } } } @@ -548,10 +607,12 @@ fn partial_relfreq_bray_additive_across_split() { // ── collect_slots_values tests ──────────────────────────────────────────────────────────── fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) { - let dir = tempdir().unwrap(); + let dir = tempdir().unwrap(); let path = dir.path().join("c.pciv"); let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap(); - for (i, &v) in counts.iter().enumerate() { b.set(i, v); } + for (i, &v) in counts.iter().enumerate() { + b.set(i, v); + } b.close().unwrap(); let r = PersistentCompactIntVec::open(&path).unwrap(); (dir, r) @@ -592,7 +653,10 @@ fn pciv_collect_slots_values_empty() { fn pciv_collect_slots_values_out_of_bounds_panics() { let (_dir, v) = make_pciv(&[10u32, 20]); let result = std::panic::catch_unwind(|| v.collect_slots_values(&[0, 2])); - assert!(result.is_err(), "collect_slots_values should panic on out-of-bounds slot"); + assert!( + result.is_err(), + "collect_slots_values should panic on out-of-bounds slot" + ); } // IntSliceView collect_slots_values (same logic, exercised through the view) diff --git a/src/obicompactvec/src/tests/sparse_intmatrix.rs b/src/obicompactvec/src/tests/sparse_intmatrix.rs index c4089da8..3c41683e 100644 --- a/src/obicompactvec/src/tests/sparse_intmatrix.rs +++ b/src/obicompactvec/src/tests/sparse_intmatrix.rs @@ -1,13 +1,14 @@ use tempfile::tempdir; use crate::{ - pack_compact_int_matrix, pack_sparse_compact_int_matrix, ColumnWeights, PersistentCompactIntMatrix, - PersistentCompactIntMatrixBuilder, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, + ColumnWeights, PersistentIntMatrix, PersistentCompactIntMatrixBuilder, + PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder, + pack_compact_int_matrix, pack_sparse_compact_int_matrix, }; /// Builds a dense `PersistentCompactIntMatrix` from column-major `u32` data /// — mirrors `tests/intmatrix.rs`'s own `make_matrix` helper. -fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) { +fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentIntMatrix) { let n = cols.first().map_or(0, |c| c.len()); let dir = tempdir().unwrap(); let counts_dir = dir.path().join("counts"); @@ -20,17 +21,21 @@ fn make_dense(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix cb.close().unwrap(); } b.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); (dir, m) } /// Builds a sparse int matrix directly from row-major `u32` data (one /// slice per row, `n_cols` values each) — mirrors `tests/sparse.rs`'s /// `make_sparse` helper. -fn make_sparse(rows: &[&[u32]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) { +fn make_sparse( + rows: &[&[u32]], + n_cols: usize, +) -> (tempfile::TempDir, PersistentSparseCompactIntMatrix) { let dir = tempdir().unwrap(); let sparse_dir = dir.path().join("sparse"); - let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap(); + let mut b = + PersistentSparseCompactIntMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap(); let mut cols = Vec::new(); let mut values = Vec::new(); for row in rows { @@ -54,11 +59,11 @@ fn basic_roundtrip_singletons_and_multi() { // shares its *support* (columns {0,1}) with another but has different // values — the whole point of not deduplicating values. let rows: Vec<&[u32]> = vec![ - &[5, 0, 0, 0], // singleton: genome 0, value 5 - &[0, 0, 3, 0], // singleton: genome 2, value 3 - &[7, 2, 0, 0], // multi: {0:7, 1:2} - &[0, 0, 4, 9], // multi: {2:4, 3:9} - &[1, 6, 0, 0], // multi: same support {0,1} as row 2, different values + &[5, 0, 0, 0], // singleton: genome 0, value 5 + &[0, 0, 3, 0], // singleton: genome 2, value 3 + &[7, 2, 0, 0], // multi: {0:7, 1:2} + &[0, 0, 4, 9], // multi: {2:4, 3:9} + &[1, 6, 0, 0], // multi: same support {0,1} as row 2, different values ]; let (_dir, m) = make_sparse(&rows, 4); assert_eq!(m.n(), 5); @@ -70,11 +75,7 @@ fn basic_roundtrip_singletons_and_multi() { #[test] fn get_matches_row() { - let rows: Vec<&[u32]> = vec![ - &[3, 0, 5], - &[0, 8, 0], - &[1, 2, 4], - ]; + let rows: Vec<&[u32]> = vec![&[3, 0, 5], &[0, 8, 0], &[1, 2, 4]]; let (_dir, m) = make_sparse(&rows, 3); for (slot, row) in rows.iter().enumerate() { for (c, &expected) in row.iter().enumerate() { @@ -85,11 +86,7 @@ fn get_matches_row() { #[test] fn fill_row_matches_row() { - let rows: Vec<&[u32]> = vec![ - &[9, 0, 2], - &[0, 4, 0], - &[1, 1, 1], - ]; + let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]]; let (_dir, m) = make_sparse(&rows, 3); let mut buf = vec![0u32; 3]; for slot in 0..3 { @@ -120,11 +117,7 @@ fn fill_sub_matrix_matches_row() { #[test] fn nonzero_iter_matches_row() { - let rows: Vec<&[u32]> = vec![ - &[9, 0, 2], - &[0, 4, 0], - &[1, 1, 1], - ]; + let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1]]; let (_dir, m) = make_sparse(&rows, 3); let slots = [2usize, 0, 1]; let mut expected: Vec<(usize, usize, u32)> = Vec::new(); @@ -143,12 +136,7 @@ fn nonzero_iter_matches_row() { #[test] fn sum_and_count_nonzero_match_naive() { - let rows: Vec<&[u32]> = vec![ - &[9, 0, 2], - &[0, 4, 0], - &[1, 1, 1], - &[0, 0, 6], - ]; + let rows: Vec<&[u32]> = vec![&[9, 0, 2], &[0, 4, 0], &[1, 1, 1], &[0, 0, 6]]; let (_dir, m) = make_sparse(&rows, 3); let mut expected_sum = [0u64; 3]; let mut expected_count = [0u64; 3]; @@ -193,14 +181,21 @@ fn reopen_after_close_matches_original() { let rows: Vec> = (0..500) .map(|i| { (0..37) - .map(|c| if (i * 7 + c * 3) % 11 == 0 { ((i + c) % 250 + 1) as u32 } else { 0 }) + .map(|c| { + if (i * 7 + c * 3) % 11 == 0 { + ((i + c) % 250 + 1) as u32 + } else { + 0 + } + }) .collect() }) .collect(); let dir = tempdir().unwrap(); let sparse_dir = dir.path().join("sparse"); { - let mut b = PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap(); + let mut b = + PersistentSparseCompactIntMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap(); let mut cols = Vec::new(); let mut values = Vec::new(); for row in &rows { @@ -229,10 +224,7 @@ fn reopen_after_close_matches_original() { fn overflow_values_roundtrip() { // Values >= 255 exercise the PersistentCompactIntVec overflow path in // both the singleton and multi value streams. - let rows: Vec<&[u32]> = vec![ - &[1000, 0, 0], - &[0, 50_000, 300], - ]; + let rows: Vec<&[u32]> = vec![&[1000, 0, 0], &[0, 50_000, 300]]; let (_dir, m) = make_sparse(&rows, 3); for (slot, &expected) in rows.iter().enumerate() { assert_eq!(&*m.row(slot), expected, "row {slot}"); @@ -246,14 +238,13 @@ fn overflow_values_roundtrip() { /// `tests::sparse::pack_sparse_bit_matrix_transposes_columnar_directly`. #[test] fn pack_sparse_compact_int_matrix_transposes_columnar_directly() { - let cols: &[&[u32]] = &[ - &[9, 0, 2, 0], - &[0, 4, 0, 7], - &[1, 1, 300, 1], - ]; + let cols: &[&[u32]] = &[&[9, 0, 2, 0], &[0, 4, 0, 7], &[1, 1, 300, 1]]; let (dir, dense) = make_dense(cols); let counts_dir = dir.path().join("counts"); - assert!(!counts_dir.join("matrix.pcmx").exists(), "still Columnar before packing"); + assert!( + !counts_dir.join("matrix.pcmx").exists(), + "still Columnar before packing" + ); let expected_rows: Vec> = (0..dense.n()).map(|s| dense.row(s)).collect(); let n_cols = dense.n_cols(); @@ -262,9 +253,18 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() { pack_sparse_compact_int_matrix(&counts_dir).unwrap(); assert!(counts_dir.join("singleton_values.pciv").exists()); - assert!(!counts_dir.join("meta.json").exists(), "columnar meta.json cleaned up"); - assert!(!counts_dir.join("col_000000.pciv").exists(), "columnar column files cleaned up"); - assert!(!counts_dir.join("matrix.pcmx").exists(), "never materialised"); + assert!( + !counts_dir.join("meta.json").exists(), + "columnar meta.json cleaned up" + ); + assert!( + !counts_dir.join("col_000000.pciv").exists(), + "columnar column files cleaned up" + ); + assert!( + !counts_dir.join("matrix.pcmx").exists(), + "never materialised" + ); let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap(); assert_eq!(sparse.n(), expected_rows.len()); @@ -282,10 +282,7 @@ fn pack_sparse_compact_int_matrix_transposes_columnar_directly() { /// it's still used as the transpose source and cleaned up afterward. #[test] fn pack_sparse_compact_int_matrix_from_already_packed_matrix() { - let cols: &[&[u32]] = &[ - &[9, 0, 2], - &[0, 4, 300], - ]; + let cols: &[&[u32]] = &[&[9, 0, 2], &[0, 4, 300]]; let (dir, dense) = make_dense(cols); let counts_dir = dir.path().join("counts"); let expected_rows: Vec> = (0..dense.n()).map(|s| dense.row(s)).collect(); @@ -296,7 +293,10 @@ fn pack_sparse_compact_int_matrix_from_already_packed_matrix() { assert!(counts_dir.join("matrix.pcmx").exists()); pack_sparse_compact_int_matrix(&counts_dir).unwrap(); - assert!(!counts_dir.join("matrix.pcmx").exists(), "packed intermediate cleaned up"); + assert!( + !counts_dir.join("matrix.pcmx").exists(), + "packed intermediate cleaned up" + ); let sparse = PersistentSparseCompactIntMatrix::open(&counts_dir).unwrap(); assert_eq!(sparse.n_cols(), n_cols); diff --git a/src/obikindex/src/layer/content_layer.rs b/src/obikindex/src/layer/content_layer.rs index 217ce03e..916d2a6c 100644 --- a/src/obikindex/src/layer/content_layer.rs +++ b/src/obikindex/src/layer/content_layer.rs @@ -26,7 +26,7 @@ use std::path::{Path, PathBuf}; use crate::layer::utils::LAYERNAME_SUFFIX; -use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; +use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix}; use obikseq::CanonicalKmer; use crate::index::error::OKIResult; @@ -63,7 +63,7 @@ pub enum KmerLayer { Count { dir: PathBuf, id: usize, - layer: TypedLayer, + layer: TypedLayer, }, Presence { dir: PathBuf, @@ -99,7 +99,7 @@ impl KmerLayer { already_open => return Ok(already_open), }; if dir.join(COUNTS_DIR).exists() { - let layer = TypedLayer::::open(&dir)?; + let layer = TypedLayer::::open(&dir)?; Ok(KmerLayer::Count { dir, id, layer }) } else { let layer = TypedLayer::::open(&dir)?; @@ -221,7 +221,10 @@ impl KmerLayer { /// format-agnostic column-major fetch, delegating to whichever matrix /// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/ /// `PersistentCompactIntMatrix::nonzero_iter`). - pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box + 'a> { + pub fn nonzero_iter<'a>( + &'a self, + slots: &'a [usize], + ) -> Box + 'a> { match self { KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots), KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots), diff --git a/src/obikindex/src/layer/tests/typed_layer.rs b/src/obikindex/src/layer/tests/typed_layer.rs index 7c941999..3de5eb9b 100644 --- a/src/obikindex/src/layer/tests/typed_layer.rs +++ b/src/obikindex/src/layer/tests/typed_layer.rs @@ -1,6 +1,6 @@ use super::*; use obicompactvec::PersistentSparseBitMatrixBuilder; -use obikseq::{set_k, Unitig}; +use obikseq::{Unitig, set_k}; use obiskio::DEFAULT_BLOCK_BITS; use tempfile::tempdir; @@ -13,7 +13,8 @@ fn write_unitigs(dir: &Path, seqs: &[&[u8]]) { } fn all_canonical_kmers(dir: &Path) -> Vec { - UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)).unwrap() + UnitigFileReader::open_sequential(&dir.join(UNITIGS_FILE)) + .unwrap() .iter_indexed_canonical_kmers() .map(|(kmer, _, _)| kmer) .collect() @@ -34,13 +35,17 @@ fn canonical_kmer_iter_matches_reader() { let dir = tempdir().unwrap(); write_unitigs(dir.path(), &[b"AAAACGT", b"TTTTGCA"]); - let from_iter: Vec = obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE)) - .unwrap() - .collect(); + let from_iter: Vec = + obiskio::CanonicalKmerIter::new(&dir.path().join(UNITIGS_FILE)) + .unwrap() + .collect(); let from_reader: Vec = all_canonical_kmers(dir.path()); assert_eq!(from_iter.len(), from_reader.len(), "different kmer counts"); - assert_eq!(from_iter, from_reader, "CanonicalKmerIter and UnitigFileReader disagree"); + assert_eq!( + from_iter, from_reader, + "CanonicalKmerIter and UnitigFileReader disagree" + ); } // ── Generic `TypedLayer` over dense vs. sparse presence storage ─────────────── @@ -64,9 +69,14 @@ fn presence_layer_generic_over_sparse_matches_dense() { // kmer iff `(kmer's raw bits + g)` is even. Doesn't need to be // biologically meaningful, just the same on both sides of the // dense/sparse comparison below. - TypedLayer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &mode, n_genomes, |kmer, g| { - (kmer.raw().wrapping_add(g as u64)) % 2 == 0 - }).unwrap(); + TypedLayer::::build_presence( + dir.path(), + DEFAULT_BLOCK_BITS, + &mode, + n_genomes, + |kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0, + ) + .unwrap(); let dense_layer = TypedLayer::::open(dir.path()).unwrap(); assert!(dense_layer.n_cols() >= 1); @@ -76,10 +86,13 @@ fn presence_layer_generic_over_sparse_matches_dense() { // same directory, distinct filenames — see // `DevDocMD/architecture/siblings.md`'s sparse-matrix section). let dense_matrix = obicompactvec::PersistentBitMatrix::open(dir.path()).unwrap(); - PersistentSparseBitMatrixBuilder::build_from_dense(&dense_matrix, &dir.path().join(PRESENCE_DIR)) - .unwrap() - .close() - .unwrap(); + PersistentSparseBitMatrixBuilder::build_from_dense( + &dense_matrix, + &dir.path().join(PRESENCE_DIR), + ) + .unwrap() + .close() + .unwrap(); let sparse_layer = TypedLayer::::open(dir.path()).unwrap(); @@ -88,7 +101,10 @@ fn presence_layer_generic_over_sparse_matches_dense() { let n = dense_layer.n(); assert_eq!(n, sparse_layer.n()); let slots: Vec = (0..n).collect(); - assert_eq!(dense_layer.sub_matrix(&slots), sparse_layer.sub_matrix(&slots)); + assert_eq!( + dense_layer.sub_matrix(&slots), + sparse_layer.sub_matrix(&slots) + ); // The matrix-agnostic, MPHF-only surface (from the generic // `impl TypedLayer`) must also agree: same kmer set, @@ -124,15 +140,25 @@ fn count_layer_transparently_reads_sparse_after_pack() { // Deterministic, arbitrary count function — same shape as the presence // test's arbitrary predicate, just producing a small count instead of a // bool. - TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| { - (0..n_genomes) - .map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32) - .collect() - }) + TypedLayer::<()>::build_with_matrix( + dir.path(), + DEFAULT_BLOCK_BITS, + &mode, + false, + n_genomes, + |kmer| { + (0..n_genomes) + .map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32) + .collect() + }, + ) .unwrap(); - let dense_layer = TypedLayer::::open(dir.path()).unwrap(); - assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar); + let dense_layer = TypedLayer::::open(dir.path()).unwrap(); + assert_eq!( + dense_layer.storage_kind(), + obicompactvec::StorageKind::Columnar + ); let n = dense_layer.n(); let slots: Vec = (0..n).collect(); let dense_sub = dense_layer.sub_matrix(&slots); @@ -146,8 +172,11 @@ fn count_layer_transparently_reads_sparse_after_pack() { // in production. obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap(); - let sparse_layer = TypedLayer::::open(dir.path()).unwrap(); - assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse); + let sparse_layer = TypedLayer::::open(dir.path()).unwrap(); + assert_eq!( + sparse_layer.storage_kind(), + obicompactvec::StorageKind::Sparse + ); assert_eq!(sparse_layer.n(), n); assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols()); assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub); @@ -167,13 +196,20 @@ fn count_layer_reports_count_content_and_columnar_storage() { set_k(4); let dir = tempdir().unwrap(); write_unitigs(dir.path(), &[b"AAAACGT"]); - TypedLayer::::build(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, |_| 1) - .unwrap(); - let layer = TypedLayer::::open(dir.path()).unwrap(); + TypedLayer::::build(dir.pat + h(), DEFAUL + T_BLOCK_BITS, &Index + Mode::Exact, |_| 1) + , + ).unwrap(); + let layer = TypedLayer::::open(dir.path()).unwrap(); assert_eq!(layer.content(), LayerContent::Count); assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); - assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact); + assert_eq!( + layer.evidence_kind(), + crate::layer::mphf_layer::EvidenceKind::Exact + ); } #[test] @@ -181,9 +217,14 @@ fn presence_layer_reports_presence_content_and_columnar_storage() { set_k(4); let dir = tempdir().unwrap(); write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]); - TypedLayer::::build_presence(dir.path(), DEFAULT_BLOCK_BITS, &IndexMode::Exact, 2, |kmer, g| { - (kmer.raw().wrapping_add(g as u64)) % 2 == 0 - }).unwrap(); + TypedLayer::::build_presence( + dir.path(), + DEFAULT_BLOCK_BITS, + &IndexMode::Exact, + 2, + |kmer, g| (kmer.raw().wrapping_add(g as u64)) % 2 == 0, + ) + .unwrap(); let layer = TypedLayer::::open(dir.path()).unwrap(); assert_eq!(layer.content(), LayerContent::Presence); diff --git a/src/obikindex/src/layer/typed_layer.rs b/src/obikindex/src/layer/typed_layer.rs index 27980039..af37d198 100644 --- a/src/obikindex/src/layer/typed_layer.rs +++ b/src/obikindex/src/layer/typed_layer.rs @@ -1,7 +1,7 @@ // use crate::layer::utils::layer_dir; use obicompactvec::{ BinaryMatrix, ColBuilder, MatrixBuilder, PersistentBitMatrix, PersistentBitMatrixBuilder, - PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentSparseBitMatrix, + PersistentCompactIntMatrixBuilder, PersistentIntMatrix, PersistentSparseBitMatrix, }; use obikseq::CanonicalKmer; use obiskio::{UnitigFileReader, UnitigFileWriter}; @@ -47,10 +47,10 @@ impl LayerData for () { fn read(&self, _slot: usize) {} } -impl LayerData for PersistentCompactIntMatrix { +impl LayerData for PersistentIntMatrix { type Item = Box<[u32]>; fn open(layer_dir: &Path) -> OKIResult { - PersistentCompactIntMatrix::open(layer_dir).map_err(OKIError::Io) + PersistentIntMatrix::open(layer_dir).map_err(OKIError::Io) } fn read(&self, slot: usize) -> Box<[u32]> { self.row(slot) @@ -112,7 +112,7 @@ pub trait HasLayerContent { const CONTENT: LayerContent; } -impl HasLayerContent for PersistentCompactIntMatrix { +impl HasLayerContent for PersistentIntMatrix { const CONTENT: LayerContent = LayerContent::Count; } @@ -135,9 +135,9 @@ pub trait HasStorageKind { fn storage_kind(&self) -> obicompactvec::StorageKind; } -impl HasStorageKind for PersistentCompactIntMatrix { +impl HasStorageKind for PersistentIntMatrix { fn storage_kind(&self) -> obicompactvec::StorageKind { - PersistentCompactIntMatrix::storage_kind(self) + PersistentIntMatrix::storage_kind(self) } } @@ -342,7 +342,7 @@ impl TypedLayer<()> { // ── Mode 2 — count matrix ───────────────────────────────────────────────────── -impl TypedLayer { +impl TypedLayer { pub fn build( out_dir: &Path, block_bits: u8, @@ -377,12 +377,12 @@ impl TypedLayer { // ── Mode 2 — count matrix column append ────────────────────────────────────── -impl TypedLayer { +impl TypedLayer { pub fn append_genome_column( layer_dir: &Path, value_of: impl Fn(usize) -> u32, ) -> OKIResult<()> { - PersistentCompactIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of) + PersistentIntMatrix::append_column(&layer_dir.join(COUNTS_DIR), value_of) .map_err(OKIError::Io) } @@ -418,7 +418,10 @@ impl TypedLayer { /// Every nonzero `(idx into slots, col, value)` triple among `slots` — /// delegates to `PersistentCompactIntMatrix::nonzero_iter`. - pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box + 'a> { + pub fn nonzero_iter<'a>( + &'a self, + slots: &'a [usize], + ) -> Box + 'a> { self.data.nonzero_iter(slots) } } @@ -477,7 +480,10 @@ impl> + BinaryMatrix + obicompactvec::ColumnWeig impl TypedLayer { /// Every nonzero `(idx into slots, col, value)` triple among `slots` — /// delegates to `PersistentBitMatrix::nonzero_iter`. - pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box + 'a> { + pub fn nonzero_iter<'a>( + &'a self, + slots: &'a [usize], + ) -> Box + 'a> { self.data.nonzero_iter(slots) } diff --git a/src/obikindexer/src/extensions/private_builder.rs b/src/obikindexer/src/extensions/private_builder.rs index 8134b99f..661d9194 100644 --- a/src/obikindexer/src/extensions/private_builder.rs +++ b/src/obikindexer/src/extensions/private_builder.rs @@ -24,14 +24,14 @@ use std::fs; use std::io; use std::path::Path; +use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; use cacheline_ef::{CachelineEf, CachelineEfVec}; use epserde::prelude::*; -use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec}; +use obicompactvec::{PersistentIntMatrix, PersistentCompactIntVec}; use obidebruinj::GraphDeBruijn; use obikindex::layer::IndexMode; use obikindex::layer::{KmerLayer, TypedLayer}; use obikindex::{KmerIndex, OKIError, OKIResult}; -use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; use obiskio::{SKError, SKFileMeta, SKFileReader}; use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; @@ -139,7 +139,9 @@ impl PrivateBuilder for KmerIndex { mode: &IndexMode, block_bits: u8, ) -> Result { - let layer0 = self.layer0(i).map_err(|e| io::Error::other(e.to_string()))?; + let layer0 = self + .layer0(i) + .map_err(|e| io::Error::other(e.to_string()))?; let layer0_dir = layer0.dir(); let dedup_path = layer0.dereplicated_superkmers_path(); if !dedup_path.exists() { @@ -193,15 +195,12 @@ impl PrivateBuilder for KmerIndex { let n_kmers = if with_counts { let n = write_graph_as_unitigs(g, layer0_dir) .map_err(|e| io::Error::other(e.to_string()))?; - TypedLayer::::build( - layer0_dir, - block_bits, - mode, - |kmer| match (&mphf1_opt, &counts1_opt) { + TypedLayer::::build(layer0_dir, block_bits, mode, |kmer| { + match (&mphf1_opt, &counts1_opt) { (Some(mphf), Some(counts)) => counts.get(mphf.index(&kmer.raw())), _ => 1, - }, - ) + } + }) .map_err(|e| io::Error::other(e.to_string()))?; n } else { diff --git a/src/obikmerge/src/partition_merge.rs b/src/obikmerge/src/partition_merge.rs index 6a60baba..b21ba8e8 100644 --- a/src/obikmerge/src/partition_merge.rs +++ b/src/obikmerge/src/partition_merge.rs @@ -34,7 +34,7 @@ fn presence(mode: MergeMode) -> bool { mod matrix_builder_tests { use tempfile::tempdir; - use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; + use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix}; use super::{ColBuilder, MatrixBuilder}; @@ -90,7 +90,7 @@ mod matrix_builder_tests { col.close().unwrap(); mb.close().unwrap(); - let m = PersistentCompactIntMatrix::open(dir.path()).unwrap(); + let m = PersistentIntMatrix::open(dir.path()).unwrap(); assert_eq!(m.n_cols(), 2); assert_eq!(&*m.row(0), &[0u32, 7]); assert_eq!(&*m.row(1), &[0u32, 42]); @@ -422,8 +422,13 @@ pub(crate) fn merge_partition( move |data: Pass2Data, push: &PipelineSender>, delta: &PipelineSender| { - if let Pass2Data::SrcLayer((col_offset, src_n, unitigs_path, src_layer, _guard)) = - data + if let Pass2Data::SrcLayer(( + col_offset, + src_n, + unitigs_path, + src_layer, + _guard, + )) = data { // _guard dropped at end of block, releasing the slot. // `src_layer` (MPHF + matrix) was already opened up @@ -455,8 +460,10 @@ pub(crate) fn merge_partition( } } if !batch.is_empty() { - push.send(Ok(Pass2Data::RawBatch((col_offset, src_n, src_layer, batch)))) - .ok(); + push.send(Ok(Pass2Data::RawBatch(( + col_offset, src_n, src_layer, batch, + )))) + .ok(); count += 1; } delta.send(count - 1).ok(); @@ -480,8 +487,9 @@ pub(crate) fn merge_partition( // per kmer — matches the underlying matrix's own // column-major layout. let slots = src_layer.hash_batch(&kmers); - let mut cols: Vec> = - (0..src_n).map(|_| Vec::with_capacity(kmers.len())).collect(); + let mut cols: Vec> = (0..src_n) + .map(|_| Vec::with_capacity(kmers.len())) + .collect(); src_layer.fill_sub_matrix(&slots, &mut cols); // Membership against dst, grouped by dst layer diff --git a/src/obikphylo/src/matrix_store.rs b/src/obikphylo/src/matrix_store.rs index 74ddf02f..67054479 100644 --- a/src/obikphylo/src/matrix_store.rs +++ b/src/obikphylo/src/matrix_store.rs @@ -1,15 +1,15 @@ +use obicompactvec::{PersistentBitMatrix, PersistentIntMatrix}; +use obikidxcache::LayeredStore; use obikindex::OKIResult; use obikindex::layer::open_data; -use obikidxcache::LayeredStore; -use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; -use obikindex::load_meta; use obikindex::KmerIndex; +use obikindex::load_meta; impl KmerIndex { /// Open all count matrices for partition `part`, one per layer. /// Layers without a `counts/` directory are skipped. - pub fn count_store(&self, part: usize) -> OKIResult> { + pub fn count_store(&self, part: usize) -> OKIResult> { let index_dir = self.index_dir(part); if !index_dir.exists() { return Ok(LayeredStore::new(vec![])); diff --git a/src/obikselect/src/select_layer.rs b/src/obikselect/src/select_layer.rs index 8b0a9700..3f1648a3 100644 --- a/src/obikselect/src/select_layer.rs +++ b/src/obikselect/src/select_layer.rs @@ -11,8 +11,8 @@ use std::io; use std::path::Path; use obicompactvec::{ - ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, - PersistentCompactIntMatrix, TempBitVec, TempCompactIntVec, + ColGroup, MatrixBuilder, MatrixGroupOps, PersistentBitMatrix, PersistentIntMatrix, TempBitVec, + TempCompactIntVec, }; use obikindex::layer::{KmerLayer, LayerContent}; use obikindex::{KmerIndex, OKIError, OKIResult}; @@ -39,13 +39,15 @@ impl AggOp { /// `--aggregate-op`. pub fn parse(s: &str) -> Result { match s.to_lowercase().as_str() { - "any" => Ok(AggOp::Any), - "all" => Ok(AggOp::All), + "any" => Ok(AggOp::Any), + "all" => Ok(AggOp::All), "none" => Ok(AggOp::None), - "sum" => Ok(AggOp::Sum), - "min" => Ok(AggOp::Min), - "max" => Ok(AggOp::Max), - other => Err(format!("unknown aggregation operator: {other}; valid: any, all, none, sum, min, max")), + "sum" => Ok(AggOp::Sum), + "min" => Ok(AggOp::Min), + "max" => Ok(AggOp::Max), + other => Err(format!( + "unknown aggregation operator: {other}; valid: any, all, none, sum, min, max" + )), } } @@ -75,7 +77,11 @@ enum AggResult { Int(TempCompactIntVec), } -fn compute_group(mat: &dyn MatrixGroupOps, spec: &OutputCol, threshold: u32) -> io::Result { +fn compute_group( + mat: &dyn MatrixGroupOps, + spec: &OutputCol, + threshold: u32, +) -> io::Result { let g = ColGroup::new(spec.label.clone(), spec.indices.clone()); Ok(match spec.op { AggOp::Any => AggResult::Bit(mat.partial_group_any(&g, threshold)?), @@ -145,18 +151,23 @@ pub(crate) fn select_partition( let group_mat: Box = match src_layer.content() { LayerContent::Count => { - Box::new(PersistentCompactIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?) + Box::new(PersistentIntMatrix::open(src_layer.dir()).map_err(OKIError::Io)?) } LayerContent::Presence => { Box::new(PersistentBitMatrix::open(src_layer.dir()).map_err(OKIError::Io)?) } }; - let data_subdir = if output_presence { "presence" } else { "counts" }; + let data_subdir = if output_presence { + "presence" + } else { + "counts" + }; let data_dir = dst_layer_dir.join(data_subdir); fs::create_dir_all(&data_dir).map_err(OKIError::Io)?; - let mut builder = MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?; + let mut builder = + MatrixBuilder::new(output_presence, n, &data_dir).map_err(OKIError::Io)?; for spec in specs { let r = compute_group(group_mat.as_ref(), spec, threshold).map_err(OKIError::Io)?; add_result(&mut builder, r).map_err(OKIError::Io)?;