diff --git a/src/obicompactvec/examples/compare_sparse_dense.rs b/src/obicompactvec/examples/compare_sparse_dense.rs new file mode 100644 index 00000000..0e5e0d08 --- /dev/null +++ b/src/obicompactvec/examples/compare_sparse_dense.rs @@ -0,0 +1,69 @@ +//! Diagnostic: build a `PersistentSparseBitMatrix` from a real dense +//! `PersistentBitMatrix` (batched `build_from_dense`) and verify the result +//! is bit-for-bit identical, on an actual large presence matrix. +//! +//! Usage: cargo run --release --example compare_sparse_dense -p obicompactvec -- +//! (layer_dir is the directory containing `presence/matrix.pbmx`, e.g. +//! `benchmark/tmp/bacteria/partitions/part_00000/index/layer_1`) + +use std::error::Error; +use std::path::Path; +use std::time::Instant; + +use obicompactvec::{PersistentBitMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder}; + +fn main() -> Result<(), Box> { + let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense "); + let layer_dir = Path::new(&layer_dir); + + let t0 = Instant::now(); + let dense = PersistentBitMatrix::open(layer_dir)?; + println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols()); + + let out_dir = tempfile::tempdir()?; + + let t0 = Instant::now(); + let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?; + println!("build_from_dense: {:?}", t0.elapsed()); + + compare(&dense, &sparse) +} + +fn compare(dense: &PersistentBitMatrix, sparse: &PersistentSparseBitMatrix) -> Result<(), Box> { + let n = dense.n(); + let n_cols = dense.n_cols(); + assert_eq!(n, sparse.n(), "n mismatch"); + assert_eq!(n_cols, sparse.n_cols(), "n_cols mismatch"); + + let mut dense_row = vec![0u32; n_cols]; + let mut sparse_row = vec![0u32; n_cols]; + let mut total_cells = 0usize; + let mut mismatched_cells = 0usize; + let mut first_mismatch = None; + + let t0 = Instant::now(); + for slot in 0..n { + dense.fill_row(slot, &mut dense_row); + sparse.fill_row(slot, &mut sparse_row); + + for c in 0..n_cols { + total_cells += 1; + if dense_row[c] != sparse_row[c] { + mismatched_cells += 1; + if first_mismatch.is_none() { + first_mismatch = Some((slot, c, dense_row[c], sparse_row[c])); + } + } + } + } + println!("comparaison bit-a-bit: {:?} ({total_cells} cellules)", t0.elapsed()); + + if let Some((slot, col, d, s)) = first_mismatch { + eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}"); + println!("MISMATCHES: {mismatched_cells} cellules differentes sur {total_cells}"); + } else { + println!("OK: toutes les {total_cells} cellules sont identiques entre dense et sparse."); + } + + Ok(()) +} diff --git a/src/obicompactvec/src/bitmatrix/persistent.rs b/src/obicompactvec/src/bitmatrix/persistent.rs index ac94e033..dacfdf99 100644 --- a/src/obicompactvec/src/bitmatrix/persistent.rs +++ b/src/obicompactvec/src/bitmatrix/persistent.rs @@ -251,7 +251,7 @@ impl PersistentBitMatrix { Box::new(slots.iter().enumerate().map(|(i, _)| (i, 0usize, 1u32))) } Self::Columnar(_) | Self::Packed(_) => { - Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c))) + Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false)) } } } diff --git a/src/obicompactvec/src/bitmatrix/sparse.rs b/src/obicompactvec/src/bitmatrix/sparse.rs index 51df6a84..c1f1b806 100644 --- a/src/obicompactvec/src/bitmatrix/sparse.rs +++ b/src/obicompactvec/src/bitmatrix/sparse.rs @@ -37,6 +37,7 @@ use super::PersistentBitMatrix; use super::col_path; use super::columnar::ColumnarBitMatrix; use super::packed::PackedBitMatrix; +use crate::views::nonzero_triples; fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") } fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") } @@ -323,6 +324,9 @@ impl crate::traits::BinaryMatrix for PersistentSparseBitMatrix { // ── PersistentSparseBitMatrixBuilder ──────────────────────────────────────── +/// Row batch size for [`PersistentSparseBitMatrixBuilder::build_from_dense`]. +const BUILD_FROM_DENSE_BATCH: usize = 4096; + pub struct PersistentSparseBitMatrixBuilder { dir: PathBuf, n_cols: usize, @@ -431,19 +435,33 @@ impl PersistentSparseBitMatrixBuilder { } /// Builds a sparse matrix from an already-built dense - /// [`PersistentBitMatrix`] — row-by-row transpose via - /// [`PersistentBitMatrix::fill_row`], for migrating an existing index. + /// [`PersistentBitMatrix`] (`Columnar` or `Packed`), for migrating an + /// existing index. + /// + /// Processes rows in batches of [`BUILD_FROM_DENSE_BATCH`] instead of + /// one row at a time: `nonzero_triples(..., sorted_slots=true)` walks + /// each column *once per batch*, sequentially — matching how + /// `PersistentBitMatrix::nonzero_iter` already batches slots for + /// queries — rather than `fill_row`'s row-major loop, which re-reads + /// every column with one `get()` per cell and jumps between all + /// `n_cols` column files on every single row. pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result { let n = dense.n(); let n_cols = dense.n_cols(); let mut builder = Self::new(n, n_cols, dir)?; - let mut buf = vec![0u32; n_cols]; - let mut genomes: Vec = Vec::new(); - for slot in 0..n { - dense.fill_row(slot, &mut buf); - genomes.clear(); - genomes.extend((0..n_cols).filter(|&c| buf[c] != 0).map(|c| c as u32)); - builder.push_row(&genomes); + let mut row_buffers: Vec> = vec![Vec::new(); BUILD_FROM_DENSE_BATCH]; + let mut start = 0; + while start < n { + let end = (start + BUILD_FROM_DENSE_BATCH).min(n); + let slots: Vec = (start..end).collect(); + for (row, c, _) in nonzero_triples(&slots, n_cols, |c| dense.col_view(c), true) { + row_buffers[row].push(c as u32); + } + for buf in row_buffers.iter_mut().take(end - start) { + builder.push_row(buf); + buf.clear(); + } + start = end; } Ok(builder) } diff --git a/src/obicompactvec/src/intmatrix.rs b/src/obicompactvec/src/intmatrix.rs index fe0f822c..438b328f 100644 --- a/src/obicompactvec/src/intmatrix.rs +++ b/src/obicompactvec/src/intmatrix.rs @@ -401,7 +401,7 @@ impl PersistentCompactIntMatrix { /// both variants reuse the shared sorted-slot batching in /// `views::nonzero_triples`. pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator + 'a { - nonzero_triples(slots, self.n_cols(), |c| self.col_view(c)) + nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false) } #[inline] diff --git a/src/obicompactvec/src/views.rs b/src/obicompactvec/src/views.rs index 93113c4e..c0672b65 100644 --- a/src/obicompactvec/src/views.rs +++ b/src/obicompactvec/src/views.rs @@ -469,10 +469,13 @@ pub(crate) fn nonzero_triples( slots: &[usize], n_cols: usize, col_view: impl Fn(usize) -> V, + sorted_slots: bool, ) -> impl Iterator { let n = slots.len(); let mut perm: Vec = (0..n).collect(); - perm.sort_by_key(|&i| slots[i]); + if !sorted_slots { + perm.sort_by_key(|&i| slots[i]); + } let sorted_slots: Vec = perm.iter().map(|&i| slots[i]).collect(); let mut hits: Vec<(usize, usize, u32)> = Vec::new(); for c in 0..n_cols {