Add sorted_slots parameter to nonzero_triples and batch build_from_dense

Extended `nonzero_triples` with a boolean flag to conditionally skip sorting when input slots are already ordered. Updated call sites to pass appropriate flags, enabling a single-pass columnar traversal in the sparse matrix builder. Refactored `build_from_dense` into a batched processing pipeline that reduces repeated file access and improves sequential read performance. Added a diagnostic example to validate the new builder against existing dense matrices.

src/obicompactvec/examples/compare_sparse_dense.rs

@@ -0,0 +1,69 @@
+//! Diagnostic: build a `PersistentSparseBitMatrix` from a real dense
+//! `PersistentBitMatrix` (batched `build_from_dense`) and verify the result
+//! is bit-for-bit identical, on an actual large presence matrix.
+//!
+//! Usage: cargo run --release --example compare_sparse_dense -p obicompactvec -- <layer_dir>
+//! (layer_dir is the directory containing `presence/matrix.pbmx`, e.g.
+//! `benchmark/tmp/bacteria/partitions/part_00000/index/layer_1`)
+
+use std::error::Error;
+use std::path::Path;
+use std::time::Instant;
+
+use obicompactvec::{PersistentBitMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense <layer_dir>");
+    let layer_dir = Path::new(&layer_dir);
+
+    let t0 = Instant::now();
+    let dense = PersistentBitMatrix::open(layer_dir)?;
+    println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
+
+    let out_dir = tempfile::tempdir()?;
+
+    let t0 = Instant::now();
+    let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
+    println!("build_from_dense: {:?}", t0.elapsed());
+
+    compare(&dense, &sparse)
+}
+
+fn compare(dense: &PersistentBitMatrix, sparse: &PersistentSparseBitMatrix) -> Result<(), Box<dyn Error>> {
+    let n = dense.n();
+    let n_cols = dense.n_cols();
+    assert_eq!(n, sparse.n(), "n mismatch");
+    assert_eq!(n_cols, sparse.n_cols(), "n_cols mismatch");
+
+    let mut dense_row = vec![0u32; n_cols];
+    let mut sparse_row = vec![0u32; n_cols];
+    let mut total_cells = 0usize;
+    let mut mismatched_cells = 0usize;
+    let mut first_mismatch = None;
+
+    let t0 = Instant::now();
+    for slot in 0..n {
+        dense.fill_row(slot, &mut dense_row);
+        sparse.fill_row(slot, &mut sparse_row);
+
+        for c in 0..n_cols {
+            total_cells += 1;
+            if dense_row[c] != sparse_row[c] {
+                mismatched_cells += 1;
+                if first_mismatch.is_none() {
+                    first_mismatch = Some((slot, c, dense_row[c], sparse_row[c]));
+                }
+            }
+        }
+    }
+    println!("comparaison bit-a-bit: {:?} ({total_cells} cellules)", t0.elapsed());
+
+    if let Some((slot, col, d, s)) = first_mismatch {
+        eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
+        println!("MISMATCHES: {mismatched_cells} cellules differentes sur {total_cells}");
+    } else {
+        println!("OK: toutes les {total_cells} cellules sont identiques entre dense et sparse.");
+    }
+
+    Ok(())
+}

src/obicompactvec/src/bitmatrix/persistent.rs

@@ -251,7 +251,7 @@ impl PersistentBitMatrix {
                 Box::new(slots.iter().enumerate().map(|(i, _)| (i, 0usize, 1u32)))
             }
             Self::Columnar(_) | Self::Packed(_) => {
-                Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c)))
+                Box::new(nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false))
             }
         }
     }

src/obicompactvec/src/bitmatrix/sparse.rs

@@ -37,6 +37,7 @@ use super::PersistentBitMatrix;
 use super::col_path;
 use super::columnar::ColumnarBitMatrix;
 use super::packed::PackedBitMatrix;
+use crate::views::nonzero_triples;
 
 fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
 fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") }
@@ -323,6 +324,9 @@ impl crate::traits::BinaryMatrix for PersistentSparseBitMatrix {
 
 // ── PersistentSparseBitMatrixBuilder ────────────────────────────────────────
 
+/// Row batch size for [`PersistentSparseBitMatrixBuilder::build_from_dense`].
+const BUILD_FROM_DENSE_BATCH: usize = 4096;
+
 pub struct PersistentSparseBitMatrixBuilder {
     dir: PathBuf,
     n_cols: usize,
@@ -431,19 +435,33 @@ impl PersistentSparseBitMatrixBuilder {
     }
 
     /// Builds a sparse matrix from an already-built dense
-    /// [`PersistentBitMatrix`] — row-by-row transpose via
+    /// [`PersistentBitMatrix`] (`Columnar` or `Packed`), for migrating an
-    /// [`PersistentBitMatrix::fill_row`], for migrating an existing index.
+    /// existing index.
+    ///
+    /// Processes rows in batches of [`BUILD_FROM_DENSE_BATCH`] instead of
+    /// one row at a time: `nonzero_triples(..., sorted_slots=true)` walks
+    /// each column *once per batch*, sequentially — matching how
+    /// `PersistentBitMatrix::nonzero_iter` already batches slots for
+    /// queries — rather than `fill_row`'s row-major loop, which re-reads
+    /// every column with one `get()` per cell and jumps between all
+    /// `n_cols` column files on every single row.
     pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result<Self> {
         let n = dense.n();
         let n_cols = dense.n_cols();
         let mut builder = Self::new(n, n_cols, dir)?;
-        let mut buf = vec![0u32; n_cols];
+        let mut row_buffers: Vec<Vec<u32>> = vec![Vec::new(); BUILD_FROM_DENSE_BATCH];
-        let mut genomes: Vec<u32> = Vec::new();
+        let mut start = 0;
-        for slot in 0..n {
+        while start < n {
-            dense.fill_row(slot, &mut buf);
+            let end = (start + BUILD_FROM_DENSE_BATCH).min(n);
-            genomes.clear();
+            let slots: Vec<usize> = (start..end).collect();
-            genomes.extend((0..n_cols).filter(|&c| buf[c] != 0).map(|c| c as u32));
+            for (row, c, _) in nonzero_triples(&slots, n_cols, |c| dense.col_view(c), true) {
-            builder.push_row(&genomes);
+                row_buffers[row].push(c as u32);
+            }
+            for buf in row_buffers.iter_mut().take(end - start) {
+                builder.push_row(buf);
+                buf.clear();
+            }
+            start = end;
         }
         Ok(builder)
     }

src/obicompactvec/src/intmatrix.rs

@@ -401,7 +401,7 @@ impl PersistentCompactIntMatrix {
     /// both variants reuse the shared sorted-slot batching in
     /// `views::nonzero_triples`.
     pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
-        nonzero_triples(slots, self.n_cols(), |c| self.col_view(c))
+        nonzero_triples(slots, self.n_cols(), |c| self.col_view(c), false)
     }
 
     #[inline]

src/obicompactvec/src/views.rs

@@ -469,10 +469,13 @@ pub(crate) fn nonzero_triples<V: NonzeroSlotsView>(
     slots: &[usize],
     n_cols: usize,
     col_view: impl Fn(usize) -> V,
+    sorted_slots: bool,
 ) -> impl Iterator<Item = (usize, usize, u32)> {
     let n = slots.len();
     let mut perm: Vec<usize> = (0..n).collect();
-    perm.sort_by_key(|&i| slots[i]);
+    if !sorted_slots {
+        perm.sort_by_key(|&i| slots[i]);
+    }
     let sorted_slots: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
     let mut hits: Vec<(usize, usize, u32)> = Vec::new();
     for c in 0..n_cols {