Add sorted_slots parameter to nonzero_triples and batch build_from_dense

Extended `nonzero_triples` with a boolean flag to conditionally skip sorting when input slots are already ordered. Updated call sites to pass appropriate flags, enabling a single-pass columnar traversal in the sparse matrix builder. Refactored `build_from_dense` into a batched processing pipeline that reduces repeated file access and improves sequential read performance. Added a diagnostic example to validate the new builder against existing dense matrices.
This commit is contained in:
Eric Coissac committed 2026-08-22 17:33:07 +02:00
1 parent 23812d1af8
commit bb380d0c7d
5 files changed
+102 -12

No files matched your search

@@ -0,0 +1,69 @@
//! Diagnostic: build a `PersistentSparseBitMatrix` from a real dense
//! `PersistentBitMatrix` (batched `build_from_dense`) and verify the result
//! is bit-for-bit identical, on an actual large presence matrix.
//!
//! Usage: cargo run --release --example compare_sparse_dense -p obicompactvec -- <layer_dir>
//! (layer_dir is the directory containing `presence/matrix.pbmx`, e.g.
//! `benchmark/tmp/bacteria/partitions/part_00000/index/layer_1`)
use std::error::Error;
use std::path::Path;
use std::time::Instant;
use obicompactvec::{PersistentBitMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
fn main() -> Result<(), Box<dyn Error>> {
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense <layer_dir>");
let layer_dir = Path::new(&layer_dir);
let t0 = Instant::now();
let dense = PersistentBitMatrix::open(layer_dir)?;
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
let out_dir = tempfile::tempdir()?;
let t0 = Instant::now();
let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
println!("build_from_dense: {:?}", t0.elapsed());
compare(&dense, &sparse)
}
fn compare(dense: &PersistentBitMatrix, sparse: &PersistentSparseBitMatrix) -> Result<(), Box<dyn Error>> {
let n = dense.n();
let n_cols = dense.n_cols();
assert_eq!(n, sparse.n(), "n mismatch");
assert_eq!(n_cols, sparse.n_cols(), "n_cols mismatch");
let mut dense_row = vec![0u32; n_cols];
let mut sparse_row = vec![0u32; n_cols];
let mut total_cells = 0usize;
let mut mismatched_cells = 0usize;
let mut first_mismatch = None;
let t0 = Instant::now();
for slot in 0..n {
dense.fill_row(slot, &mut dense_row);
sparse.fill_row(slot, &mut sparse_row);
for c in 0..n_cols {
total_cells += 1;
if dense_row[c] != sparse_row[c] {
mismatched_cells += 1;
if first_mismatch.is_none() {
first_mismatch = Some((slot, c, dense_row[c], sparse_row[c]));
}
}
}
}
println!("comparaison bit-a-bit: {:?} ({total_cells} cellules)", t0.elapsed());
if let Some((slot, col, d, s)) = first_mismatch {
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
println!("MISMATCHES: {mismatched_cells} cellules differentes sur {total_cells}");
} else {
println!("OK: toutes les {total_cells} cellules sont identiques entre dense et sparse.");
}
Ok(())
}