Add sorted_slots parameter to nonzero_triples and batch build_from_dense
Extended `nonzero_triples` with a boolean flag to conditionally skip sorting when input slots are already ordered. Updated call sites to pass appropriate flags, enabling a single-pass columnar traversal in the sparse matrix builder. Refactored `build_from_dense` into a batched processing pipeline that reduces repeated file access and improves sequential read performance. Added a diagnostic example to validate the new builder against existing dense matrices.
This commit is contained in:
1 parent
23812d1af8
commit
bb380d0c7d
5 files changed
+102
-12
No files matched your search
@@ -0,0 +1,69 @@
|
||||
//! Diagnostic: build a `PersistentSparseBitMatrix` from a real dense
|
||||
//! `PersistentBitMatrix` (batched `build_from_dense`) and verify the result
|
||||
//! is bit-for-bit identical, on an actual large presence matrix.
|
||||
//!
|
||||
//! Usage: cargo run --release --example compare_sparse_dense -p obicompactvec -- <layer_dir>
|
||||
//! (layer_dir is the directory containing `presence/matrix.pbmx`, e.g.
|
||||
//! `benchmark/tmp/bacteria/partitions/part_00000/index/layer_1`)
|
||||
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense <layer_dir>");
|
||||
let layer_dir = Path::new(&layer_dir);
|
||||
|
||||
let t0 = Instant::now();
|
||||
let dense = PersistentBitMatrix::open(layer_dir)?;
|
||||
println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());
|
||||
|
||||
let out_dir = tempfile::tempdir()?;
|
||||
|
||||
let t0 = Instant::now();
|
||||
let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
|
||||
println!("build_from_dense: {:?}", t0.elapsed());
|
||||
|
||||
compare(&dense, &sparse)
|
||||
}
|
||||
|
||||
fn compare(dense: &PersistentBitMatrix, sparse: &PersistentSparseBitMatrix) -> Result<(), Box<dyn Error>> {
|
||||
let n = dense.n();
|
||||
let n_cols = dense.n_cols();
|
||||
assert_eq!(n, sparse.n(), "n mismatch");
|
||||
assert_eq!(n_cols, sparse.n_cols(), "n_cols mismatch");
|
||||
|
||||
let mut dense_row = vec![0u32; n_cols];
|
||||
let mut sparse_row = vec![0u32; n_cols];
|
||||
let mut total_cells = 0usize;
|
||||
let mut mismatched_cells = 0usize;
|
||||
let mut first_mismatch = None;
|
||||
|
||||
let t0 = Instant::now();
|
||||
for slot in 0..n {
|
||||
dense.fill_row(slot, &mut dense_row);
|
||||
sparse.fill_row(slot, &mut sparse_row);
|
||||
|
||||
for c in 0..n_cols {
|
||||
total_cells += 1;
|
||||
if dense_row[c] != sparse_row[c] {
|
||||
mismatched_cells += 1;
|
||||
if first_mismatch.is_none() {
|
||||
first_mismatch = Some((slot, c, dense_row[c], sparse_row[c]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("comparaison bit-a-bit: {:?} ({total_cells} cellules)", t0.elapsed());
|
||||
|
||||
if let Some((slot, col, d, s)) = first_mismatch {
|
||||
eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
|
||||
println!("MISMATCHES: {mismatched_cells} cellules differentes sur {total_cells}");
|
||||
} else {
|
||||
println!("OK: toutes les {total_cells} cellules sont identiques entre dense et sparse.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user