Introduce compare_sparse CLI tool to verify index consistency

Adds a new binary that validates bit-level consistency between dense and sparse K-mer index representations by sampling slots across partitions and reporting discrepancies. Refactors sibling cache matrix initialization into a centralized factory method to simplify control flow and standardize error handling. Introduces diagnostic configurations, benchmark scripts, and an ignored test case to support k-mer resolution analysis.

src/bin/compare_sparse.rs

@@ -0,0 +1,130 @@
+use std::path::PathBuf;
+
+use obicompactvec::{BinaryMatrix, PersistentBitMatrix, PersistentSparseBitMatrix};
+use obikindex::KmerIndex;
+
+fn main() -> anyhow::Result<()> {
+    let orig_root = std::env::args().nth(1).expect("usage: compare_sparse <orig_dense> <sparse>");
+    let sparse_root = std::env::args().nth(2).expect("usage: compare_sparse <orig_dense> <sparse>");
+
+    let orig = KmerIndex::open(&orig_root)?;
+    let sparse = KmerIndex::open(&sparse_root)?;
+
+    let n_parts = orig.n_partitions();
+    let n_layers = orig.n_layers_per_partition()?;
+    let n_genomes = orig.meta().genomes.len();
+
+    println!("index orig:  {} partitions, {} layers, {} genomes", n_parts, n_layers, n_genomes);
+    println!("index sparse:{} partitions, {} layers, {} genomes", sparse.n_partitions(), sparse.n_layers_per_partition()?, sparse.meta().genomes.len());
+
+    let mut total_layers = 0usize;
+    let mut mismatches = 0usize;
+    let mut slot_checked = 0usize;
+
+    for part in 0..n_parts {
+        let part_dir_orig = orig.partition().part_dir(part);
+        let part_dir_sparse = sparse.partition().part_dir(part);
+
+        if !part_dir_orig.join("index").exists() || !part_dir_sparse.join("index").exists() {
+            continue;
+        }
+
+        for layer in 0..n_layers {
+            let layer_dir_orig = part_dir_orig.join("index").join(format!("layer_{layer}"));
+            let layer_dir_sparse = part_dir_sparse.join("index").join(format!("layer_{layer}"));
+
+            if !layer_dir_orig.exists() || !layer_dir_sparse.exists() {
+                continue;
+            }
+
+            let presence_orig = layer_dir_orig.join("presence");
+            let presence_sparse = layer_dir_sparse.join("presence");
+
+            if !presence_orig.exists() || !presence_sparse.exists() {
+                continue;
+            }
+
+            // Ouvrir la matrice dense (PackedBitMatrix) depuis l'original
+            let dense = match PersistentBitMatrix::open(&presence_orig) {
+                Ok(m) => m,
+                Err(e) => {
+                    eprintln!("ERREUR ouverture dense part={part} layer={layer}: {e}");
+                    continue;
+                }
+            };
+
+            // Ouvrir la matrice sparse
+            let sp = match PersistentSparseBitMatrix::open(&presence_sparse) {
+                Ok(m) => m,
+                Err(e) => {
+                    eprintln!("ERREUR ouverture sparse part={part} layer={layer}: {e}");
+                    continue;
+                }
+            };
+
+            if dense.n_cols() != sp.n_cols() {
+                eprintln!("ERREUR n_cols différent part={part} layer={layer}: dense={} sparse={}", dense.n_cols(), sp.n_cols());
+                continue;
+            }
+
+            let n = dense.n();
+            if n != sp.n() {
+                eprintln!("ERREUR n différent part={part} layer={layer}: dense={} sparse={}", n, sp.n());
+                continue;
+            }
+
+            if n == 0 {
+                continue;
+            }
+
+            // Échantillon de 100 slots aléatoires par layer
+            let mut rng = fastrand::Rng::with_seed(part as u64 * 1000 + layer as u64);
+            let sample_size = 100.min(n);
+            let mut checked_any = false;
+
+            for _ in 0..sample_size {
+                let slot = rng.usize(0..n);
+                let mut dense_row = vec![0u32; dense.n_cols()];
+                let mut sparse_row = vec![0u32; sp.n_cols()];
+
+                dense.fill_row(slot, &mut dense_row);
+                sp.fill_row(slot, &mut sparse_row);
+
+                if dense_row != sparse_row {
+                    // Premier mismatch pour ce layer : diagnostiquer la colonne A
+                    if !checked_any {
+                        let a_col = 0; // A est la colonne 0
+                        let mut a_dense = 0u32;
+                        let mut a_sparse = 0u32;
+                        // scanner tous les slots pour compter les différences sur la colonne A
+                        let mut diff_a = 0usize;
+                        for s in 0..n {
+                            let mut d = vec![0u32; dense.n_cols()];
+                            let mut s2 = vec![0u32; sp.n_cols()];
+                            dense.fill_row(s, &mut d);
+                            sp.fill_row(s, &mut s2);
+                            if d[a_col] != s2[a_col] {
+                                diff_a += 1;
+                            }
+                        }
+                        eprintln!("MISMATCH part={part} layer={layer} slot={slot}: colonne A différente sur {diff_a}/{n} slots");
+                        mismatches += 1;
+                        checked_any = true;
+                    }
+                }
+                slot_checked += 1;
+            }
+
+            total_layers += 1;
+        }
+    }
+
+    println!("Vérifié {} layers, {} slots échantillonnés", total_layers, slot_checked);
+    if mismatches == 0 {
+        println!("OK : aucune différence détectée.");
+    } else {
+        println!("MISMATCHES détectés dans {} layers", mismatches);
+    }
+
+    Ok(())
+}