Introduce compare_sparse CLI tool to verify index consistency

Adds a new binary that validates bit-level consistency between dense and sparse K-mer index representations by sampling slots across partitions and reporting discrepancies. Refactors sibling cache matrix initialization into a centralized factory method to simplify control flow and standardize error handling. Introduces diagnostic configurations, benchmark scripts, and an ignored test case to support k-mer resolution analysis.

src/obikphylo/src/siblings/cache.rs

@@ -1,10 +1,12 @@
 use rayon::prelude::*;
 
+use std::path::Path;
+
 use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, PersistentSparseBitMatrix};
 use obikpartitionner::KmerPartition;
 use obikseq::CanonicalKmer;
-use obilayeredmap::Layer;
-use obilayeredmap::meta::PartitionMeta;
+use obilayeredmap::{Layer, OLMResult};
+use obilayeredmap::meta::{IndexMode, PartitionMeta};
 use obisys::progress_bar;
 
 use obikindex::OKIResult;
@@ -50,6 +52,31 @@ pub(super) enum Mat {
 }
 
 impl Mat {
+    /// Open one layer's matrix, auto-detecting count vs. dense-presence vs.
+    /// sparse-presence from what is actually on disk — the single source of
+    /// truth for *every* caller that opens a layer's own matrix (this
+    /// module's cross-partition [`PartitionCache::build`] and
+    /// `family_scan::scan_layer_families`'s own-layer lookup alike), so the
+    /// two can never disagree about which format a layer's `presence/` was
+    /// packed to. Before this existed, `scan_layer_families` open-coded its
+    /// own (non-sparse-aware) copy of this decision: on a `pack --sparse`d
+    /// layer, `presence/matrix.pbmx` no longer exists (removed by
+    /// `pack_sparse_bit_matrix`), so a caller that only checks for it and
+    /// otherwise unconditionally opens `PersistentBitMatrix` doesn't error —
+    /// `PersistentBitMatrix::open`'s own auto-detection falls all the way
+    /// through to the `Implicit` (mono-genome, all-present) case, silently
+    /// corrupting every read.
+    pub(super) fn open(layer_dir: &Path, mode: &IndexMode, with_counts: bool) -> OLMResult<Self> {
+        if with_counts && layer_dir.join("counts").exists() {
+            return Layer::<PersistentCompactIntMatrix>::open(layer_dir, mode).map(Mat::Count);
+        }
+        if layer_dir.join("presence").join("is_multi.prsb").exists() {
+            Layer::<PersistentSparseBitMatrix>::open(layer_dir, mode).map(Mat::SparsePresence)
+        } else {
+            Layer::<PersistentBitMatrix>::open(layer_dir, mode).map(Mat::Presence)
+        }
+    }
+
     fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
         match self {
             Mat::Count(l) => l.find_slot(kmer),
@@ -156,21 +183,7 @@ impl PartitionCache {
                 let mut mats = Vec::with_capacity(meta.n_layers);
                 for l in 0..meta.n_layers {
                     let layer_dir = index_dir.join(format!("layer_{l}"));
-                    let use_counts = with_counts && layer_dir.join("counts").exists();
-                    // `pack --sparse` converts a layer's `presence/` directory
-                    // in place, leaving `is_multi.prsb` as the on-disk marker
-                    // that distinguishes the sparse format
-                    // (`obicompactvec::PersistentSparseBitMatrix`) from the
-                    // dense one — see `bitmatrix/sparse.rs`'s module docs.
-                    let is_sparse = layer_dir.join("presence").join("is_multi.prsb").exists();
-                    let mat = if use_counts {
-                        Layer::<PersistentCompactIntMatrix>::open(&layer_dir, &meta.mode).ok().map(Mat::Count)
-                    } else if is_sparse {
-                        Layer::<PersistentSparseBitMatrix>::open(&layer_dir, &meta.mode).ok().map(Mat::SparsePresence)
-                    } else {
-                        Layer::<PersistentBitMatrix>::open(&layer_dir, &meta.mode).ok().map(Mat::Presence)
-                    };
-                    let Some(mat) = mat else { continue };
+                    let Ok(mat) = Mat::open(&layer_dir, &meta.mode, with_counts) else { continue };
                     mats.push(mat);
                 }
                 pb.inc(1);

src/obikphylo/src/siblings/family_scan.rs

@@ -53,9 +53,7 @@ use std::sync::atomic::{AtomicU8, Ordering};
 
 use rayon::prelude::*;
 
-use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
 use obikseq::CanonicalKmer;
-use obilayeredmap::Layer;
 use obilayeredmap::meta::PartitionMeta;
 use obipipeline::{ThrottleGuard, throttle};
 
@@ -195,12 +193,7 @@ pub(super) fn scan_layer_families(
     let meta = PartitionMeta::load(index_dir).map_err(olm_to_ok)?;
     let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
 
-    let use_counts = with_counts && layer_dir.join("counts").exists();
-    let mat = if use_counts {
-        Mat::Count(Layer::<PersistentCompactIntMatrix>::open(layer_dir, &meta.mode).map_err(olm_to_ok)?)
-    } else {
-        Mat::Presence(Layer::<PersistentBitMatrix>::open(layer_dir, &meta.mode).map_err(olm_to_ok)?)
-    };
+    let mat = Mat::open(layer_dir, &meta.mode, with_counts).map_err(olm_to_ok)?;
     let n_cols = mat.n_cols().min(n_genomes);
 
     let ctx = Arc::new(LayerCtx { mat, n_parts, n_genomes, n_cols, k });

src/obikphylo/src/siblings/tests.rs

@@ -1089,3 +1089,96 @@ fn bench_real_persistent_sparse_bit_matrix_on_disk_size() {
 
     let _ = std::fs::remove_dir_all(&out_dir);
 }
+
+#[test]
+#[ignore]
+fn diag_real_index_verify_kmer_resolution() {
+    use std::sync::Arc;
+    let idx = KmerIndex::open("/Users/coissac/Sync/travail/__MOI__/obikmer/benchmark/global_index_presence")
+        .expect("open real index");
+    let n_parts = idx.n_partitions();
+    let n_genomes = idx.meta().genomes.len();
+    let with_counts = idx.meta().config.with_counts;
+    let k = idx.kmer_size();
+    let n_bits = n_parts.trailing_zeros() as usize;
+    let partition = obikpartitionner::KmerPartition::open_with_config(
+        idx.root_path(), idx.kmer_size(), idx.minimizer_size(), n_bits,
+    ).unwrap();
+    let cache = Arc::new(super::cache::PartitionCache::build(&partition, n_parts, with_counts).unwrap());
+    
+    // The k-mer with A+other (mask 0b0011 = A+C)
+    let kmer_str = "AGCTAGCTATTGAGCCTGGTCCGTATGAAAC";
+    let kmer = CanonicalKmer::from_str(kmer_str, k).unwrap();
+    let rc = kmer.revcomp();
+    let rc_str = kmer_to_string(&rc, k);
+    
+    println!("kmer_forward={} kmer_revcomp={}", kmer_str, rc_str);
+    println!("kmer partition={}", kmer.partition(n_parts));
+    
+    // Check if this k-mer is found in its own partition
+    let dest_part = kmer.partition(n_parts);
+    if let Some(layer) = cache.find(dest_part, kmer) {
+        println!("FOUND in partition {} layer {}", dest_part, layer);
+    } else {
+        println!("NOT FOUND in partition {}", dest_part);
+    }
+    
+    // Now check the variant with A at central position
+    // The mask is 0b0011 = A+C, so there should be a variant with A
+    // Let's generate the canonical neighbors and check which one has A
+    let mut found_variants = Vec::new();
+    for variant in kmer.central_canonical_neighbors() {
+        if variant == kmer {
+            continue;
+        }
+        let base = central_base(variant, k);
+        if base == 0 {  // A
+            let v_part = variant.partition(n_parts);
+            println!("variant with A: partition={} found={}", v_part, cache.find(v_part, variant).is_some());
+            found_variants.push(variant);
+        }
+    }
+    
+    // Also check the C variant
+    for variant in kmer.central_canonical_neighbors() {
+        if variant == kmer {
+            continue;
+        }
+        let base = central_base(variant, k);
+        if base == 1 {  // C
+            let v_part = variant.partition(n_parts);
+            println!("variant with C: partition={} found={}", v_part, cache.find(v_part, variant).is_some());
+        }
+    }
+}
+
+fn kmer_to_string(kmer: &CanonicalKmer, k: usize) -> String {
+    let mut s = String::with_capacity(k);
+    for i in 0..k {
+        let n = kmer.nucleotide(i);
+        s.push(match n {
+            0 => 'A',
+            1 => 'C',
+            2 => 'G',
+            3 => 'T',
+            _ => unreachable!(),
+        });
+    }
+    s
+}
+
+fn kmer_revcomp_to_string(kmer: &CanonicalKmer, k: usize) -> String {
+    let rc = kmer.revcomp();
+    let mut s = String::with_capacity(k);
+    for i in 0..k {
+        let n = rc.nucleotide(i);
+        s.push(match n {
+            0 => 'A',
+            1 => 'C',
+            2 => 'G',
+            3 => 'T',
+            _ => unreachable!(),
+        });
+    }
+    s
+}