Add sparse storage variant to PersistentCompactIntMatrix

Introduce a new `Sparse` format alongside existing `Columnar` and `Packed` variants, enabling optimized row-major pairwise counting for distance and similarity metrics via the `CountPartials` trait. Update storage detection priorities, extend matrix dispatch logic to sparse backends, and correct diagonal/off-diagonal formulas in bit matrix partial computations. Expand layer APIs with format-agnostic `nonzero_iter`, update usage documentation for the `--sparse` flag, and add comprehensive tests verifying roundtrip integrity and metric equivalence against dense implementations.

src/obikindex/src/layer/content_layer.rs

@@ -217,6 +217,18 @@ impl KmerLayer {
         }
     }
 
+    /// Every nonzero `(idx into slots, col, value)` triple among `slots` —
+    /// format-agnostic column-major fetch, delegating to whichever matrix
+    /// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
+    /// `PersistentCompactIntMatrix::nonzero_iter`).
+    pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
+        match self {
+            KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
+            KmerLayer::Count { layer, .. } => Box::new(layer.nonzero_iter(slots)),
+            KmerLayer::Empty { .. } => panic!("Layer::nonzero_iter() called on an Empty layer"),
+        }
+    }
+
     /// Per-genome column weights — count sum or presence k-mer count,
     /// depending on content (see `obicompactvec::ColumnWeights`).
     pub fn col_weights(&self) -> ndarray::Array1<u64> {

src/obikindex/src/layer/tests/typed_layer.rs

@@ -98,6 +98,68 @@ fn presence_layer_generic_over_sparse_matches_dense() {
     }
 }
 
+// ── `PersistentCompactIntMatrix` transparently absorbing `Sparse` ────────────
+//
+// Unlike presence (`PersistentSparseBitMatrix` is a standalone type, plugged
+// into `TypedLayer` via its own `LayerData`/`HasLayerContent` impls, see
+// `presence_layer_generic_over_sparse_matches_dense` above), counts don't
+// get a second `TypedLayer<PersistentSparseCompactIntMatrix>` instantiation
+// — sparsity is absorbed inside the `PersistentCompactIntMatrix` enum
+// itself, so `TypedLayer<PersistentCompactIntMatrix>` (the only impl block
+// for counts) must keep working unchanged once the on-disk format is
+// repacked to `Sparse`. This is the regression test for the bug that
+// motivated adding the `Sparse` variant: before it existed, `open()` had no
+// path back to a layer packed with `pack_matrices(sparse=true)` — see
+// `obikindex::index::kmer_index::KmerIndex::pack_matrices`, which already
+// called `pack_sparse_compact_int_matrix` on `counts/` without anything
+// downstream able to read the result back.
+#[test]
+fn count_layer_transparently_reads_sparse_after_pack() {
+    set_k(4);
+    let dir = tempdir().unwrap();
+    write_unitigs(dir.path(), &[b"AAATCTA", b"CTTCGCC", b"TGATACG"]);
+    let n_genomes = 3;
+    let mode = IndexMode::Exact;
+
+    // Deterministic, arbitrary count function — same shape as the presence
+    // test's arbitrary predicate, just producing a small count instead of a
+    // bool.
+    TypedLayer::<()>::build_with_matrix(dir.path(), DEFAULT_BLOCK_BITS, &mode, false, n_genomes, |kmer| {
+        (0..n_genomes)
+            .map(|g| ((kmer.raw().wrapping_add(g as u64)) % 5) as u32)
+            .collect()
+    })
+    .unwrap();
+
+    let dense_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
+    assert_eq!(dense_layer.storage_kind(), obicompactvec::StorageKind::Columnar);
+    let n = dense_layer.n();
+    let slots: Vec<usize> = (0..n).collect();
+    let dense_sub = dense_layer.sub_matrix(&slots);
+    let dense_weights = dense_layer.col_weights();
+    let dense_slots: Vec<Option<usize>> = all_canonical_kmers(dir.path())
+        .iter()
+        .map(|&kmer| dense_layer.find_slot(kmer))
+        .collect();
+
+    // Repack in place — same directory, matching how `pack --sparse` works
+    // in production.
+    obicompactvec::pack_sparse_compact_int_matrix(&dir.path().join(COUNTS_DIR)).unwrap();
+
+    let sparse_layer = TypedLayer::<PersistentCompactIntMatrix>::open(dir.path()).unwrap();
+    assert_eq!(sparse_layer.storage_kind(), obicompactvec::StorageKind::Sparse);
+    assert_eq!(sparse_layer.n(), n);
+    assert_eq!(sparse_layer.n_cols(), dense_layer.n_cols());
+    assert_eq!(sparse_layer.sub_matrix(&slots), dense_sub);
+    assert_eq!(sparse_layer.col_weights(), dense_weights);
+
+    let sparse_slots: Vec<Option<usize>> = all_canonical_kmers(dir.path())
+        .iter()
+        .map(|&kmer| sparse_layer.find_slot(kmer))
+        .collect();
+    assert_eq!(sparse_slots, dense_slots);
+}
+
 // ── content / storage_kind / evidence_kind (runtime introspection) ─────────
 
 #[test]

src/obikindex/src/layer/typed_layer.rs

@@ -415,6 +415,12 @@ impl TypedLayer<PersistentCompactIntMatrix> {
     pub fn col_weights(&self) -> ndarray::Array1<u64> {
         obicompactvec::ColumnWeights::col_weights(&self.data)
     }
+
+    /// Every nonzero `(idx into slots, col, value)` triple among `slots` —
+    /// delegates to `PersistentCompactIntMatrix::nonzero_iter`.
+    pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a {
+        self.data.nonzero_iter(slots)
+    }
 }
 
 // ── Mode 3 — presence/absence matrix ─────────────────────────────────────────
@@ -469,6 +475,12 @@ impl<D: LayerData<Item = Box<[bool]>> + BinaryMatrix + obicompactvec::ColumnWeig
 }
 
 impl TypedLayer<PersistentBitMatrix> {
+    /// Every nonzero `(idx into slots, col, value)` triple among `slots` —
+    /// delegates to `PersistentBitMatrix::nonzero_iter`.
+    pub fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
+        self.data.nonzero_iter(slots)
+    }
+
     pub fn append_genome_column(
         layer_dir: &Path,
         value_of: impl Fn(usize) -> bool,