Files
obikmer/src/obikindex/src/layer/content_layer.rs
T
Eric Coissac 36c7247acb feat: add k-mer sharing estimation and Poisson-binomial null model
Add a new `estimate --index` command to compare observed k-mer sharing against
a random-sequence model using a Poisson-binomial distribution. Introduce the
`row_presence_histogram` API to support this analysis across various matrix
types. Update documentation and dependencies accordingly.

- Add `src/obikstats/src/null_model.rs` implementing `SharingNullModel` and
  `SharingRow` for computing cumulative probabilities and false discovery rates
  based on a Poisson-binomial distribution.
- Add `src/obikstats/src/sharing.rs` defining `GenomeSharing` and `SharingSpectrum`
  to compute k-mer sharing spectra using parallel map-reduce.
- Add `src/obikmer/src/cmd/estimate/sharing.rs` with `run` and `write_csv` to
  execute sharing estimation and output CSV results containing observed, expected,
  and deviation metrics.
- Modify `src/obikmer/src/cmd/estimate/mod.rs` to support the new `--index`
  subcommand, adding `EstimateArgs` fields for index path, presence threshold,
  and output file.
- Add `src/obicompactvec/src/row_histogram.rs` implementing `row_presence_histogram`
  for `PersistentIntMatrix`, `PersistentBitMatrix`, and sparse matrices to count
  rows present in exactly $j$ columns.
- Update `src/obikindex/src/layer/content_layer.rs` and `src/obikindex/src/layer/typed_layer.rs`
  to expose the new `row_presence_histogram` method, dispatching based on layer type.
- Add tests in `src/obicompactvec/src/tests/bitmatrix.rs`, `intmatrix.rs`, and
  `sparse.rs` to verify `count_geq` and `row_presence_histogram` against naive
  implementations for various matrix structures.
- Update `UserDocMD/theory/kmer_indexing/kmers.md` to document the Poisson-binomial
  approach, $D_r$ bounds, and FDR calculations, replacing previous binomial approximations.
- Update `UserDocMD/usage/estimate.md` to describe the two uses of `estimate`:
  parameter estimation and index comparison via `--index`.
- Add `wiki` target to `Makefile` for regenerating documentation from user docs.
- Bump `obikmer` package version from 1.4.5 to 1.4.6 in `Cargo.toml` and `Cargo.lock`.
2026-11-19 17:50:29 +01:00

458 lines
21 KiB
Rust

//! [`Layer`] — the format-erased layer handle every caller outside this
//! crate should reach for. `TypedLayer<D>` (`layer.rs`) is monomorphic: a
//! `Vec<TypedLayer<D>>` needs one `D` fixed at compile time for every
//! element, which real partitions don't respect (`with_counts` can differ
//! layer to layer across merges; see
//! `DevDocMD/implementation/partition_layer_cache.md`). `Layer` is the
//! sum type that lets a partition's layers be held together regardless —
//! `Count`/`Presence` each wrap the one concrete `TypedLayer<D>` that
//! content implies, chosen by [`Layer::open`]'s own disk probe.
//!
//! This is exactly the shape `obikphylo::siblings::cache::Mat` used to
//! reimplement locally, minus its one sibling-specific method
//! (`iter_minorants_batch`, which stays an extension trait over there —
//! `obikindex::layer` has no business knowing about sibling annexes).
//!
//! `Layer` is meant to eventually represent a layer's *whole* life —
//! empty shell, under construction, ready to read — not just the
//! ready-to-read state, so the same type follows a layer from creation
//! through querying instead of construction living as disconnected free
//! functions elsewhere that happen to write into the same directory (see
//! `DevDocMD/implementation/partition_layer_cache.md`). [`Layer::Empty`]
//! is the first step: a directory and nothing else, able to hand out the
//! paths a builder needs, not yet able to build anything itself.
use std::path::{Path, PathBuf};
use crate::layer::utils::LAYERNAME_SUFFIX;
use ndarray::{Array1, Array2};
use obicompactvec::{BitPartials, ColumnWeights, CountPartials, PersistentBitMatrix, PersistentIntMatrix};
use obikseq::CanonicalKmer;
use crate::index::error::OKIResult;
use crate::layer::mphf_layer::{
EVIDENCE_FILE, EvidenceKind, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE,
};
use crate::layer::typed_layer::{COUNTS_DIR, LayerContent, PRESENCE_DIR, TypedLayer};
use crate::partition::KmerPartition;
/// Superkmer-file extension used by every pre-layer-construction artifact
/// below (`raw`/`dereplicated`) — an implementation detail of the SK file
/// format, never meant to leak as a literal string past this module.
const SK_EXT: &str = "skmer.zst";
/// One layer, at any point in its life — see the module docs. Only
/// [`Empty`](Layer::Empty) and the two ready-to-read states
/// (`Count`/`Presence`) exist so far; states in between (unitigs written,
/// MPHF built, evidence built, matrix not yet built) are not represented
/// yet.
///
/// Every variant carries `dir` + `id` — a layer's identity never depends on
/// its state. `id` is its own layer number within the partition (not yet
/// consumed by anything — `KmerPartition` doesn't hand it in yet, that
/// lands with `KmerLayer::at`). `dir`/`id` live here, on `KmerLayer` itself,
/// not inside `TypedLayer`/`MphfLayer`: those only ever need `dir`
/// transiently, while opening — `KmerLayer` is what has to answer `.dir()`
/// after the fact, so it's the one that keeps it.
pub enum KmerLayer {
/// Just a directory — nothing built yet. Every method below other than
/// the path accessors panics on this variant: calling them means the
/// caller assumed a layer was ready when it wasn't, an implementation
/// error to surface loudly, not paper over with a default value.
Empty { dir: PathBuf, id: usize },
Count {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentIntMatrix>,
},
Presence {
dir: PathBuf,
id: usize,
layer: TypedLayer<PersistentBitMatrix>,
},
}
impl KmerLayer {
pub fn new(partition: &KmerPartition, i: usize) -> Self {
let dir = partition.dir().join(format!("{LAYERNAME_SUFFIX}{i}"));
KmerLayer::Empty { dir, id: i }
}
/// An empty shell at `dir` — creates the directory (if it doesn't
/// already exist) but nothing inside it. The starting state for
/// building a new layer.
pub fn create(self) -> std::io::Result<Self> {
std::fs::create_dir_all(self.dir())?;
Ok(self)
}
/// Open this layer, auto-detecting count vs. presence from what is
/// actually on disk (`counts/` present, else presence) — the single
/// source of truth every caller that opens a layer's own matrix should
/// share, so two callers can never disagree about which format a given
/// layer is. Dense vs. sparse presence storage is `PersistentBitMatrix::
/// open`'s own concern (a 4-way `Columnar`/`Packed`/`Sparse`/`Implicit`
/// enum internally), not decided here. A no-op (returns `self`
/// unchanged) if already `Count`/`Presence`.
pub fn open(self) -> OKIResult<Self> {
let (dir, id) = match self {
KmerLayer::Empty { dir, id } => (dir, id),
already_open => return Ok(already_open),
};
if dir.join(COUNTS_DIR).exists() {
let layer = TypedLayer::<PersistentIntMatrix>::open(&dir)?;
Ok(KmerLayer::Count { dir, id, layer })
} else {
let layer = TypedLayer::<PersistentBitMatrix>::open(&dir)?;
Ok(KmerLayer::Presence { dir, id, layer })
}
}
// ── Paths — valid in every state, `dir` never varies with content ────
//
// The `mphf.bin`/`unitigs.bin`/… naming stays defined once, here,
// rather than re-declared as string literals at every call site that
// writes into a layer directory (the same duplication `layer_dir`/
// `index_dir` fixed one level up — see `DevDocMD/implementation/
// partition_layer_cache.md`).
/// This layer's own directory.
pub fn dir(&self) -> &Path {
match self {
KmerLayer::Empty { dir, .. } => dir,
KmerLayer::Count { dir, .. } => dir,
KmerLayer::Presence { dir, .. } => dir,
}
}
pub fn mphf_path(&self) -> PathBuf {
self.dir().join(MPHF_FILE)
}
pub fn unitigs_path(&self) -> PathBuf {
self.dir().join(UNITIGS_FILE)
}
pub fn evidence_path(&self) -> PathBuf {
self.dir().join(EVIDENCE_FILE)
}
pub fn fingerprint_path(&self) -> PathBuf {
self.dir().join(FINGERPRINT_FILE)
}
pub fn counts_dir(&self) -> PathBuf {
self.dir().join(COUNTS_DIR)
}
pub fn presence_dir(&self) -> PathBuf {
self.dir().join(PRESENCE_DIR)
}
/// Path of this layer's raw, not-yet-dereplicated superkmer file —
/// written by whichever algorithm routes superkmers into this layer
/// (today: `obikindexer::algorithms::partitionner::PartitionRouter`),
/// read by whichever algorithm dereplicates it (today:
/// `obikindexer::algorithms::dereplicator::Dereplicator`). Naming this
/// once here, rather than in either algorithm submodule, is what lets
/// the two agree on the filename without depending on each other
/// directly — see `DevDocMD/implementation/partition_layer_cache.md`.
pub fn raw_superkmers_path(&self) -> PathBuf {
self.dir().join(format!("raw.{SK_EXT}"))
}
/// Path of this layer's dereplicated superkmer file — written by
/// `obikindexer::algorithms::dereplicator::Dereplicator`, read by
/// whichever algorithm counts kmer abundances from it (today:
/// `obikindexer::algorithms::partitionner::PartitionRouter::count_kmer`)
/// and, later, by `obikindex::build_index_layer` to build the real layer.
pub fn dereplicated_superkmers_path(&self) -> PathBuf {
self.dir().join(format!("dereplicated.{SK_EXT}"))
}
// ── Ready-only surface ───────────────────────────────────────────────
pub fn content(&self) -> LayerContent {
match self {
KmerLayer::Count { .. } => LayerContent::Count,
KmerLayer::Presence { .. } => LayerContent::Presence,
KmerLayer::Empty { .. } => panic!("Layer::content() called on an Empty layer"),
}
}
pub fn evidence_kind(&self) -> EvidenceKind {
match self {
KmerLayer::Count { layer, .. } => layer.evidence_kind(),
KmerLayer::Presence { layer, .. } => layer.evidence_kind(),
KmerLayer::Empty { .. } => panic!("Layer::evidence_kind() called on an Empty layer"),
}
}
pub fn n(&self) -> usize {
match self {
KmerLayer::Count { layer, .. } => layer.n(),
KmerLayer::Presence { layer, .. } => layer.n(),
KmerLayer::Empty { .. } => panic!("Layer::n() called on an Empty layer"),
}
}
pub fn find_slot(&self, kmer: CanonicalKmer) -> Option<usize> {
match self {
KmerLayer::Count { layer, .. } => layer.find_slot(kmer),
KmerLayer::Presence { layer, .. } => layer.find_slot(kmer),
KmerLayer::Empty { .. } => panic!("Layer::find_slot() called on an Empty layer"),
}
}
/// Batch form of [`find_slot`](Self::find_slot): `out[i]` is
/// `find_slot(kmers[i])`, MPHF lookups prefetched together. `out` is
/// cleared and reused.
pub fn find_slot_batch(&self, kmers: &[CanonicalKmer], out: &mut Vec<Option<usize>>) {
match self {
KmerLayer::Count { layer, .. } => layer.find_slot_batch(kmers, out),
KmerLayer::Presence { layer, .. } => layer.find_slot_batch(kmers, out),
KmerLayer::Empty { .. } => panic!("Layer::find_slot_batch() called on an Empty layer"),
}
}
/// Raw MPHF batch lookup: kmer → slot, no membership check — for
/// callers that already know every kmer is a member of *this* layer, so
/// the evidence check `find_slot`/`find` would perform is redundant.
pub fn hash_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize> {
match self {
KmerLayer::Count { layer, .. } => layer.hash_batch(kmers),
KmerLayer::Presence { layer, .. } => layer.hash_batch(kmers),
KmerLayer::Empty { .. } => panic!("Layer::hash_batch() called on an Empty layer"),
}
}
pub fn n_cols(&self) -> usize {
match self {
KmerLayer::Count { layer, .. } => layer.n_cols(),
KmerLayer::Presence { layer, .. } => layer.n_cols(),
KmerLayer::Empty { .. } => panic!("Layer::n_cols() called on an Empty layer"),
}
}
/// Every nonzero `(idx into slots, col, value)` triple among `slots` —
/// format-agnostic column-major fetch, delegating to whichever matrix
/// this layer actually holds (see `obicompactvec::PersistentBitMatrix`/
/// `PersistentCompactIntMatrix::nonzero_iter`).
pub fn nonzero_iter<'a>(
&'a self,
slots: &'a [usize],
) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> {
match self {
KmerLayer::Presence { layer, .. } => layer.nonzero_iter(slots),
KmerLayer::Count { layer, .. } => layer.nonzero_iter(slots),
KmerLayer::Empty { .. } => panic!("Layer::nonzero_iter() called on an Empty layer"),
}
}
/// Per-genome column weights — count sum or presence k-mer count,
/// depending on content (see `obicompactvec::ColumnWeights`).
pub fn col_weights(&self) -> ndarray::Array1<u64> {
match self {
KmerLayer::Count { layer, .. } => layer.col_weights(),
KmerLayer::Presence { layer, .. } => layer.col_weights(),
KmerLayer::Empty { .. } => panic!("Layer::col_weights() called on an Empty layer"),
}
}
/// Per-genome number of distinct k-mers present, a k-mer being present
/// when its count reaches `threshold` — presence layers hold no counts,
/// so `threshold` has no effect there (see
/// `obicompactvec::ColumnWeights::count_geq`).
pub fn count_geq(&self, threshold: u32) -> ndarray::Array1<u64> {
match self {
KmerLayer::Count { layer, .. } => layer.count_geq(threshold),
KmerLayer::Presence { layer, .. } => layer.count_geq(threshold),
KmerLayer::Empty { .. } => panic!("Layer::count_geq() called on an Empty layer"),
}
}
/// `hist[j]` = number of k-mers of this layer present in exactly `j`
/// genomes (length `n_cols() + 1`), a k-mer being present when its count
/// reaches `threshold` — presence layers hold no counts, so `threshold`
/// has no effect there. Additive across layers.
pub fn row_presence_histogram(&self, threshold: u32) -> OKIResult<Vec<u64>> {
match self {
KmerLayer::Count { layer, .. } => layer.row_presence_histogram(threshold),
KmerLayer::Presence { layer, .. } => layer.row_presence_histogram(),
KmerLayer::Empty { .. } => panic!("Layer::row_presence_histogram() called on an Empty layer"),
}
}
/// Batch, genome-major "carries" for a set of `slots` — `out[g][i]` =
/// whether genome `g` (0..`out.len()`) carries `slots[i]`. `out` must
/// have one entry per genome column, each resized to `slots.len()`.
/// `Count` needs one intermediate `Vec<Vec<u32>>` fetch (the underlying
/// store only has an int sub-matrix, not a bool one), converted to
/// presence (`!= 0`) in place.
pub fn fill_sub_matrix_carries(&self, slots: &[usize], out: &mut [Vec<bool>]) {
match self {
KmerLayer::Presence { layer, .. } => layer.fill_sub_matrix(slots, out),
KmerLayer::Count { layer, .. } => {
let mut counts: Vec<Vec<u32>> = out.iter().map(|_| Vec::new()).collect();
layer.fill_sub_matrix(slots, &mut counts);
for (o, c) in out.iter_mut().zip(counts.iter()) {
o.clear();
o.extend(c.iter().map(|&v| v != 0));
}
}
KmerLayer::Empty { .. } => {
panic!("Layer::fill_sub_matrix_carries() called on an Empty layer")
}
}
}
// content_layer.rs, à côté de fill_sub_matrix_carries
/// Batch, genome-major column extraction — `out[g][i]` = genome `g`'s raw
/// value at `slots[i]` (count as-is, presence promoted to 0/1). Column-first,
/// like the underlying storage: one read pass per genome column, not one per
/// kmer.
pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<u32>]) {
match self {
KmerLayer::Count { layer, .. } => layer.fill_sub_matrix(slots, out),
KmerLayer::Presence { layer, .. } => {
let mut bools: Vec<Vec<bool>> = out.iter().map(|_| Vec::new()).collect();
layer.fill_sub_matrix(slots, &mut bools);
for (o, b) in out.iter_mut().zip(bools.iter()) {
o.clear();
o.extend(b.iter().map(|&v| v as u32));
}
}
KmerLayer::Empty { .. } => panic!("Layer::fill_sub_matrix() called on an Empty layer"),
}
}
/// Iterate over all canonical kmers in the layer, in deterministic order.
pub fn iter_kmers(&self) -> crate::layer::mphf_layer::KmerIter {
match self {
KmerLayer::Count { layer, .. } => layer.iter_kmers(),
KmerLayer::Presence { layer, .. } => layer.iter_kmers(),
KmerLayer::Empty { .. } => panic!("Layer::iter_kmers() called on an Empty layer"),
}
}
/// Iterate over all canonical kmers, each paired with its zero-based
/// sequence index in `unitigs.bin`.
pub fn enumerate_kmers(&self) -> std::iter::Enumerate<crate::layer::mphf_layer::KmerIter> {
match self {
KmerLayer::Count { layer, .. } => layer.enumerate_kmers(),
KmerLayer::Presence { layer, .. } => layer.enumerate_kmers(),
KmerLayer::Empty { .. } => panic!("Layer::enumerate_kmers() called on an Empty layer"),
}
}
/// Iterate over the layer's canonical kmers in batches of `n`.
pub fn iter_kmers_batch(&self, n: usize) -> crate::layer::mphf_layer::KmerBatchIter {
match self {
KmerLayer::Count { layer, .. } => layer.iter_kmers_batch(n),
KmerLayer::Presence { layer, .. } => layer.iter_kmers_batch(n),
KmerLayer::Empty { .. } => panic!("Layer::iter_kmers_batch() called on an Empty layer"),
}
}
/// Iterate over batches, each paired with the zero-based index of the
/// first kmer in the batch.
pub fn enumerate_kmers_batch(
&self,
n: usize,
) -> Box<dyn Iterator<Item = (usize, Vec<CanonicalKmer>)> + Send + 'static> {
match self {
KmerLayer::Count { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)),
KmerLayer::Presence { layer, .. } => Box::new(layer.enumerate_kmers_batch(n)),
KmerLayer::Empty { .. } => {
panic!("Layer::enumerate_kmers_batch() called on an Empty layer")
}
}
}
/// Iterate over the layer's unitigs (whole sequences, not decomposed
/// into k-mers) — the "layer level" step of the layer/partition/index
/// chain of unitig iterators; see `KmerPartition::iter_unitigs`/
/// `KmerIndex::iter_unitigs`.
pub fn iter_unitigs(&self) -> crate::layer::mphf_layer::UnitigIter {
match self {
KmerLayer::Count { layer, .. } => layer.iter_unitigs(),
KmerLayer::Presence { layer, .. } => layer.iter_unitigs(),
KmerLayer::Empty { .. } => panic!("Layer::iter_unitigs() called on an Empty layer"),
}
}
}
// ── obicompactvec's statistics traits — read-only accessors, same spirit
// as `col_weights`/`nonzero_iter` above, just formalised as trait impls so
// generic aggregation code (`obikidxcache::IndexCache`'s callers) can rely
// on `KmerLayer: ColumnWeights`/`CountPartials`/`BitPartials` rather than
// duck-typing matching method names. `CountPartials`/`BitPartials` each
// only make sense for the content this layer actually holds — calling the
// wrong one panics, same convention as `col()`/`content()` above.
impl ColumnWeights for KmerLayer {
fn col_weights(&self) -> Array1<u64> {
self.col_weights()
}
fn count_geq(&self, threshold: u32) -> Array1<u64> {
self.count_geq(threshold)
}
}
impl CountPartials for KmerLayer {
fn partial_bray(&self) -> Array2<u64> {
match self {
KmerLayer::Count { layer, .. } => layer.partial_bray(),
KmerLayer::Presence { .. } => panic!("CountPartials::partial_bray() called on a Presence layer"),
KmerLayer::Empty { .. } => panic!("CountPartials::partial_bray() called on an Empty layer"),
}
}
fn partial_euclidean(&self) -> Array2<f64> {
match self {
KmerLayer::Count { layer, .. } => layer.partial_euclidean(),
KmerLayer::Presence { .. } => panic!("CountPartials::partial_euclidean() called on a Presence layer"),
KmerLayer::Empty { .. } => panic!("CountPartials::partial_euclidean() called on an Empty layer"),
}
}
fn partial_threshold_jaccard(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
match self {
KmerLayer::Count { layer, .. } => layer.partial_threshold_jaccard(threshold),
KmerLayer::Presence { .. } => panic!("CountPartials::partial_threshold_jaccard() called on a Presence layer"),
KmerLayer::Empty { .. } => panic!("CountPartials::partial_threshold_jaccard() called on an Empty layer"),
}
}
fn partial_relfreq_bray(&self, global: &Array1<u64>) -> Array2<f64> {
match self {
KmerLayer::Count { layer, .. } => layer.partial_relfreq_bray(global),
KmerLayer::Presence { .. } => panic!("CountPartials::partial_relfreq_bray() called on a Presence layer"),
KmerLayer::Empty { .. } => panic!("CountPartials::partial_relfreq_bray() called on an Empty layer"),
}
}
fn partial_relfreq_euclidean(&self, global: &Array1<u64>) -> Array2<f64> {
match self {
KmerLayer::Count { layer, .. } => layer.partial_relfreq_euclidean(global),
KmerLayer::Presence { .. } => panic!("CountPartials::partial_relfreq_euclidean() called on a Presence layer"),
KmerLayer::Empty { .. } => panic!("CountPartials::partial_relfreq_euclidean() called on an Empty layer"),
}
}
fn partial_hellinger(&self, global: &Array1<u64>) -> Array2<f64> {
match self {
KmerLayer::Count { layer, .. } => layer.partial_hellinger(global),
KmerLayer::Presence { .. } => panic!("CountPartials::partial_hellinger() called on a Presence layer"),
KmerLayer::Empty { .. } => panic!("CountPartials::partial_hellinger() called on an Empty layer"),
}
}
}
impl BitPartials for KmerLayer {
fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) {
match self {
KmerLayer::Presence { layer, .. } => layer.partial_jaccard(),
KmerLayer::Count { .. } => panic!("BitPartials::partial_jaccard() called on a Count layer"),
KmerLayer::Empty { .. } => panic!("BitPartials::partial_jaccard() called on an Empty layer"),
}
}
fn partial_hamming(&self) -> Array2<u64> {
match self {
KmerLayer::Presence { layer, .. } => layer.partial_hamming(),
KmerLayer::Count { .. } => panic!("BitPartials::partial_hamming() called on a Count layer"),
KmerLayer::Empty { .. } => panic!("BitPartials::partial_hamming() called on an Empty layer"),
}
}
}