Files
obikmer/src/obikphylo/src/siblings/algorithms/snp_distance.rs
T
Eric Coissac 996ebaf977 Refactor algorithms to use configuration flags
Introduce boolean flags for `AlignmentReducer` and `PairwiseReducer` to control behavior. Update masking logic and introduce the `SurvivingFamily` structure for explicit data handling.
2026-10-13 08:44:19 +02:00

399 lines
16 KiB
Rust

//! `snp-*` `--distance` values — closed-form (method-of-moments) pairwise
//! corrections over the central-position SNP substitution spectrum, all
//! derived from one shared [`PairwiseTally`] (see its own module docs: a
//! single `O(n²)` structure, built once, from which every correction below
//! is cheap post-processing, never a second index scan). See
//! `DevDocMD/theory/evolutionary_distances.md`, "`--distance` unification"
//! for the design discussion and the literature this implements
//! ([`ape`](https://github.com/emmanuelparadis/ape)'s `dist.dna` C source,
//! `src/dist_dna.c`, verified formula-by-formula against the upstream
//! implementation rather than re-derived from memory).
//!
//! Deliberately **not** included here: LogDet/paralinear (needs the true
//! *directional* per-pair base co-occurrence matrix — `PairwiseTally`
//! only keeps the symmetrised substitution counts `BasePairTally` itself
//! wants, which loses exactly the compositional-asymmetry information
//! LogDet exists to detect), Tajima-Nei (needs each genome's *own* base
//! composition, not the pair-pooled estimate `base_freq` below provides),
//! F84 and the ML-fit HKY85 tree usage (no formula independently verified
//! against a primary source at implementation time). Adding any of these
//! later is a new function in this module plus, for LogDet/Tajima-Nei, a
//! new field on `PairStats`/a per-genome accumulator — not an architecture
//! change.
use ndarray::Array2;
use obikindex::OKIResult;
use super::pairwise::{PairwiseTally, PartitionDispersion};
/// One `snp-*` `--distance` value. `pub`: part of the public signature of
/// [`crate::siblings::extensions::SiblingExt`]'s distance-computing
/// callers.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SnpDistanceKind {
/// Uncorrected p-distance (`snp / (snp + shared)`).
Raw,
/// Jukes-Cantor (JC69): corrects for multiple substitutions per site,
/// equal rates, equal base frequencies.
Jc,
/// Kimura 2-parameter (K80): + transition/transversion rate bias.
K2p,
/// Kimura 3-parameter (K81/K3ST): splits transversions into the two
/// categories A↔C/G↔T and A↔T/C↔G, each its own rate.
K81,
/// Felsenstein 81 (F81): JC69 + unequal (pair-pooled empirical) base
/// frequencies, no ts/tv distinction.
F81,
/// Tamura 3-parameter (T92): K80 + GC-content bias.
T92,
/// Tamura-Nei (TN93): unequal base frequencies + separate purine
/// (A↔G) / pyrimidine (C↔T) transition rates + a transversion rate —
/// the richest closed-form correction implemented here.
Tn93,
/// Transversions-only p-distance — diagnostic/deep-divergence variant,
/// deliberately uncorrected (dropping transitions, which saturate
/// first, is itself the correction).
Tv,
}
impl SnpDistanceKind {
/// Whether `--gamma-shape` applies to this correction — every
/// rate-based model here except the two with no "-ln(x)" term to
/// reinterpret as a gamma mixture (`Raw`, nothing to correct; `Tv`,
/// deliberately uncorrected).
pub fn supports_gamma(self) -> bool {
!matches!(self, SnpDistanceKind::Raw | SnpDistanceKind::Tv)
}
}
/// `--gamma-shape` value. `Disabled` (no flag given) leaves every
/// `corrected_log` call at its plain `-ln(x)` term; `Fixed` threads the
/// given `alpha` through unchanged; `Auto` estimates `alpha` once, up
/// front, from [`PartitionDispersion::estimate_alpha`] over the same
/// sampling pass that builds the [`PairwiseTally`] — see that method's own
/// docs for the estimator itself.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum GammaShape {
Disabled,
Fixed(f64),
Auto,
}
/// `-ln(x)`, gamma-mixture corrected when `alpha` is given: replaces the
/// single-rate `-ln(x)` term with the standard Jin-Nei (1990) gamma
/// substitution `alpha * (x^(-1/alpha) - 1)` — the mechanical term-wise
/// rewrite every gamma-corrected distance in the literature (JC+Γ, K80+Γ,
/// F81+Γ, ...) applies to each log term of its base formula; verified here
/// against `ape`'s own JC69/K80/F81 gamma branches (`dist_dna.c`), then
/// generalised the same way to every other log term this module uses
/// (K81/T92/TN93 each being multi-term sums of exactly this shape).
fn corrected_log(x: f64, alpha: Option<f64>) -> f64 {
match alpha {
Some(alpha) => alpha * (x.powf(-1.0 / alpha) - 1.0),
None => -x.ln(),
}
}
pub(crate) fn raw(tally: &PairwiseTally, i: usize, j: usize) -> f64 {
let c = tally.categories(i, j);
let l = c.n_eligible();
if l == 0 {
return f64::NAN;
}
(c.ts1 + c.ts2 + c.tv1 + c.tv2) as f64 / l as f64
}
pub(crate) fn tv_only(tally: &PairwiseTally, i: usize, j: usize) -> f64 {
let c = tally.categories(i, j);
let l = c.n_eligible();
if l == 0 {
return f64::NAN;
}
(c.tv1 + c.tv2) as f64 / l as f64
}
pub(crate) fn jc(tally: &PairwiseTally, i: usize, j: usize, alpha: Option<f64>) -> f64 {
let p = raw(tally, i, j);
0.75 * corrected_log(1.0 - 4.0 * p / 3.0, alpha)
}
pub(crate) fn k2p(tally: &PairwiseTally, i: usize, j: usize, alpha: Option<f64>) -> f64 {
let c = tally.categories(i, j);
let l = c.n_eligible();
if l == 0 {
return f64::NAN;
}
let p = (c.ts1 + c.ts2) as f64 / l as f64;
let q = (c.tv1 + c.tv2) as f64 / l as f64;
let a1 = 1.0 - 2.0 * p - q;
let a2 = 1.0 - 2.0 * q;
0.5 * corrected_log(a1, alpha) + 0.25 * corrected_log(a2, alpha)
}
pub(crate) fn k81(tally: &PairwiseTally, i: usize, j: usize, alpha: Option<f64>) -> f64 {
let c = tally.categories(i, j);
let l = c.n_eligible();
if l == 0 {
return f64::NAN;
}
let p = (c.ts1 + c.ts2) as f64 / l as f64;
let q = c.tv1 as f64 / l as f64;
let r = c.tv2 as f64 / l as f64;
let a1 = 1.0 - 2.0 * p - 2.0 * q;
let a2 = 1.0 - 2.0 * p - 2.0 * r;
let a3 = 1.0 - 2.0 * q - 2.0 * r;
0.25 * (corrected_log(a1, alpha) + corrected_log(a2, alpha) + corrected_log(a3, alpha))
}
pub(crate) fn f81(tally: &PairwiseTally, i: usize, j: usize, alpha: Option<f64>) -> f64 {
let p = raw(tally, i, j);
let freq = tally.base_freq(i, j);
let e = 1.0 - freq.iter().map(|f| f * f).sum::<f64>();
match alpha {
Some(a) => e * a * ((1.0 - p / e).powf(-1.0 / a) - 1.0),
None => -e * (1.0 - p / e).ln(),
}
}
pub(crate) fn t92(tally: &PairwiseTally, i: usize, j: usize, alpha: Option<f64>) -> f64 {
let c = tally.categories(i, j);
let l = c.n_eligible();
if l == 0 {
return f64::NAN;
}
let p = (c.ts1 + c.ts2) as f64 / l as f64;
let q = (c.tv1 + c.tv2) as f64 / l as f64;
let freq = tally.base_freq(i, j);
let gc = freq[1] + freq[2]; // C + G
let wg = 2.0 * gc * (1.0 - gc);
let a1 = 1.0 - p / wg - q;
let a2 = 1.0 - 2.0 * q;
wg * corrected_log(a1, alpha) + 0.5 * (1.0 - wg) * corrected_log(a2, alpha)
}
pub(crate) fn tn93(tally: &PairwiseTally, i: usize, j: usize, alpha: Option<f64>) -> f64 {
let c = tally.categories(i, j);
let l = c.n_eligible();
if l == 0 {
return f64::NAN;
}
let freq = tally.base_freq(i, j);
let g_r = freq[0] + freq[2]; // A + G (purines)
let g_y = freq[1] + freq[3]; // C + T (pyrimidines)
let k1 = 2.0 * freq[0] * freq[2] / g_r;
let k2 = 2.0 * freq[1] * freq[3] / g_y;
let k3 = 2.0 * (g_r * g_y - freq[0] * freq[2] * g_y / g_r - freq[1] * freq[3] * g_r / g_y);
let p1 = c.ts1 as f64 / l as f64; // A<->G
let p2 = c.ts2 as f64 / l as f64; // C<->T
let q = (c.tv1 + c.tv2) as f64 / l as f64;
let w1 = 1.0 - p1 / k1 - q / (2.0 * g_r);
let w2 = 1.0 - p2 / k2 - q / (2.0 * g_y);
let w3 = 1.0 - q / (2.0 * g_r * g_y);
k1 * corrected_log(w1, alpha) + k2 * corrected_log(w2, alpha) + k3 * corrected_log(w3, alpha)
}
fn formula(kind: SnpDistanceKind) -> fn(&PairwiseTally, usize, usize, Option<f64>) -> f64 {
match kind {
SnpDistanceKind::Raw => |t, i, j, _| raw(t, i, j),
SnpDistanceKind::Tv => |t, i, j, _| tv_only(t, i, j),
SnpDistanceKind::Jc => jc,
SnpDistanceKind::K2p => k2p,
SnpDistanceKind::K81 => k81,
SnpDistanceKind::F81 => f81,
SnpDistanceKind::T92 => t92,
SnpDistanceKind::Tn93 => tn93,
}
}
/// Everything downstream of an already-built
/// [`PairwiseTally`]/[`PartitionDispersion`]: the `--gamma-shape` support
/// check, `alpha` resolution (fixed/auto/disabled), and the final `n×n`
/// matrix build — `O(n²)` post-processing, never a scan of its own. The
/// tally/dispersion themselves come from a
/// [`super::pairwise::PairwiseReducer`] that already ran through
/// [`super::scan::sampled_scan`] — see [`super::sankoff::SankoffBundle::snp_distance`]
/// for why reusing an already-scanned reducer, rather than a second
/// [`sampled_scan`](super::scan::sampled_scan) call, is a correctness
/// requirement (not just a performance nicety) whenever `--sankoff` and a
/// `snp-*` `--distance` are requested together.
pub fn distance_matrix(
tally: &PairwiseTally,
dispersion: &PartitionDispersion,
kind: SnpDistanceKind,
gamma_shape: GammaShape,
) -> OKIResult<Array2<f64>> {
if gamma_shape != GammaShape::Disabled && !kind.supports_gamma() {
return Err(obikindex::OKIError::InvalidInput(
"--gamma-shape has no effect on this --distance value".into(),
));
}
let alpha = match gamma_shape {
GammaShape::Disabled => None,
GammaShape::Fixed(a) => Some(a),
GammaShape::Auto => match dispersion.estimate_alpha() {
Some(a) => {
tracing::info!(
"--gamma-shape auto: estimated alpha = {a:.4} (method-of-moments, \
over-dispersion of substitution rate across partitions)"
);
Some(a)
}
None => {
tracing::warn!(
"--gamma-shape auto: no measurable among-partition rate over-dispersion \
detected — proceeding without the gamma correction"
);
None
}
},
};
let n_genomes = tally.n_genomes();
let f = formula(kind);
Ok(Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| {
if i == j { 0.0 } else { f(tally, i, j, alpha) }
}))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::siblings::FamilyMask;
use crate::siblings::algorithms::SurvivingFamily;
use super::super::pairwise::reduce_pairwise;
// Base bit positions, matching `PairwiseTally::categories`' convention
// (0=A, 1=C, 2=G, 3=T).
const A: u8 = 1 << 0;
const C: u8 = 1 << 1;
const G: u8 = 1 << 2;
const T: u8 = 1 << 3;
/// Two-genome synthetic tally: one variable family per `(genome0,
/// genome1)` pair in `pairs`, plus `n_shared` fully-agreeing families to
/// pad the eligible-locus denominator — cycled over all 4 bases (not a
/// single one) so the pooled base composition stays non-degenerate
/// (`F81`/`T92`/`TN93`'s denominators — `E`/`wg` — are exactly `0` for a
/// single-base composition, an edge case irrelevant to real data, not
/// something these tests need to exercise).
fn tally_2genomes(pairs: &[(u8, u8)], n_shared: usize) -> PairwiseTally {
// No masking requested (`free_loss = no_ambiguity = false`) —
// `masked_state(m, false, false)` is always `Some(m)`, so the raw
// mask is exactly what `subsample::sample_layer` would have
// computed here.
let mut families = Vec::new();
for &(a, b) in pairs {
families.push(SurvivingFamily {
family_idx: families.len(),
mask: FamilyMask::EMPTY.with(a.trailing_zeros() as u8).with(b.trailing_zeros() as u8),
masked: vec![Some(a), Some(b)],
});
}
const BASES: [u8; 4] = [A, C, G, T];
for i in 0..n_shared {
let b = BASES[i % 4];
let other = BASES[(i + 1) % 4];
families.push(SurvivingFamily {
family_idx: families.len(),
mask: FamilyMask::EMPTY.with(b.trailing_zeros() as u8).with(other.trailing_zeros() as u8),
masked: vec![Some(b), Some(b)],
});
}
let mut tally = PairwiseTally::new(2);
let mut dispersion = PartitionDispersion::default();
reduce_pairwise(&families, 0, &mut tally, &mut dispersion);
tally
}
#[test]
fn categories_classify_each_substitution_type() {
// A<->G (ts1), C<->T (ts2), A<->C (tv1), A<->T (tv2), + 1 shared.
let tally = tally_2genomes(&[(A, G), (C, T), (A, C), (A, T)], 1);
let c = tally.categories(0, 1);
assert_eq!((c.ts1, c.ts2, c.tv1, c.tv2, c.shared), (1, 1, 1, 1, 1));
assert_eq!(c.n_eligible(), 5);
}
#[test]
fn identical_genomes_give_zero_distance_under_every_correction() {
// No substitutions at all — every formula's "-ln(1)" term is 0.
let tally = tally_2genomes(&[], 10);
for kind in [
SnpDistanceKind::Raw,
SnpDistanceKind::Jc,
SnpDistanceKind::K2p,
SnpDistanceKind::K81,
SnpDistanceKind::F81,
SnpDistanceKind::T92,
SnpDistanceKind::Tn93,
SnpDistanceKind::Tv,
] {
let d = formula(kind)(&tally, 0, 1, None);
assert!(d.abs() < 1e-9, "{kind:?} gave {d}, expected ~0");
}
}
#[test]
fn raw_matches_hand_computed_ratio() {
// 1 substitution (A<->G) out of 5 eligible loci (1 subst + 4 shared).
let tally = tally_2genomes(&[(A, G)], 4);
assert!((raw(&tally, 0, 1) - 0.2).abs() < 1e-12);
}
#[test]
fn jc_matches_hand_computed_correction() {
// p = 1/5 = 0.2 -> d = -0.75 * ln(1 - 4*0.2/3) = -0.75 * ln(1 - 4/15).
let tally = tally_2genomes(&[(A, G)], 4);
let expected = -0.75 * (1.0 - 4.0f64 * 0.2 / 3.0).ln();
assert!((jc(&tally, 0, 1, None) - expected).abs() < 1e-12);
}
#[test]
fn jc_uncorrected_undershoots_raw_p_distance() {
// JC always corrects *upward* from the raw p-distance (multiple-hit
// correction only adds distance, never removes it) for any p in its
// domain.
let tally = tally_2genomes(&[(A, G), (C, T)], 20);
let p = raw(&tally, 0, 1);
let d = jc(&tally, 0, 1, None);
assert!(d > p, "JC-corrected {d} should exceed raw p-distance {p}");
}
#[test]
fn gamma_correction_converges_to_uncorrected_as_alpha_grows() {
// The Jin-Nei substitution alpha*(x^(-1/alpha) - 1) -> -ln(x) as
// alpha -> infinity — a basic sanity check on `corrected_log`
// independent of any hand-checked literature value.
let tally = tally_2genomes(&[(A, G), (A, C)], 20);
let uncorrected = jc(&tally, 0, 1, None);
let large_alpha = jc(&tally, 0, 1, Some(1.0e6));
assert!(
(uncorrected - large_alpha).abs() < 1e-3,
"uncorrected {uncorrected} vs large-alpha gamma {large_alpha}"
);
}
#[test]
fn gamma_shape_rejected_for_unsupported_kind() {
assert!(!SnpDistanceKind::Raw.supports_gamma());
assert!(!SnpDistanceKind::Tv.supports_gamma());
assert!(SnpDistanceKind::Jc.supports_gamma());
}
#[test]
fn base_freq_sums_to_one_and_matches_pooled_counts() {
// 2 A/G substitutions (genome0=A, genome1=G each time) + 1 shared
// A/A family -> pooled bases across both genomes: A appears 4 times
// (1 per substitution's genome0 side, 2 from the shared locus's
// both genomes), G appears 2 times (1 per substitution's genome1
// side), total 6 (2 genomes * 3 loci).
let tally = tally_2genomes(&[(A, G), (A, G)], 1);
let freq = tally.base_freq(0, 1);
assert!((freq.iter().sum::<f64>() - 1.0).abs() < 1e-12);
assert!((freq[0] - 4.0 / 6.0).abs() < 1e-12); // A
assert!((freq[2] - 2.0 / 6.0).abs() < 1e-12); // G
assert!((freq[1] - 0.0).abs() < 1e-12); // C
}
}