fix: prevent probability underflow in pairwise cost matrix

Replaces premature exponentiation-based row normalization with log-sum-exp arithmetic to prevent tiny probabilities from collapsing to exactly zero. This eliminates spurious infinite costs for valid but rare transitions while preserving correct IEEE 754 semantics for genuinely unobserved pairs. Adds explicit guards against NaN in degenerate rows and includes a regression test verifying finite costs for probabilities as low as 1e-200.

src/obikphylo/src/cardcomp.rs

@@ -146,52 +146,66 @@ fn best_pairing_cost(lost: &[u8], gained: &[u8], p_comp: &[[f64; 4]; 4]) -> f64
 /// only by composition matching (shared-base retention and paired
 /// substitutions), never by a state pair's cardinality difference alone.
 pub fn pairwise_cost_matrix(p_card: &[[f64; 5]; 5], p_comp: &[[f64; 4]; 4], free_loss: bool) -> [[f64; 16]; 16] {
-    let mut raw = [[0.0f64; 16]; 16];
+    let mut log_p = [[0.0f64; 16]; 16]; // ln(P), *before* row-normalisation
     for a in 0u8..16 {
         for b in 0u8..16 {
             let shared = a & b;
             let lost: Vec<u8> = (0..4).filter(|&i| a & (1 << i) != 0 && b & (1 << i) == 0).collect();
             let gained: Vec<u8> = (0..4).filter(|&i| b & (1 << i) != 0 && a & (1 << i) == 0).collect();
 
-            let mut log_p = 0.0; // accumulate ln(P), so 0.0 = probability 1
+            let mut lp = 0.0; // accumulate ln(P), so 0.0 = probability 1
             if !free_loss {
                 let card_a = a.count_ones() as usize;
                 let card_b = b.count_ones() as usize;
                 let p_c = p_card[card_a][card_b];
-                log_p += if p_c > 0.0 { p_c.ln() } else { f64::NEG_INFINITY };
+                lp += if p_c > 0.0 { p_c.ln() } else { f64::NEG_INFINITY };
             }
 
             for i in 0..4u8 {
                 if shared & (1 << i) != 0 {
                     let p = p_comp[i as usize][i as usize];
-                    log_p += if p > 0.0 { p.ln() } else { f64::NEG_INFINITY };
+                    lp += if p > 0.0 { p.ln() } else { f64::NEG_INFINITY };
                 }
             }
-            log_p -= best_pairing_cost(&lost, &gained, p_comp);
+            lp -= best_pairing_cost(&lost, &gained, p_comp);
 
-            raw[a as usize][b as usize] = log_p.exp();
+            log_p[a as usize][b as usize] = lp;
         }
     }
 
-    // Row-normalise to a proper transition probability matrix.
-    let mut p = [[0.0f64; 16]; 16];
-    for a in 0..16 {
-        let row_sum: f64 = raw[a].iter().sum();
-        if row_sum > 0.0 {
-            for b in 0..16 {
-                p[a][b] = raw[a][b] / row_sum;
-            }
-        }
-    }
-
-    // Cost = -ln(P), then symmetrise (see doc comment: parsimony on an
-    // unrooted tree requires a symmetric cost matrix).
+    // Row-normalise and convert to cost entirely in log-space
+    // (log-sum-exp), never exponentiating a raw `log_p` value directly —
+    // for a transition reachable only through several low-probability
+    // steps, `log_p.exp()` can underflow to exactly `0.0` (anything below
+    // roughly `-709` does, in `f64`), silently turning a real, if small,
+    // probability into a hard `+∞` cost. Observed in practice: a
+    // calibrated matrix with many such exact-zero entries is numerically
+    // unstable for IQ-TREE's own likelihood-derivative computation
+    // ("Numerical underflow for lh-derivative"). `log_sum_exp` never
+    // exponentiates anything above `0` (every term is shifted by the
+    // row's own max first), so it stays accurate across the full range
+    // `f64` can represent, not just what survives a direct `exp()`.
     let mut cost = [[0.0f64; 16]; 16];
     for a in 0..16 {
+        let row_max = log_p[a].iter().cloned().fold(f64::NEG_INFINITY, f64::max);
+        if row_max == f64::NEG_INFINITY {
+            // Every transition out of this state has probability 0 in the
+            // calibration data — genuinely unreachable, not underflow.
+            cost[a] = [f64::INFINITY; 16];
+            continue;
+        }
+        let log_sum = row_max + log_p[a].iter().map(|&lp| (lp - row_max).exp()).sum::<f64>().ln();
         for b in 0..16 {
-            cost[a][b] = if p[a][b] > 0.0 { -p[a][b].ln() } else { f64::INFINITY };
+            // `log_sum - log_p[a][b]` is `+∞` automatically when
+            // `log_p[a][b] == -∞` (IEEE 754: finite − (−∞) = +∞) — no
+            // separate branch needed for a genuinely zero-probability
+            // transition.
+            cost[a][b] = log_sum - log_p[a][b];
         }
     }
+
+    // Symmetrise (see doc comment: parsimony on an unrooted tree requires
+    // a symmetric cost matrix).
     let mut sym = [[0.0f64; 16]; 16];
     for a in 0..16 {
         for b in 0..16 {
@@ -289,4 +303,34 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn underflow_prone_transition_gets_finite_cost_not_infinite() {
+        // Every off-diagonal composition probability is tiny (1e-200) but
+        // not exactly zero — small enough that the naive `log_p.exp()`
+        // (exponentiate before row-normalising) hard-underflows to `0.0`
+        // in `f64` well before the probability is truly zero, silently
+        // turning a real, if minuscule, probability into `+∞` cost. The
+        // log-sum-exp normalisation in `pairwise_cost_matrix` must keep
+        // this finite instead.
+        let p_card = [[0.2f64; 5]; 5];
+        let tiny = 1e-200;
+        let mut p_comp = [[tiny; 4]; 4];
+        for i in 0..4 {
+            p_comp[i][i] = 1.0 - 3.0 * tiny;
+        }
+
+        let cost = pairwise_cost_matrix(&p_card, &p_comp, false);
+
+        // {A,C} -> {G,T}: no shared bases, both members substituted — the
+        // shape most prone to underflow (several tiny-probability factors
+        // multiplied together).
+        let a = 0b0011u8; // A, C
+        let b = 0b1100u8; // G, T
+        assert!(
+            cost[a as usize][b as usize].is_finite(),
+            "cost must stay finite for a merely tiny (not exactly zero) probability, got {}",
+            cost[a as usize][b as usize]
+        );
+    }
 }