feat: add Sankoff cost-matrix calibration CLI and serialization

Introduce the --sankoff flag to generate subsampled calibration bundles and pairwise cost matrices. Expose internal sibling algorithm types as public to support external consumption. Implement YAML, CSV, and FASTA output serialization using serde, gated by a configurable substitution saturation ceiling and requiring the --subsample flag.
This commit is contained in:
Eric Coissac
2026-08-28 21:19:20 +02:00
parent 9043868228
commit 1536217ac1
11 changed files with 320 additions and 35 deletions
@@ -22,7 +22,7 @@ use super::pairwise::{BasePairTally, CardinalityTally};
/// Row-stochastic 5×5 cardinality transition probabilities (`0..=4`),
/// diagonal included ("stay at the same cardinality"), from
/// [`CardinalityTally`]'s pooled co-occurrence counts.
pub(crate) fn cardinality_transition_probs(tally: &CardinalityTally) -> [[f64; 5]; 5] {
pub fn cardinality_transition_probs(tally: &CardinalityTally) -> [[f64; 5]; 5] {
let mut p = [[0.0f64; 5]; 5];
for a in 0..5 {
let row_sum: u64 = tally.counts[a].iter().sum();
@@ -40,7 +40,7 @@ pub(crate) fn cardinality_transition_probs(tally: &CardinalityTally) -> [[f64; 5
/// (`A,C,G,T`), diagonal included ("stay the same base"), from
/// [`BasePairTally`]'s pooled substitution (off-diagonal) and agreement
/// (`same`, diagonal) counts — unambiguous, cardinality-1 loci only.
pub(crate) fn composition_transition_probs(tally: &BasePairTally) -> [[f64; 4]; 4] {
pub fn composition_transition_probs(tally: &BasePairTally) -> [[f64; 4]; 4] {
let mut p = [[0.0f64; 4]; 4];
for a in 0..4 {
let row_sum = tally.same[a] + (0..4).map(|b| tally.counts[a][b]).sum::<u64>();
@@ -143,7 +143,7 @@ fn best_pairing_cost(lost: &[u8], gained: &[u8], p_comp: &[[f64; 4]; 4]) -> f64
/// factor dropped, cost is driven only by composition matching
/// (shared-base retention and paired substitutions), never by a state
/// pair's cardinality difference alone.
pub(crate) fn pairwise_cost_matrix(
pub fn pairwise_cost_matrix(
p_card: &[[f64; 5]; 5],
p_comp: &[[f64; 4]; 4],
free_loss: bool,
+4 -3
View File
@@ -27,17 +27,18 @@ mod subsample;
use obikidxcache::index_cache::IndexCache;
pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix};
pub use alignment::SnpAlignment;
pub use pairwise::{BasePairTally, CardinalityTally, RawSnpDistanceOutput};
pub use sankoff::SankoffBundle;
pub use stats::SiblingAnnexStats;
pub use subsample::EntropyBias;
pub(crate) use alignment::snp_pseudo_alignment;
pub(crate) use annex::build_layer_sibling_annex;
pub(crate) use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix};
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4, iter_full_entropy};
pub(crate) use family_scan::{Selection, scan_layer_families};
pub(crate) use pairwise::{BasePairTally, CardinalityTally, RawSnpDistanceOutput};
pub(crate) use sankoff::{SankoffBundle, sankoff_bundle};
pub(crate) use sankoff::sankoff_bundle;
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
pub(crate) use subsample::sample_index;
@@ -178,8 +178,9 @@ impl PairwiseTally {
}
/// Raw p-distance restricted to loci that are single-copy in **both**
/// genomes of a pair — see `PairwiseTally`'s module docs.
pub(crate) struct RawSnpDistanceOutput {
/// genomes of a pair — see `PairwiseTally`'s module docs. `pub`: part of
/// [`super::SankoffBundle`]'s public signature.
pub struct RawSnpDistanceOutput {
/// n×n count of eligible loci where the two genomes' single forms differ.
pub snp: Array2<u64>,
/// n×n count of eligible loci where the two genomes' single forms agree.
@@ -187,8 +188,9 @@ pub(crate) struct RawSnpDistanceOutput {
}
/// Symmetric 6-category base-pair substitution tally (indexed
/// `0=A,1=C,2=G,3=T`), pooled over [`PairwiseTally::included`] genome pairs.
pub(crate) struct BasePairTally {
/// `0=A,1=C,2=G,3=T`), pooled over [`PairwiseTally::included`] genome
/// pairs. `pub`: part of [`super::SankoffBundle`]'s public signature.
pub struct BasePairTally {
/// `counts[a][b] == counts[b][a]` = number of eligible loci, pooled over
/// included genome pairs, where the two genomes' single forms are `a`
/// and `b`. Diagonal always `0` — an `a == b` locus is counted in
@@ -207,8 +209,9 @@ pub(crate) struct BasePairTally {
/// [`PairwiseTally::cardinality_tally`]'s own docs for why this is *not*
/// gated by `ratio_ceiling` the way [`BasePairTally`] is), restricted to
/// variable families (`family_size() >= 2`) — matching
/// `snp_pseudo_alignment`'s own scope.
pub(crate) struct CardinalityTally {
/// `snp_pseudo_alignment`'s own scope. `pub`: part of
/// [`super::SankoffBundle`]'s public signature.
pub struct CardinalityTally {
/// `counts[a][b] == counts[b][a]` = number of family sites, pooled over
/// included genome pairs, where one genome's family cardinality is `a`
/// and the other's is `b`. Diagonal is real data here (both genomes at
@@ -19,8 +19,9 @@ use super::subsample::{EntropyBias, sample_index};
/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs,
/// computed together from one shared, possibly-subsampled/entropy-biased
/// selection — see the module docs.
pub(crate) struct SankoffBundle {
/// selection — see the module docs. `pub`: part of the public signature of
/// [`crate::siblings::extensions::SiblingExt::sankoff_bundle`].
pub struct SankoffBundle {
pub alignment: SnpAlignment,
pub raw: RawSnpDistanceOutput,
pub base_pair_tally: BasePairTally,
@@ -14,9 +14,10 @@ use obikindex::{OKIError, OKIResult};
use obisys::progress_bar;
use crate::siblings::algorithms::{
EntropyBias, Selection, SiblingAnnexStats, SnpAlignment, build_layer_sibling_annex,
family_entropy, family_entropy_4, is_fast_mode, scan_layer_families,
sibling_annex_stats, sibling_family_size_histogram, snp_pseudo_alignment,
EntropyBias, SankoffBundle, Selection, SiblingAnnexStats, SnpAlignment,
build_layer_sibling_annex, family_entropy, family_entropy_4, is_fast_mode,
sankoff_bundle, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram,
snp_pseudo_alignment,
};
use crate::siblings::extensions::SiblingBuilder;
use crate::siblings::ENTROPY_ANNEX_FILE_NAME;
@@ -94,6 +95,32 @@ pub trait SiblingExt {
/// use [`sibling_family_size_histogram`](Self::sibling_family_size_histogram)
/// instead when only the global histogram is needed.
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
/// Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree`
/// pipeline: one shared, possibly-subsampled/entropy-biased selection
/// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same
/// meaning as [`snp_pseudo_alignment`](Self::snp_pseudo_alignment)'s
/// own) drives a single scan producing the pseudo-alignment *and* every
/// tally the Sankoff cost-matrix calibration
/// (`crate::siblings::cardinality_transition_probs`/
/// `composition_transition_probs`/`pairwise_cost_matrix`) needs —
/// never two independent draws of the same index. `ratio_ceiling`
/// (`--sankoff-ratio-ceiling`) excludes genome pairs too close to
/// substitution saturation from `SankoffBundle::base_pair_tally`'s
/// pool (a saturated pair's base composition is noise, not signal) —
/// `SankoffBundle::cardinality_tally` is *not* gated by it (see
/// `crate::siblings::CardinalityTally`'s own docs for why that
/// wouldn't make sense: cardinality reflects each genome's own
/// coverage/duplication structure, not the pair's mutual divergence).
fn sankoff_bundle(
&self,
n: usize,
free_loss: bool,
no_ambiguity: bool,
excluded: &[bool],
entropy_bias: Option<EntropyBias>,
ratio_ceiling: f64,
) -> OKIResult<SankoffBundle>;
}
impl SiblingExt for IndexCache {
@@ -226,4 +253,16 @@ impl SiblingExt for IndexCache {
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats> {
sibling_annex_stats(self)
}
fn sankoff_bundle(
&self,
n: usize,
free_loss: bool,
no_ambiguity: bool,
excluded: &[bool],
entropy_bias: Option<EntropyBias>,
ratio_ceiling: f64,
) -> OKIResult<SankoffBundle> {
sankoff_bundle(self, n, free_loss, no_ambiguity, excluded, entropy_bias, ratio_ceiling)
}
}
+5 -1
View File
@@ -29,7 +29,11 @@ mod siblingannex;
pub(crate) use entropy_annex::{ENTROPY_ANNEX_FILE_NAME, EntropyAnnex, EntropyAnnexBuilder};
pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use algorithms::{EntropyBias, SiblingAnnexStats, SnpAlignment};
pub use algorithms::{
BasePairTally, CardinalityTally, EntropyBias, RawSnpDistanceOutput, SankoffBundle,
SiblingAnnexStats, SnpAlignment, cardinality_transition_probs, composition_transition_probs,
pairwise_cost_matrix,
};
pub use extensions::SiblingExt;
pub(crate) const ANNEX_FILE_NAME: &str = "siblings.psib";