diff --git a/src/Cargo.lock b/src/Cargo.lock index 2aac7e69..beb2b3ff 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1594,7 +1594,6 @@ dependencies = [ "obikindex", "obikindexer", "obikseq", - "obiksession", "obipipeline", "obiread", "obiskbuilder", diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 888c7367..e77ffa98 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -24,43 +24,6 @@ use phylip::write_phylip_relaxed; use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params}; use tnt::write_sankoff_tnt; -/// Deterministic byte encoding of the `snp-*` `--distance` selection -/// parameters, for `obiksession::Session::open`'s params-conflict check. -/// `obiksession` treats this as an opaque blob (see its own crate docs) — -/// only this module needs to know the layout, and only well enough to -/// produce/compare it, never to pretty-print a per-field diff (v1 keeps -/// the conflict message generic rather than decoding both sides field by -/// field — see its use in `run` below). -fn session_params_blob( - n: Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, -) -> Vec { - let mut buf = Vec::new(); - match n { - Some(v) => { - buf.push(1); - buf.extend_from_slice(&(v as u64).to_le_bytes()); - } - None => buf.push(0), - } - buf.push(free_loss as u8); - buf.push(no_ambiguity as u8); - buf.extend_from_slice(&(excluded.len() as u64).to_le_bytes()); - buf.extend(excluded.iter().map(|&b| b as u8)); - match entropy_bias { - Some(EntropyBias { mu, sigma }) => { - buf.push(1); - buf.extend_from_slice(&mu.to_le_bytes()); - buf.extend_from_slice(&sigma.to_le_bytes()); - } - None => buf.push(0), - } - buf -} - pub use args::PhyloArgs; pub fn run(args: PhyloArgs) { @@ -273,63 +236,83 @@ pub fn run(args: PhyloArgs) { None }; - // ── `--session`: persist the snp-* sample across separate invocations ── - // Only covers `snp-*` `--distance` today (see `--session`'s own docs); - // `None` when `--session` isn't given, matching today's always-resample - // default exactly. - let session = args.session.as_ref().map(|dir| { - let params = session_params_blob( - args.subsample, - args.free_loss, - args.no_ambiguity, - &snp_exclude_mask, - entropy_bias, + // ── `--session` ────────────────────────────────────────────────────────── + // Temporarily unavailable: the per-driver checkpoint/resume plumbing + // it relied on (`algorithms::session_cache`) was removed along with + // the three independent sampling drivers it patched over (see + // `algorithms::scan`'s own module docs) — persistence needs to be + // rebuilt on top of the new single `sampled_scan` trunk instead of + // silently becoming a no-op. + if args.session.is_some() { + eprintln!( + "error: --session is temporarily unavailable (being rebuilt on top of the unified \ + sampling trunk) — rerun without --session/--session-force" ); - match obiksession::Session::open(dir, ¶ms, args.session_force) { - Ok(Ok(session)) => { - match session.outcome() { - obiksession::SessionOutcome::Created => { - info!("--session {}: starting fresh", dir.display()); - } - obiksession::SessionOutcome::Reused => { - info!("--session {}: found, will reuse its cached sample if present", dir.display()); - } - } - session - } - Ok(Err(_conflict)) => { - eprintln!( - "error: --session {} was built with different selection parameters \ - (--subsample/--free-loss/--no-ambiguity/--exclude-genome/--min-shared-family/\ - --entropy/--entropy-sd) than this run — pass --session-force to discard its \ - cached sample and start fresh, or point --session at a different directory", - dir.display() - ); - std::process::exit(1); - } - Err(e) => { - eprintln!("error opening --session {}: {e}", dir.display()); - std::process::exit(1); - } - } + std::process::exit(1); + } + + // ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ── + // All three consume exactly the same selection parameters + // (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/ + // `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from + // the *same* random sample when combined in one invocation — so + // whichever of them are requested here share exactly one + // `sampled_scan` call, each contributing only the reducer(s) it needs, + // instead of each running its own scan (which used to silently draw a + // different sample per flag — see `obikphylo::siblings::LayerReducer`'s + // own module docs). + let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree; + let snp_kind = args.distance.as_snp(); + let need_alignment = args.pseudo_alignment || sankoff_requested; + let need_pairwise = sankoff_requested || snp_kind.is_some(); + + if args.pseudo_alignment && args.subsample.is_none() { + eprintln!("error: --pseudo-alignment requires --subsample "); + std::process::exit(1); + } + if sankoff_requested && args.subsample.is_none() { + eprintln!("error: --sankoff requires --subsample "); + std::process::exit(1); + } + + let mut alignment_reducer = need_alignment.then(|| { + obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity) }); + let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n)); - // ── SNP pseudo-alignment (`--pseudo-alignment`) ───────────────────────────── - if args.pseudo_alignment { - let Some(subsample_n) = args.subsample else { - eprintln!("error: --pseudo-alignment requires --subsample "); - std::process::exit(1); - }; + if need_alignment || need_pairwise { + let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new(); + if let Some(r) = alignment_reducer.as_mut() { + reducers.push(r); + } + if let Some(r) = pairwise_reducer.as_mut() { + reducers.push(r); + } - info!("sampling SNP pseudo-alignment (target {subsample_n} site(s))"); - let t = Stage::start("pseudo_alignment"); - let alignment = cache - .snp_pseudo_alignment(subsample_n, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias) + info!( + "sampling SNP sites{}", + match args.subsample { + Some(n) => format!(" (target {n} site(s))"), + None => " (exhaustive)".into(), + } + ); + let t = Stage::start("sampled_scan"); + obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers) .unwrap_or_else(|e| { - eprintln!("error building pseudo-alignment: {e}"); + eprintln!("error sampling SNP sites: {e}"); std::process::exit(1); }); rep.push(t.stop()); + } + + // Finished once here (not left as reducers) so `--pseudo-alignment` + // and the Sankoff group can each read it below without fighting over + // which one consumes it — see `SankoffBundle::finish`'s own docs. + let alignment = alignment_reducer.map(obikphylo::siblings::AlignmentReducer::finish); + + // ── SNP pseudo-alignment (`--pseudo-alignment`) ───────────────────────────── + if args.pseudo_alignment { + let alignment = alignment.as_ref().expect("scanned above: need_alignment"); let path = args.output.as_ref() .map(|p| format!("{}_alignment.fasta", p.display())) @@ -350,33 +333,17 @@ pub fn run(args: PhyloArgs) { // `snp-*` `--distance` branch below can reuse its already-sampled // tally instead of resampling — see `SankoffBundle::snp_distance`'s own // docs for why that's required for correctness, not just speed, when - // both are requested in the same invocation (they always share the - // same `n`/`free_loss`/`no_ambiguity`/excluded-set/`entropy_bias`: the - // CLI has no way to give them different values in one run). + // both are requested in the same invocation. let mut bundle = None; - if args.sankoff || args.tnt || args.phyg || args.iqtree { - let Some(subsample_n) = args.subsample else { - eprintln!("error: --sankoff requires --subsample "); - std::process::exit(1); - }; - - info!("sampling Sankoff calibration bundle (target {subsample_n} site(s))"); - let t = Stage::start("sankoff_bundle"); - let b = cache - .sankoff_bundle( - subsample_n, - args.free_loss, - args.no_ambiguity, - &snp_exclude_mask, - entropy_bias, - args.sankoff_ratio_ceiling, - session.as_ref(), - ) - .unwrap_or_else(|e| { - eprintln!("error computing Sankoff calibration bundle: {e}"); - std::process::exit(1); - }); - rep.push(t.stop()); + if sankoff_requested { + let alignment = alignment.clone().expect("scanned above: need_alignment"); + let pairwise = pairwise_reducer.take().expect("scanned above: need_pairwise"); + let b = obikphylo::siblings::SankoffBundle::finish( + alignment, + pairwise, + args.sankoff_ratio_ceiling, + &snp_exclude_mask, + ); let p_card = cardinality_transition_probs(&b.cardinality_tally); let p_comp = composition_transition_probs(&b.base_pair_tally); @@ -441,7 +408,7 @@ pub fn run(args: PhyloArgs) { eprintln!("error: --shared-kmers has no meaning for a snp-* --distance value"); std::process::exit(1); } - let kind = args.distance.as_snp().expect("DistanceArg is always classic or snp"); + let kind = snp_kind.expect("DistanceArg is always classic or snp"); let gamma_shape = args.gamma_shape.map_or(GammaShape::Disabled, Into::into); let t = Stage::start("snp_distance"); let matrix = match &bundle { @@ -453,23 +420,9 @@ pub fn run(args: PhyloArgs) { b.snp_distance(kind, gamma_shape) } None => { - info!( - "computing {kind:?} SNP distance for {n} genome(s){}", - match args.subsample { - Some(n) => format!(" (subsampled, target {n} site(s))"), - None => " (exhaustive)".into(), - } - ); - cache.snp_distance( - kind, - args.subsample, - args.free_loss, - args.no_ambiguity, - &snp_exclude_mask, - entropy_bias, - gamma_shape, - session.as_ref(), - ) + info!("computing {kind:?} SNP distance for {n} genome(s) (reusing the sampled scan above)"); + let (tally, dispersion) = pairwise_reducer.take().expect("scanned above: need_pairwise").finish(); + obikphylo::siblings::distance_matrix(&tally, &dispersion, kind, gamma_shape) } } .unwrap_or_else(|e| { diff --git a/src/obikphylo/Cargo.toml b/src/obikphylo/Cargo.toml index 08a6a0dd..a088f00d 100644 --- a/src/obikphylo/Cargo.toml +++ b/src/obikphylo/Cargo.toml @@ -12,7 +12,6 @@ obicompactvec = { path = "../obicompactvec" } obikidxcache = { path = "../obikidxcache" } obiskbuilder = { path = "../obiskbuilder" } obipipeline = { path = "../obipipeline" } -obiksession = { path = "../obiksession" } kodama = "0.3.0" memmap2 = "0.9" ndarray = "0.17" diff --git a/src/obikphylo/src/siblings/algorithms/alignment.rs b/src/obikphylo/src/siblings/algorithms/alignment.rs index 63339bc4..1371fc41 100644 --- a/src/obikphylo/src/siblings/algorithms/alignment.rs +++ b/src/obikphylo/src/siblings/algorithms/alignment.rs @@ -1,27 +1,25 @@ //! SNP-only pseudo-alignment: one row (IUPAC-coded) per genome, one column -//! per sampled family — built directly on [`sample_index`], so every family -//! this produces has already survived both its Bernoulli draw and post-hoc +//! per sampled family — built as a [`LayerReducer`] over +//! [`super::scan::sampled_scan`]'s per-layer batches, so every family this +//! produces has already survived both its Bernoulli draw and post-hoc //! masking ([`super::masking::survives_masking`]). use obikidxcache::index_cache::IndexCache; -use obikindex::OKIResult; use rkyv::{Archive, Deserialize, Serialize}; use super::masking::{iupac_code, masked_state}; -use super::subsample::{EntropyBias, SurvivingFamily, sample_index}; +use super::scan::LayerReducer; +use super::subsample::SurvivingFamily; /// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the -/// same length (one byte per sampled family, in the order [`sample_index`] -/// produced them: layer by layer, `family_idx` order within each). An -/// excluded genome (`excluded[g]` in -/// [`snp_pseudo_alignment`]) has no row at all, not a blanked one — its +/// same length (one byte per sampled family, in the order +/// [`super::scan::sampled_scan`] produced them: layer by layer, +/// `family_idx` order within each). An excluded genome (`excluded[g]` in +/// [`AlignmentReducer::new`]) has no row at all, not a blanked one — its /// index simply doesn't appear in `genome_indices`. `pub`, not /// `pub(crate)`: part of the public signature of -/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`]. -/// -/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): `--session` persistence -/// for `SankoffBundle` — same reasoning as `PairwiseTally`'s own derive. -#[derive(Archive, Serialize, Deserialize)] +/// [`crate::siblings::extensions::SiblingExt`]'s sampling-based methods. +#[derive(Clone, Archive, Serialize, Deserialize)] pub struct SnpAlignment { pub sequences: Vec>, /// This index's own genome numbering (matching `IndexCache::meta().genomes()`'s @@ -31,50 +29,43 @@ pub struct SnpAlignment { pub genome_indices: Vec, } -/// See [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`] -/// for the public-facing docs — this is its implementation. A single -/// "reduce" ([`reduce_alignment`]) over [`sample_index`]'s per-layer -/// batches — the only consumer today, but structured so a future sibling -/// reduce (e.g. the Sankoff cost-matrix calibration's cardinality/ -/// composition tallies) can run over the exact same batches, each cloning -/// its own `Arc`, without this function changing at all. -pub(crate) fn snp_pseudo_alignment( - cache: &IndexCache, - n: usize, +/// Accumulates a [`SnpAlignment`] over a [`super::scan::sampled_scan`] +/// pass — a thin [`LayerReducer`] wrapper around [`reduce_alignment`], so +/// it can run alongside any other reducer (e.g. +/// `algorithms::pairwise::PairwiseReducer`) over the exact same batches in +/// one shared scan, instead of each needing its own `sample_index` call. +pub struct AlignmentReducer { + genome_indices: Vec, + sequences: Vec>, free_loss: bool, no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, -) -> OKIResult { - let n_genomes = cache.meta().genomes().len(); - let genome_indices: Vec = (0..n_genomes) - .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) - .collect(); - let mut sequences: Vec> = vec![Vec::new(); genome_indices.len()]; - - sample_index( - cache, - n, - free_loss, - no_ambiguity, - excluded, - entropy_bias, - 0, - |_partition, _layer, _raw_index, survivors| { - reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences); - }, - )?; - - Ok(SnpAlignment { sequences, genome_indices }) } -/// Appends one layer's worth of columns onto `sequences` — a "reduce" over -/// [`sample_index`]'s per-layer batch, self-contained (no sampling/masking -/// logic of its own, just character encoding) so it can be called -/// independently of, and alongside, any other reduce over the same batch — -/// `pub(super)`: `algorithms::sankoff::sankoff_bundle` runs this one -/// alongside `pairwise::reduce_pairwise` over the same `on_layer` call. -pub(super) fn reduce_alignment( +impl AlignmentReducer { + pub fn new(cache: &IndexCache, excluded: &[bool], free_loss: bool, no_ambiguity: bool) -> Self { + let n_genomes = cache.meta().genomes().len(); + let genome_indices: Vec = (0..n_genomes) + .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) + .collect(); + let sequences = vec![Vec::new(); genome_indices.len()]; + Self { genome_indices, sequences, free_loss, no_ambiguity } + } + + pub fn finish(self) -> SnpAlignment { + SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices } + } +} + +impl LayerReducer for AlignmentReducer { + fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { + reduce_alignment(survivors, &self.genome_indices, self.free_loss, self.no_ambiguity, &mut self.sequences); + } +} + +/// Appends one layer's worth of columns onto `sequences` — self-contained +/// (no sampling/masking logic of its own, just character encoding), the +/// actual reduce [`AlignmentReducer`] wraps. +fn reduce_alignment( survivors: &[SurvivingFamily], genome_indices: &[usize], free_loss: bool, diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 3d832fd8..7eff4aff 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -22,29 +22,27 @@ mod masking; mod minorant_selection; mod pairwise; mod sankoff; -mod session_cache; +mod scan; mod snp_distance; mod stats; mod subsample; use obikidxcache::index_cache::IndexCache; -pub use alignment::SnpAlignment; +pub use alignment::{AlignmentReducer, SnpAlignment}; pub use cardcomp::{ cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix, }; -pub use pairwise::{BasePairTally, CardinalityTally, RawSnpDistanceOutput}; +pub use pairwise::{BasePairTally, CardinalityTally, PairwiseReducer, RawSnpDistanceOutput}; pub use sankoff::SankoffBundle; -pub use snp_distance::{GammaShape, SnpDistanceKind}; +pub use scan::{LayerReducer, sampled_scan}; +pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix}; pub use stats::SiblingAnnexStats; pub use subsample::{EntropyBias, SurvivingFamily}; -pub(crate) use alignment::snp_pseudo_alignment; pub(crate) use annex::build_layer_sibling_annex; pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4}; pub(crate) use family_scan::{Selection, scan_layer_families}; -pub(crate) use sankoff::sankoff_bundle; -pub(crate) use snp_distance::snp_distance; pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram}; /// Whether every layer number in `cache` fits in a `FamilyMask` field diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index a651428c..e5934b3e 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -44,16 +44,15 @@ struct PairStats { cardinality: [[u64; 5]; 5], } -/// See the module docs. `pub(crate)`: consumed by -/// `algorithms::sankoff`'s orchestration, not yet exposed past this crate. +/// See the module docs. `pub`: returned by [`PairwiseReducer::finish`]. /// -/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): lets `--session` dump/ -/// restore a tally across separate CLI invocations as opaque bytes handed -/// to `obiksession::Session::store`/`restore` — `obiksession` itself knows -/// nothing about this type, it only stores/checksums/mmaps the bytes this -/// crate produces (see `obiksession`'s own crate docs on that split). +/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): kept from when this type +/// was dumped to `--session` as opaque bytes via `obiksession::Session`; +/// no active user right now (`--session` support was removed pending a +/// rebuild on top of `algorithms::scan::sampled_scan`), but cheap to keep +/// derived for when it returns. #[derive(Archive, Serialize, Deserialize)] -pub(crate) struct PairwiseTally { +pub struct PairwiseTally { n_genomes: usize, /// Upper triangle only (`i < j`), flat-indexed via [`Self::flat_index`]. pairs: Vec, @@ -314,7 +313,7 @@ pub struct CardinalityTally { /// (`PairStats` accumulates over every genome pair unconditionally; /// exclusion is a filter applied only in derived, post-hoc views). #[derive(Default, Clone, Archive, Serialize, Deserialize)] -pub(crate) struct PartitionDispersion { +pub struct PartitionDispersion { /// One entry per partition id seen so far (grows lazily — partitions /// are visited in increasing order in practice, but nothing here /// assumes that). @@ -462,6 +461,34 @@ pub(crate) fn reduce_pairwise( dispersion.record(partition, partition_subst, partition_eligible); } +/// Accumulates a [`PairwiseTally`]/[`PartitionDispersion`] pair over a +/// [`super::scan::sampled_scan`] pass — a thin +/// [`super::scan::LayerReducer`] wrapper around [`reduce_pairwise`], so it +/// can run alongside any other reducer (e.g. +/// `algorithms::alignment::AlignmentReducer`) over the exact same batches +/// in one shared scan, instead of each needing its own `sample_index` +/// call. +pub struct PairwiseReducer { + tally: PairwiseTally, + dispersion: PartitionDispersion, +} + +impl PairwiseReducer { + pub fn new(n_genomes: usize) -> Self { + Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() } + } + + pub fn finish(self) -> (PairwiseTally, PartitionDispersion) { + (self.tally, self.dispersion) + } +} + +impl super::scan::LayerReducer for PairwiseReducer { + fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { + reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion); + } +} + #[cfg(test)] mod dispersion_tests { use super::PartitionDispersion; diff --git a/src/obikphylo/src/siblings/algorithms/sankoff.rs b/src/obikphylo/src/siblings/algorithms/sankoff.rs index 1e3bdb22..c7117724 100644 --- a/src/obikphylo/src/siblings/algorithms/sankoff.rs +++ b/src/obikphylo/src/siblings/algorithms/sankoff.rs @@ -1,62 +1,36 @@ -//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` -//! pipeline: one [`sample_index`] call (one Bernoulli draw, shared by -//! everyone — the same family selection feeds the alignment and every -//! tally, never two independent samples of the same index) driving two -//! reduces over the same `on_layer` batches — [`reduce_alignment`] and -//! [`reduce_pairwise`] — so [`SnpAlignment`], [`RawSnpDistanceOutput`], -//! [`BasePairTally`] and [`CardinalityTally`] all come out of a single -//! scan. The latter three are cheap `O(n²)` post-processing over -//! [`PairwiseTally`] once the scan is done (see its own module docs for why -//! that removes the old two-pass `raw_snp_distance`-before-`base_pair_tally` -//! dependency entirely). +//! [`SankoffBundle`]: the pure post-processing glue for the +//! `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline — turns an already-run +//! [`super::scan::sampled_scan`] pass's [`AlignmentReducer`]/ +//! [`PairwiseReducer`] output into every value that pipeline needs +//! ([`SnpAlignment`], [`RawSnpDistanceOutput`], [`BasePairTally`], +//! [`CardinalityTally`]). No sampling of its own: the caller (`cmd/phylo`) +//! runs the shared scan itself, with whichever reducers a given CLI +//! invocation actually needs (alignment only, pairwise only, or both when +//! `--sankoff` and a `snp-*` `--distance` are requested together) — see +//! `algorithms::scan`'s own module docs for why that single shared call, +//! not a per-output driver, is what guarantees every output in one +//! invocation comes from the same site selection. use ndarray::Array2; -use obikidxcache::index_cache::IndexCache; -use obikindex::{OKIError, OKIResult}; -use obiksession::Session; +use obikindex::OKIResult; -use super::alignment::{SnpAlignment, reduce_alignment}; +use super::alignment::SnpAlignment; use super::pairwise::{ - BasePairTally, CardinalityTally, PairwiseTally, PartitionDispersion, RawSnpDistanceOutput, - reduce_pairwise, -}; -use super::session_cache::{ - CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint, - store_tally_checkpoint, + BasePairTally, CardinalityTally, PairwiseReducer, PairwiseTally, PartitionDispersion, + RawSnpDistanceOutput, }; use super::snp_distance::{GammaShape, SnpDistanceKind, distance_matrix}; -use super::subsample::{EntropyBias, sample_index}; -/// The alignment's own `--session` artifact — kept in lockstep with the -/// shared `pairwise_tally`/`partition_dispersion`/`progress` artifacts -/// (`session_cache`): both are folded from the *same* `on_layer` call, so -/// checkpointing them together, under the same `progress` marker, is what -/// keeps a resumed alignment's column count consistent with how many -/// layers the resumed tally actually reflects. -const ALIGNMENT_ARTIFACT: &str = "snp_alignment"; - -fn restore_alignment(session: &Session) -> Option { - let bytes = session.restore(ALIGNMENT_ARTIFACT).ok().flatten()?; - rkyv::from_bytes::(&bytes[..]).ok() -} - -fn store_alignment(session: &Session, alignment: &SnpAlignment) -> OKIResult<()> { - let bytes = rkyv::to_bytes::(alignment) - .map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?; - session.store(ALIGNMENT_ARTIFACT, &bytes).map_err(OKIError::Io) -} - -/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs, -/// computed together from one shared, possibly-subsampled/entropy-biased -/// selection — see the module docs. `pub`: part of the public signature of -/// [`crate::siblings::extensions::SiblingExt::sankoff_bundle`]. +/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline +/// needs, built together from one already-scanned +/// [`AlignmentReducer`]/[`PairwiseReducer`] pair. `pub`: part of the +/// public signature of [`crate::siblings::extensions::SiblingExt`]'s +/// sampling-based callers. /// /// `tally`/`dispersion` are kept (not just consumed into `raw`/ /// `base_pair_tally`/`cardinality_tally`) so [`Self::snp_distance`] can -/// compute a `snp-*` `--distance` matrix from this *exact* sample — see -/// that method's own docs for why reusing it, rather than a fresh -/// `SiblingExt::snp_distance` call, is a correctness requirement here, not -/// just a performance nicety. +/// compute a `snp-*` `--distance` matrix from this *exact* sample without +/// a second scan. pub struct SankoffBundle { pub alignment: SnpAlignment, pub raw: RawSnpDistanceOutput, @@ -67,124 +41,44 @@ pub struct SankoffBundle { } impl SankoffBundle { + /// Assemble a bundle from an already-finished [`SnpAlignment`] (see + /// `AlignmentReducer::finish`) plus a [`PairwiseReducer`] that both ran + /// through the same [`super::scan::sampled_scan`] call — pure + /// post-processing, `O(n²)` in genome count, never a second scan. + /// `alignment` is taken already-finished, not as an `AlignmentReducer`, + /// because a caller that *also* needs the alignment on its own (e.g. + /// `--pseudo-alignment` combined with `--sankoff` in one invocation) + /// has to read it before/alongside building this bundle — `SnpAlignment` + /// derives `Clone` for exactly that case. `ratio_ceiling` + /// (`--sankoff-ratio-ceiling`) gates [`Self::base_pair_tally`] only + /// (see [`PairwiseTally::cardinality_tally`]'s own docs for why + /// [`CardinalityTally`] uses a different — `excluded`-only — inclusion + /// rule). + pub fn finish( + alignment: SnpAlignment, + pairwise: PairwiseReducer, + ratio_ceiling: f64, + excluded: &[bool], + ) -> Self { + let (tally, dispersion) = pairwise.finish(); + let raw = tally.raw_snp_distance(); + let included = tally.included(ratio_ceiling, excluded); + let base_pair_tally = tally.base_pair_tally(&included); + let cardinality_tally = tally.cardinality_tally(excluded); + Self { alignment, raw, base_pair_tally, cardinality_tally, tally, dispersion } + } + /// A `snp-*` `--distance` matrix computed from this bundle's own - /// already-sampled tally — no new [`sample_index`] call. - /// - /// Exists specifically for the case where `--sankoff`/`--tnt`/`--phyg`/ - /// `--iqtree` and a `snp-*` `--distance` are requested in the same CLI - /// invocation: both consume exactly the same selection parameters - /// (`n`/`free_loss`/`no_ambiguity`/excluded set/`entropy_bias` — the - /// CLI has no way to give them different values in one run), so a - /// second independent `SiblingExt::snp_distance` call wouldn't just - /// waste the sampling work, it would silently sample a **different** - /// set of sites (`sample_layer`'s `rand::rng()` is a thread-local - /// generator that advances across calls, not reseeded each time) — - /// defeating the actual point of running several algorithms together, - /// which is comparing them on one identical site selection. + /// already-sampled tally — no new scan. Exists specifically for the + /// case where `--sankoff`/`--tnt`/`--phyg`/`--iqtree` and a `snp-*` + /// `--distance` are requested in the same CLI invocation: both consume + /// exactly the same selection parameters, so recomputing from a second + /// [`super::scan::sampled_scan`] call would silently draw a + /// **different** sample (`rand::rng()` is a thread-local generator + /// that advances across calls, not reseeded each time) — defeating the + /// actual point of running several algorithms together, which is + /// comparing them on one identical site selection. pub fn snp_distance(&self, kind: SnpDistanceKind, gamma_shape: GammaShape) -> OKIResult> { distance_matrix(&self.tally, &self.dispersion, kind, gamma_shape) } } - -/// See [`crate::siblings::extensions::SiblingExt::sankoff_bundle`] for the -/// public-facing docs — this is its implementation. `ratio_ceiling` gates -/// [`BasePairTally`] only (see [`PairwiseTally::cardinality_tally`]'s own -/// docs for why [`CardinalityTally`] uses a different — `excluded`-only — -/// inclusion rule). -/// -/// `session`: `--session`, `None` means no persistence (always resample). -/// `Some(session)` must already be open under the exact selection -/// parameters this call is about to use — the caller's job, same contract -/// as [`super::snp_distance::snp_distance`]'s own `session` parameter, -/// which this shares an artifact namespace with (see `session_cache`'s -/// module docs): a tally checkpointed here is restorable by a later -/// `--distance snp-*` call against the same session, and vice versa. -pub(crate) fn sankoff_bundle( - cache: &IndexCache, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ratio_ceiling: f64, - session: Option<&Session>, -) -> OKIResult { - let n_genomes = cache.meta().genomes().len(); - let genome_indices: Vec = (0..n_genomes) - .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) - .collect(); - - let restored = session.and_then(|s| Some((restore_tally_checkpoint(s)?, restore_alignment(s)?))); - let (mut tally, mut dispersion, mut sequences, skip_layers) = match restored { - Some(((tally, dispersion, skip_layers), alignment)) => { - (tally, dispersion, alignment.sequences, skip_layers) - } - None => { - let sequences = vec![Vec::new(); genome_indices.len()]; - (PairwiseTally::new(n_genomes), PartitionDispersion::default(), sequences, 0) - } - }; - - if skip_layers != usize::MAX { - // Same checkpoint cadence and rationale as `snp_distance`'s own - // resumable loop (see `session_cache::CHECKPOINT_INTERVAL*`'s own - // docs) — the alignment's `sequences` are folded in lockstep with - // the tally so a resumed run's column count always matches how - // many layers the restored tally actually reflects. - let mut last_checkpoint_layer = skip_layers; - let mut last_checkpoint_at = std::time::Instant::now(); - sample_index( - cache, - n, - free_loss, - no_ambiguity, - excluded, - entropy_bias, - skip_layers, - |partition, _layer, raw_index, survivors| { - reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences); - reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion); - let Some(session) = session else { return }; - let layers_done = raw_index + 1; - let due_by_count = - layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS; - let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL; - if !(due_by_count || due_by_time) { - return; - } - let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() }; - let stored = store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64) - .and_then(|()| store_alignment(session, &alignment)); - match stored { - Ok(()) => { - last_checkpoint_layer = layers_done; - last_checkpoint_at = std::time::Instant::now(); - } - Err(e) => { - tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}"); - } - } - }, - )?; - - if let Some(session) = session { - let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() }; - store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?; - store_alignment(session, &alignment)?; - } - } - - let raw = tally.raw_snp_distance(); - let included = tally.included(ratio_ceiling, excluded); - let base_pair_tally = tally.base_pair_tally(&included); - let cardinality_tally = tally.cardinality_tally(excluded); - - Ok(SankoffBundle { - alignment: SnpAlignment { sequences, genome_indices }, - raw, - base_pair_tally, - cardinality_tally, - tally, - dispersion, - }) -} diff --git a/src/obikphylo/src/siblings/algorithms/scan.rs b/src/obikphylo/src/siblings/algorithms/scan.rs new file mode 100644 index 00000000..99112fcc --- /dev/null +++ b/src/obikphylo/src/siblings/algorithms/scan.rs @@ -0,0 +1,70 @@ +//! The single trunk every sampled sibling-family computation runs +//! through: one [`sample_index`] pass, fanned out to whichever +//! [`LayerReducer`]s the caller actually needs. Replaces three previously +//! independent drivers (a pseudo-alignment-only one, a tally-only one, and +//! a hand-fused alignment+tally one for `--sankoff`) that each ran their +//! own `sample_index` call — meaning two of them combined in the same CLI +//! invocation used to draw two different random samples for what was +//! supposed to be one shared site selection. A caller that needs several +//! outputs from the same sample now passes several reducers to one +//! [`sampled_scan`] call instead of calling several higher-level functions +//! and hoping they agree. + +use obikidxcache::index_cache::IndexCache; +use obikindex::OKIResult; + +use super::stats::sibling_family_size_histogram; +use super::subsample::{EntropyBias, SurvivingFamily, sample_index}; + +/// One accumulator fed every layer's [`SurvivingFamily`] batch during a +/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of +/// its own), so any number of these can run side by side over the exact +/// same batches without knowing about each other. Implement this once per +/// kind of output the trunk should be able to produce (see +/// `algorithms::alignment::AlignmentReducer`, `algorithms::pairwise::PairwiseReducer`). +pub trait LayerReducer { + fn reduce(&mut self, partition: usize, layer: usize, survivors: &[SurvivingFamily]); +} + +/// `n`: `Some(target)` samples proportionally per layer, up to `target` +/// sites index-wide (`--subsample`); `None` means exhaustive — every +/// non-monomorphic minorant of the whole index, driven through the exact +/// same [`sample_index`] machinery at a quota equal to the total eligible +/// count (so every family is drawn with certainty) rather than a separate +/// full-scan code path. +fn resolve_target(cache: &IndexCache, n: Option) -> OKIResult { + match n { + Some(target) => Ok(target), + None => { + let counts = sibling_family_size_histogram(cache)?; + Ok((counts[1] + counts[2] + counts[3]) as usize) + } + } +} + +/// Run one shared, possibly-subsampled/entropy-biased selection over +/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see +/// [`sample_index`]'s own docs for what each controls), feeding every +/// surviving family of every visited layer to every reducer in +/// `reducers`, in the same order, before moving to the next layer. A +/// no-op (not an error) if the resolved target is `0`. +pub fn sampled_scan( + cache: &IndexCache, + n: Option, + free_loss: bool, + no_ambiguity: bool, + excluded: &[bool], + entropy_bias: Option, + reducers: &mut [&mut dyn LayerReducer], +) -> OKIResult<()> { + let target = resolve_target(cache, n)?; + if target == 0 { + return Ok(()); + } + sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| { + for reducer in reducers.iter_mut() { + reducer.reduce(partition, layer, &survivors); + } + })?; + Ok(()) +} diff --git a/src/obikphylo/src/siblings/algorithms/session_cache.rs b/src/obikphylo/src/siblings/algorithms/session_cache.rs deleted file mode 100644 index a4ae6709..00000000 --- a/src/obikphylo/src/siblings/algorithms/session_cache.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Shared `--session` checkpoint plumbing for [`super::snp_distance`] and -//! [`super::sankoff`] — both fold [`super::subsample::sample_index`]'s -//! layer-by-layer output into a [`PairwiseTally`]/[`PartitionDispersion`] -//! pair, and both persist it under the *same* artifact names on purpose: -//! a tally checkpointed by one is restorable by the other. Concretely, -//! `--sankoff --session DIR` today followed by `--distance snp-k2p -//! --session DIR` tomorrow restores the first run's tally instead of -//! resampling, even though only `--sankoff` originally built it — -//! `snp_distance`'s own selection-parameter conflict check (via -//! `obiksession::Session::open`, the caller's responsibility) is what -//! guarantees that's actually safe to do. - -use obikindex::{OKIError, OKIResult}; -use obiksession::Session; - -use super::pairwise::{PairwiseTally, PartitionDispersion}; - -pub(crate) const TALLY_ARTIFACT: &str = "pairwise_tally"; -pub(crate) const DISPERSION_ARTIFACT: &str = "partition_dispersion"; -pub(crate) const PROGRESS_ARTIFACT: &str = "progress"; -/// Sentinel `progress` value meaning "every layer was processed — this -/// tally is complete," distinct from any real layer count so a resumed run -/// can tell "finished" apart from "stopped after N layers" without relying -/// on N happening to equal the index's actual layer count. -pub(crate) const PROGRESS_COMPLETE: u64 = u64::MAX; - -/// How much in-progress sampling work a checkpoint can lose: at most -/// `CHECKPOINT_INTERVAL_LAYERS` layers *or* `CHECKPOINT_INTERVAL` of wall -/// time, whichever comes first — not "checkpoint every layer" (rewriting -/// the whole, potentially large, tally/dispersion after every single one of -/// what can be thousands of layers would make the checkpointing overhead -/// itself dominate), and not "only at the end" either (which would defeat -/// the point: on a long run against a real production index, losing 90% of -/// an already-completed scan to a crash is exactly the expensive case -/// resumability exists to avoid — the actual justification, not "the tally -/// is cheap to rebuild," which was an earlier, wrong framing of this -/// tradeoff caught on review). -pub(crate) const CHECKPOINT_INTERVAL_LAYERS: usize = 8; -pub(crate) const CHECKPOINT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30); - -/// Restore a checkpointed `(PairwiseTally, PartitionDispersion, resume-from -/// layer index)` triple from `session` — `None` on any miss (an artifact -/// not cached, an I/O error, or a deserialization failure), all treated -/// identically: something's missing or unusable, fall back to starting -/// from scratch (`skip_layers = 0`) rather than surfacing a hard error over -/// what's meant to be a transparent cache. The resume index is -/// [`PROGRESS_COMPLETE`] when a prior run finished every layer. Layers -/// processed after a resume draw a fresh, independent random sample (see -/// `subsample::sample_index`'s own `skip_layers` docs) — deliberately not -/// bit-identical to what an uninterrupted run would have produced, -/// consistent with wanting run-to-run sampling variance rather than fixed -/// reproducibility. -/// -/// Full owned deserialization (`rkyv::from_bytes`) for the tally/dispersion, -/// not zero-copy (`rkyv::access`) — simpler to integrate with -/// `PairwiseTally`'s existing query methods, which all take `&self` by -/// value semantics, not an `Archived`. Still skips the -/// expensive part (`sample_index` re-scanning the sibling annex for every -/// already-completed layer); the deserialize of an already-in-memory-sized -/// byte buffer is comparatively cheap. Making this genuinely zero-copy -/// later would mean every `PairwiseTally` accessor (`categories`, -/// `base_freq`, ...) working generically over `Archived` -/// too — a bigger, separate change. -pub(crate) fn restore_tally_checkpoint(session: &Session) -> Option<(PairwiseTally, PartitionDispersion, usize)> { - let tally_bytes = session.restore(TALLY_ARTIFACT).ok().flatten()?; - let dispersion_bytes = session.restore(DISPERSION_ARTIFACT).ok().flatten()?; - let progress_bytes = session.restore(PROGRESS_ARTIFACT).ok().flatten()?; - let tally = rkyv::from_bytes::(&tally_bytes[..]).ok()?; - let dispersion = - rkyv::from_bytes::(&dispersion_bytes[..]).ok()?; - let progress_raw: [u8; 8] = progress_bytes.get(..8)?.try_into().ok()?; - let progress = u64::from_le_bytes(progress_raw); - let skip_layers = if progress == PROGRESS_COMPLETE { usize::MAX } else { progress as usize }; - if skip_layers == usize::MAX { - tracing::info!("--session: restored a complete cached SNP tally — skipping resampling"); - } else { - tracing::info!("--session: resuming a cached SNP tally from layer {skip_layers} onward"); - } - Some((tally, dispersion, skip_layers)) -} - -/// Store `tally`/`dispersion` plus how far sampling has gotten -/// (`layers_done`, or [`PROGRESS_COMPLETE`] once every layer has been -/// processed) — a checkpoint [`restore_tally_checkpoint`] can resume from -/// later, whether that "later" is after a crash or a clean, separate -/// invocation. -pub(crate) fn store_tally_checkpoint( - session: &Session, - tally: &PairwiseTally, - dispersion: &PartitionDispersion, - progress: u64, -) -> OKIResult<()> { - let tally_bytes = rkyv::to_bytes::(tally) - .map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?; - session.store(TALLY_ARTIFACT, &tally_bytes).map_err(OKIError::Io)?; - let dispersion_bytes = rkyv::to_bytes::(dispersion) - .map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?; - session.store(DISPERSION_ARTIFACT, &dispersion_bytes).map_err(OKIError::Io)?; - session.store(PROGRESS_ARTIFACT, &progress.to_le_bytes()).map_err(OKIError::Io)?; - Ok(()) -} diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 1460fa32..8a6528e4 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -22,21 +22,13 @@ //! change. use ndarray::Array2; -use obikidxcache::index_cache::IndexCache; use obikindex::OKIResult; -use obiksession::Session; use super::pairwise::{PairwiseTally, PartitionDispersion}; -use super::session_cache::{ - CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint, - store_tally_checkpoint, -}; -use super::sibling_family_size_histogram; -use super::subsample::{EntropyBias, sample_index}; -/// One `snp-*` `--distance` value. `pub`: part of -/// [`crate::siblings::extensions::SiblingExt::snp_distance`]'s public -/// signature. +/// One `snp-*` `--distance` value. `pub`: part of the public signature of +/// [`crate::siblings::extensions::SiblingExt`]'s distance-computing +/// callers. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SnpDistanceKind { /// Uncorrected p-distance (`snp / (snp + shared)`). @@ -213,140 +205,18 @@ fn formula(kind: SnpDistanceKind) -> fn(&PairwiseTally, usize, usize, Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - gamma_shape: GammaShape, - session: Option<&Session>, -) -> OKIResult> { - // Fail fast, before paying for `sample_index`, not just inside - // `distance_matrix` (which runs after sampling either way — the right - // place for the check when called from `SankoffBundle::snp_distance`, - // where sampling already happened as part of `--sankoff` itself). - if gamma_shape != GammaShape::Disabled && !kind.supports_gamma() { - return Err(obikindex::OKIError::InvalidInput( - "--gamma-shape has no effect on this --distance value".into(), - )); - } - - let (mut tally, mut dispersion, skip_layers) = match session.and_then(restore_tally_checkpoint) { - Some(triple) => triple, - None => { - let n_genomes = cache.meta().genomes().len(); - (PairwiseTally::new(n_genomes), PartitionDispersion::default(), 0) - } - }; - - if skip_layers != usize::MAX { - let (target, entropy_bias) = match n { - Some(target) => (target, entropy_bias), - None => { - let counts = sibling_family_size_histogram(cache)?; - let total_eligible = (counts[1] + counts[2] + counts[3]) as usize; - (total_eligible, None) - } - }; - - if target > 0 { - // Checkpoint every `CHECKPOINT_INTERVAL_LAYERS` layers or - // `CHECKPOINT_INTERVAL` of wall time, whichever comes first — - // see those constants' own docs for why "just recompute - // everything" isn't the right call to make on a long run - // against a real production index. - let mut last_checkpoint_layer = skip_layers; - let mut last_checkpoint_at = std::time::Instant::now(); - sample_index( - cache, - target, - free_loss, - no_ambiguity, - excluded, - entropy_bias, - skip_layers, - |partition, _layer, raw_index, survivors| { - super::pairwise::reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion); - let Some(session) = session else { return }; - let layers_done = raw_index + 1; - let due_by_count = - layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS; - let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL; - if !(due_by_count || due_by_time) { - return; - } - match store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64) { - Ok(()) => { - last_checkpoint_layer = layers_done; - last_checkpoint_at = std::time::Instant::now(); - } - Err(e) => { - tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}"); - } - } - }, - )?; - } - - if let Some(session) = session { - store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?; - } - } - - distance_matrix(&tally, &dispersion, kind, gamma_shape) -} - -/// The post-sampling half of [`snp_distance`] — everything downstream of -/// an already-built [`PairwiseTally`]/[`PartitionDispersion`]: the -/// `--gamma-shape` support check, `alpha` resolution (fixed/auto/disabled), -/// and the final `n×n` matrix build. Factored out so -/// [`super::sankoff::SankoffBundle::snp_distance`] can reuse a tally it -/// already has (from `--sankoff`/`--tnt`/`--phyg`/`--iqtree` sharing the -/// exact same selection parameters as a `snp-*` `--distance` in the same -/// CLI invocation) instead of re-running [`sample_index`] — which, beyond -/// the wasted work, would also draw a **different** random sample the -/// second time (`rand::rng()` is a thread-local generator that advances -/// across calls, never reseeded per call), silently breaking the -/// "same site selection for every algorithm in this run" guarantee that's -/// the actual point of combining these flags in one invocation. -pub(crate) fn distance_matrix( +/// Everything downstream of an already-built +/// [`PairwiseTally`]/[`PartitionDispersion`]: the `--gamma-shape` support +/// check, `alpha` resolution (fixed/auto/disabled), and the final `n×n` +/// matrix build — `O(n²)` post-processing, never a scan of its own. The +/// tally/dispersion themselves come from a +/// [`super::pairwise::PairwiseReducer`] that already ran through +/// [`super::scan::sampled_scan`] — see [`super::sankoff::SankoffBundle::snp_distance`] +/// for why reusing an already-scanned reducer, rather than a second +/// [`sampled_scan`](super::scan::sampled_scan) call, is a correctness +/// requirement (not just a performance nicety) whenever `--sankoff` and a +/// `snp-*` `--distance` are requested together. +pub fn distance_matrix( tally: &PairwiseTally, dispersion: &PartitionDispersion, kind: SnpDistanceKind, diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index e56b6c99..c80bf521 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -9,16 +9,13 @@ use std::io::{BufWriter, Write}; use std::path::Path; -use ndarray::Array2; use obikidxcache::index_cache::IndexCache; use obikindex::{OKIError, OKIResult}; use obisys::progress_bar; use crate::siblings::algorithms::{ - EntropyBias, GammaShape, SankoffBundle, Selection, SiblingAnnexStats, SnpAlignment, - SnpDistanceKind, build_layer_sibling_annex, family_entropy, family_entropy_4, is_fast_mode, - sankoff_bundle, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram, - snp_distance, snp_pseudo_alignment, + Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4, + is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram, }; use crate::siblings::extensions::SiblingBuilder; use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap}; @@ -58,28 +55,6 @@ pub trait SiblingExt { /// `--subsample`/`--entropy-bias` yet (see the project memory on this). fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>; - /// Build a SNP-only pseudo-alignment (`algorithms::subsample::sample_index` - /// — proportional per-layer sampling, `n` sites index-wide, target not - /// guarantee) from an already-built sibling annex (run - /// [`build_sibling_annex`](Self::build_sibling_annex) first). `free_loss` - /// (`--free-loss`), `no_ambiguity` (`--no-ambiguity`) and `excluded` - /// (`--exclude-genome`, `excluded[g]` true to drop genome `g`) all feed - /// straight into the sampling itself, not a later filter — see - /// `algorithms::masking`'s docs: a family whose polymorphism only - /// existed via states/genomes these exclude is caught and discarded - /// during sampling, not left in the output as a false site. Every - /// column of the result has already survived that check, and an - /// excluded genome has no row at all in the result (see - /// `SnpAlignment::genome_indices`). - fn snp_pseudo_alignment( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ) -> OKIResult; - /// The global family-size histogram alone (`SiblingAnnexStats::counts`, /// no `per_genome`) — reads only the already-built annex /// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits, @@ -97,84 +72,6 @@ pub trait SiblingExt { /// instead when only the global histogram is needed. fn sibling_annex_stats(&self) -> OKIResult; - /// Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` - /// pipeline: one shared, possibly-subsampled/entropy-biased selection - /// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same - /// meaning as [`snp_pseudo_alignment`](Self::snp_pseudo_alignment)'s - /// own) drives a single scan producing the pseudo-alignment *and* every - /// tally the Sankoff cost-matrix calibration - /// (`crate::siblings::cardinality_transition_probs`/ - /// `composition_transition_probs`/`pairwise_cost_matrix`) needs — - /// never two independent draws of the same index. `ratio_ceiling` - /// (`--sankoff-ratio-ceiling`) excludes genome pairs too close to - /// substitution saturation from `SankoffBundle::base_pair_tally`'s - /// pool (a saturated pair's base composition is noise, not signal) — - /// `SankoffBundle::cardinality_tally` is *not* gated by it (see - /// `crate::siblings::CardinalityTally`'s own docs for why that - /// wouldn't make sense: cardinality reflects each genome's own - /// coverage/duplication structure, not the pair's mutual divergence). - /// `session`: `--session`, `None` disables persistence (always - /// resample). `Some` must already be open under these exact selection - /// parameters (the caller's responsibility) — restores a cached tally/ - /// alignment when present instead of resampling, and checkpoints a - /// fresh one back as sampling progresses otherwise. Shares its - /// artifact namespace with [`Self::snp_distance`]'s own `session` - /// parameter internally: a tally checkpointed by one is restorable by - /// the other, given the same session and selection parameters. - fn sankoff_bundle( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ratio_ceiling: f64, - session: Option<&obiksession::Session>, - ) -> OKIResult; - - /// A `snp-*` `--distance` matrix — one of the closed-form corrections - /// in [`SnpDistanceKind`], over the central-position SNP substitution - /// spectrum (requires an already-built sibling annex, run - /// [`build_sibling_annex`](Self::build_sibling_annex) first). - /// - /// `n`: `Some(target)` samples proportionally per layer exactly like - /// [`sankoff_bundle`](Self::sankoff_bundle)/ - /// [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) (`--subsample`); - /// `None` means exhaustive — every non-monomorphic minorant of the - /// whole index, not an approximation (see - /// `algorithms::snp_distance::snp_distance`'s own docs for how this - /// reuses the same sampling machinery at `p = 1` rather than a second - /// driver). `free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same - /// meaning as `snp_pseudo_alignment`'s own (the last forced to `None` - /// when `n` is `None`: entropy-biasing only makes sense for a genuine - /// subsample). - /// - /// `gamma_shape`: `--gamma-shape`, the Jin-Nei rate-heterogeneity - /// correction — [`GammaShape::Disabled`] turns it off, - /// [`GammaShape::Fixed`]/[`GammaShape::Auto`] are rejected with - /// [`OKIError::InvalidInput`] for a `kind` that doesn't support it - /// ([`SnpDistanceKind::supports_gamma`]). `Auto` estimates `alpha` from - /// the data itself (method-of-moments over per-partition substitution - /// rate dispersion) rather than requiring a user-supplied value. - /// - /// `session`: `--session`, `None` disables persistence (always - /// resample). `Some` must already be open under these exact selection - /// parameters (the caller's responsibility, via - /// `obiksession::Session::open`'s params-conflict check) — restores a - /// cached tally when present instead of resampling, and stores a - /// freshly computed one back for next time otherwise. - fn snp_distance( - &self, - kind: SnpDistanceKind, - n: Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - gamma_shape: GammaShape, - session: Option<&obiksession::Session>, - ) -> OKIResult>; - /// The Family Overlap annex — number of shared *variable* families per /// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs). /// Built (alongside the entropy annex) if missing, same as every other @@ -183,8 +80,8 @@ pub trait SiblingExt { /// (`FamilyOverlap::mean_row(g) < threshold` — same statistic /// `--family-overlap`'s matrix shows, just averaged per row — /// triggers adding `g` to the excluded-genome set consumed by - /// `snp_pseudo_alignment`/`sankoff_bundle`/`snp_distance`; this needs - /// no computation of its own beyond that check). + /// `algorithms::scan::sampled_scan`'s callers; this needs no + /// computation of its own beyond that check). fn family_overlap(&self) -> OKIResult; } @@ -312,17 +209,6 @@ impl SiblingExt for IndexCache { Ok(()) } - fn snp_pseudo_alignment( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ) -> OKIResult { - snp_pseudo_alignment(self, n, free_loss, no_ambiguity, excluded, entropy_bias) - } - fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> { sibling_family_size_histogram(self) } @@ -331,33 +217,6 @@ impl SiblingExt for IndexCache { sibling_annex_stats(self) } - fn sankoff_bundle( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ratio_ceiling: f64, - session: Option<&obiksession::Session>, - ) -> OKIResult { - sankoff_bundle(self, n, free_loss, no_ambiguity, excluded, entropy_bias, ratio_ceiling, session) - } - - fn snp_distance( - &self, - kind: SnpDistanceKind, - n: Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - gamma_shape: GammaShape, - session: Option<&obiksession::Session>, - ) -> OKIResult> { - snp_distance(self, kind, n, free_loss, no_ambiguity, excluded, entropy_bias, gamma_shape, session) - } - fn family_overlap(&self) -> OKIResult { self.ensure_entropy_and_overlap_annexes()?; let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME); diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index 104a4636..c15d0901 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -37,9 +37,10 @@ pub use iter::{ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use algorithms::{ - BasePairTally, CardinalityTally, EntropyBias, GammaShape, RawSnpDistanceOutput, SankoffBundle, - SiblingAnnexStats, SnpAlignment, SnpDistanceKind, cardinality_transition_probs, - composition_transition_probs, pairwise_cost_matrix, + AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer, + PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment, + SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix, + pairwise_cost_matrix, sampled_scan, }; pub use extensions::SiblingExt;