Refactor algorithms for unified sampling and reduction
Introduces a unified sampling trunk, new reducer traits, and refined logic for SNP distance calculation and session handling. Enforces strict parameter sharing across all algorithms to ensure consistent sampling.
This commit is contained in:
1 parent
0d1ecac233
commit
1702addbf9
12 files changed
+320
-760
No files matched your search
Generated
-1
@@ -1594,7 +1594,6 @@ dependencies = [
|
|||||||
"obikindex",
|
"obikindex",
|
||||||
"obikindexer",
|
"obikindexer",
|
||||||
"obikseq",
|
"obikseq",
|
||||||
"obiksession",
|
|
||||||
"obipipeline",
|
"obipipeline",
|
||||||
"obiread",
|
"obiread",
|
||||||
"obiskbuilder",
|
"obiskbuilder",
|
||||||
|
|||||||
@@ -24,43 +24,6 @@ use phylip::write_phylip_relaxed;
|
|||||||
use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
|
use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
|
||||||
use tnt::write_sankoff_tnt;
|
use tnt::write_sankoff_tnt;
|
||||||
|
|
||||||
/// Deterministic byte encoding of the `snp-*` `--distance` selection
|
|
||||||
/// parameters, for `obiksession::Session::open`'s params-conflict check.
|
|
||||||
/// `obiksession` treats this as an opaque blob (see its own crate docs) —
|
|
||||||
/// only this module needs to know the layout, and only well enough to
|
|
||||||
/// produce/compare it, never to pretty-print a per-field diff (v1 keeps
|
|
||||||
/// the conflict message generic rather than decoding both sides field by
|
|
||||||
/// field — see its use in `run` below).
|
|
||||||
fn session_params_blob(
|
|
||||||
n: Option<usize>,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
) -> Vec<u8> {
|
|
||||||
let mut buf = Vec::new();
|
|
||||||
match n {
|
|
||||||
Some(v) => {
|
|
||||||
buf.push(1);
|
|
||||||
buf.extend_from_slice(&(v as u64).to_le_bytes());
|
|
||||||
}
|
|
||||||
None => buf.push(0),
|
|
||||||
}
|
|
||||||
buf.push(free_loss as u8);
|
|
||||||
buf.push(no_ambiguity as u8);
|
|
||||||
buf.extend_from_slice(&(excluded.len() as u64).to_le_bytes());
|
|
||||||
buf.extend(excluded.iter().map(|&b| b as u8));
|
|
||||||
match entropy_bias {
|
|
||||||
Some(EntropyBias { mu, sigma }) => {
|
|
||||||
buf.push(1);
|
|
||||||
buf.extend_from_slice(&mu.to_le_bytes());
|
|
||||||
buf.extend_from_slice(&sigma.to_le_bytes());
|
|
||||||
}
|
|
||||||
None => buf.push(0),
|
|
||||||
}
|
|
||||||
buf
|
|
||||||
}
|
|
||||||
|
|
||||||
pub use args::PhyloArgs;
|
pub use args::PhyloArgs;
|
||||||
|
|
||||||
pub fn run(args: PhyloArgs) {
|
pub fn run(args: PhyloArgs) {
|
||||||
@@ -273,63 +236,83 @@ pub fn run(args: PhyloArgs) {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── `--session`: persist the snp-* sample across separate invocations ──
|
// ── `--session` ──────────────────────────────────────────────────────────
|
||||||
// Only covers `snp-*` `--distance` today (see `--session`'s own docs);
|
// Temporarily unavailable: the per-driver checkpoint/resume plumbing
|
||||||
// `None` when `--session` isn't given, matching today's always-resample
|
// it relied on (`algorithms::session_cache`) was removed along with
|
||||||
// default exactly.
|
// the three independent sampling drivers it patched over (see
|
||||||
let session = args.session.as_ref().map(|dir| {
|
// `algorithms::scan`'s own module docs) — persistence needs to be
|
||||||
let params = session_params_blob(
|
// rebuilt on top of the new single `sampled_scan` trunk instead of
|
||||||
args.subsample,
|
// silently becoming a no-op.
|
||||||
args.free_loss,
|
if args.session.is_some() {
|
||||||
args.no_ambiguity,
|
eprintln!(
|
||||||
&snp_exclude_mask,
|
"error: --session is temporarily unavailable (being rebuilt on top of the unified \
|
||||||
entropy_bias,
|
sampling trunk) — rerun without --session/--session-force"
|
||||||
);
|
);
|
||||||
match obiksession::Session::open(dir, ¶ms, args.session_force) {
|
std::process::exit(1);
|
||||||
Ok(Ok(session)) => {
|
}
|
||||||
match session.outcome() {
|
|
||||||
obiksession::SessionOutcome::Created => {
|
// ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ──
|
||||||
info!("--session {}: starting fresh", dir.display());
|
// All three consume exactly the same selection parameters
|
||||||
}
|
// (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/
|
||||||
obiksession::SessionOutcome::Reused => {
|
// `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from
|
||||||
info!("--session {}: found, will reuse its cached sample if present", dir.display());
|
// the *same* random sample when combined in one invocation — so
|
||||||
}
|
// whichever of them are requested here share exactly one
|
||||||
}
|
// `sampled_scan` call, each contributing only the reducer(s) it needs,
|
||||||
session
|
// instead of each running its own scan (which used to silently draw a
|
||||||
}
|
// different sample per flag — see `obikphylo::siblings::LayerReducer`'s
|
||||||
Ok(Err(_conflict)) => {
|
// own module docs).
|
||||||
eprintln!(
|
let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree;
|
||||||
"error: --session {} was built with different selection parameters \
|
let snp_kind = args.distance.as_snp();
|
||||||
(--subsample/--free-loss/--no-ambiguity/--exclude-genome/--min-shared-family/\
|
let need_alignment = args.pseudo_alignment || sankoff_requested;
|
||||||
--entropy/--entropy-sd) than this run — pass --session-force to discard its \
|
let need_pairwise = sankoff_requested || snp_kind.is_some();
|
||||||
cached sample and start fresh, or point --session at a different directory",
|
|
||||||
dir.display()
|
if args.pseudo_alignment && args.subsample.is_none() {
|
||||||
);
|
eprintln!("error: --pseudo-alignment requires --subsample <N>");
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
Err(e) => {
|
if sankoff_requested && args.subsample.is_none() {
|
||||||
eprintln!("error opening --session {}: {e}", dir.display());
|
eprintln!("error: --sankoff requires --subsample <N>");
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
let mut alignment_reducer = need_alignment.then(|| {
|
||||||
|
obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity)
|
||||||
});
|
});
|
||||||
|
let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n));
|
||||||
|
|
||||||
// ── SNP pseudo-alignment (`--pseudo-alignment`) ─────────────────────────────
|
if need_alignment || need_pairwise {
|
||||||
if args.pseudo_alignment {
|
let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new();
|
||||||
let Some(subsample_n) = args.subsample else {
|
if let Some(r) = alignment_reducer.as_mut() {
|
||||||
eprintln!("error: --pseudo-alignment requires --subsample <N>");
|
reducers.push(r);
|
||||||
std::process::exit(1);
|
}
|
||||||
};
|
if let Some(r) = pairwise_reducer.as_mut() {
|
||||||
|
reducers.push(r);
|
||||||
|
}
|
||||||
|
|
||||||
info!("sampling SNP pseudo-alignment (target {subsample_n} site(s))");
|
info!(
|
||||||
let t = Stage::start("pseudo_alignment");
|
"sampling SNP sites{}",
|
||||||
let alignment = cache
|
match args.subsample {
|
||||||
.snp_pseudo_alignment(subsample_n, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias)
|
Some(n) => format!(" (target {n} site(s))"),
|
||||||
|
None => " (exhaustive)".into(),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let t = Stage::start("sampled_scan");
|
||||||
|
obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers)
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
eprintln!("error building pseudo-alignment: {e}");
|
eprintln!("error sampling SNP sites: {e}");
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
});
|
});
|
||||||
rep.push(t.stop());
|
rep.push(t.stop());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finished once here (not left as reducers) so `--pseudo-alignment`
|
||||||
|
// and the Sankoff group can each read it below without fighting over
|
||||||
|
// which one consumes it — see `SankoffBundle::finish`'s own docs.
|
||||||
|
let alignment = alignment_reducer.map(obikphylo::siblings::AlignmentReducer::finish);
|
||||||
|
|
||||||
|
// ── SNP pseudo-alignment (`--pseudo-alignment`) ─────────────────────────────
|
||||||
|
if args.pseudo_alignment {
|
||||||
|
let alignment = alignment.as_ref().expect("scanned above: need_alignment");
|
||||||
|
|
||||||
let path = args.output.as_ref()
|
let path = args.output.as_ref()
|
||||||
.map(|p| format!("{}_alignment.fasta", p.display()))
|
.map(|p| format!("{}_alignment.fasta", p.display()))
|
||||||
@@ -350,33 +333,17 @@ pub fn run(args: PhyloArgs) {
|
|||||||
// `snp-*` `--distance` branch below can reuse its already-sampled
|
// `snp-*` `--distance` branch below can reuse its already-sampled
|
||||||
// tally instead of resampling — see `SankoffBundle::snp_distance`'s own
|
// tally instead of resampling — see `SankoffBundle::snp_distance`'s own
|
||||||
// docs for why that's required for correctness, not just speed, when
|
// docs for why that's required for correctness, not just speed, when
|
||||||
// both are requested in the same invocation (they always share the
|
// both are requested in the same invocation.
|
||||||
// same `n`/`free_loss`/`no_ambiguity`/excluded-set/`entropy_bias`: the
|
|
||||||
// CLI has no way to give them different values in one run).
|
|
||||||
let mut bundle = None;
|
let mut bundle = None;
|
||||||
if args.sankoff || args.tnt || args.phyg || args.iqtree {
|
if sankoff_requested {
|
||||||
let Some(subsample_n) = args.subsample else {
|
let alignment = alignment.clone().expect("scanned above: need_alignment");
|
||||||
eprintln!("error: --sankoff requires --subsample <N>");
|
let pairwise = pairwise_reducer.take().expect("scanned above: need_pairwise");
|
||||||
std::process::exit(1);
|
let b = obikphylo::siblings::SankoffBundle::finish(
|
||||||
};
|
alignment,
|
||||||
|
pairwise,
|
||||||
info!("sampling Sankoff calibration bundle (target {subsample_n} site(s))");
|
args.sankoff_ratio_ceiling,
|
||||||
let t = Stage::start("sankoff_bundle");
|
&snp_exclude_mask,
|
||||||
let b = cache
|
);
|
||||||
.sankoff_bundle(
|
|
||||||
subsample_n,
|
|
||||||
args.free_loss,
|
|
||||||
args.no_ambiguity,
|
|
||||||
&snp_exclude_mask,
|
|
||||||
entropy_bias,
|
|
||||||
args.sankoff_ratio_ceiling,
|
|
||||||
session.as_ref(),
|
|
||||||
)
|
|
||||||
.unwrap_or_else(|e| {
|
|
||||||
eprintln!("error computing Sankoff calibration bundle: {e}");
|
|
||||||
std::process::exit(1);
|
|
||||||
});
|
|
||||||
rep.push(t.stop());
|
|
||||||
|
|
||||||
let p_card = cardinality_transition_probs(&b.cardinality_tally);
|
let p_card = cardinality_transition_probs(&b.cardinality_tally);
|
||||||
let p_comp = composition_transition_probs(&b.base_pair_tally);
|
let p_comp = composition_transition_probs(&b.base_pair_tally);
|
||||||
@@ -441,7 +408,7 @@ pub fn run(args: PhyloArgs) {
|
|||||||
eprintln!("error: --shared-kmers has no meaning for a snp-* --distance value");
|
eprintln!("error: --shared-kmers has no meaning for a snp-* --distance value");
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
let kind = args.distance.as_snp().expect("DistanceArg is always classic or snp");
|
let kind = snp_kind.expect("DistanceArg is always classic or snp");
|
||||||
let gamma_shape = args.gamma_shape.map_or(GammaShape::Disabled, Into::into);
|
let gamma_shape = args.gamma_shape.map_or(GammaShape::Disabled, Into::into);
|
||||||
let t = Stage::start("snp_distance");
|
let t = Stage::start("snp_distance");
|
||||||
let matrix = match &bundle {
|
let matrix = match &bundle {
|
||||||
@@ -453,23 +420,9 @@ pub fn run(args: PhyloArgs) {
|
|||||||
b.snp_distance(kind, gamma_shape)
|
b.snp_distance(kind, gamma_shape)
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
info!(
|
info!("computing {kind:?} SNP distance for {n} genome(s) (reusing the sampled scan above)");
|
||||||
"computing {kind:?} SNP distance for {n} genome(s){}",
|
let (tally, dispersion) = pairwise_reducer.take().expect("scanned above: need_pairwise").finish();
|
||||||
match args.subsample {
|
obikphylo::siblings::distance_matrix(&tally, &dispersion, kind, gamma_shape)
|
||||||
Some(n) => format!(" (subsampled, target {n} site(s))"),
|
|
||||||
None => " (exhaustive)".into(),
|
|
||||||
}
|
|
||||||
);
|
|
||||||
cache.snp_distance(
|
|
||||||
kind,
|
|
||||||
args.subsample,
|
|
||||||
args.free_loss,
|
|
||||||
args.no_ambiguity,
|
|
||||||
&snp_exclude_mask,
|
|
||||||
entropy_bias,
|
|
||||||
gamma_shape,
|
|
||||||
session.as_ref(),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ obicompactvec = { path = "../obicompactvec" }
|
|||||||
obikidxcache = { path = "../obikidxcache" }
|
obikidxcache = { path = "../obikidxcache" }
|
||||||
obiskbuilder = { path = "../obiskbuilder" }
|
obiskbuilder = { path = "../obiskbuilder" }
|
||||||
obipipeline = { path = "../obipipeline" }
|
obipipeline = { path = "../obipipeline" }
|
||||||
obiksession = { path = "../obiksession" }
|
|
||||||
kodama = "0.3.0"
|
kodama = "0.3.0"
|
||||||
memmap2 = "0.9"
|
memmap2 = "0.9"
|
||||||
ndarray = "0.17"
|
ndarray = "0.17"
|
||||||
|
|||||||
@@ -1,27 +1,25 @@
|
|||||||
//! SNP-only pseudo-alignment: one row (IUPAC-coded) per genome, one column
|
//! SNP-only pseudo-alignment: one row (IUPAC-coded) per genome, one column
|
||||||
//! per sampled family — built directly on [`sample_index`], so every family
|
//! per sampled family — built as a [`LayerReducer`] over
|
||||||
//! this produces has already survived both its Bernoulli draw and post-hoc
|
//! [`super::scan::sampled_scan`]'s per-layer batches, so every family this
|
||||||
|
//! produces has already survived both its Bernoulli draw and post-hoc
|
||||||
//! masking ([`super::masking::survives_masking`]).
|
//! masking ([`super::masking::survives_masking`]).
|
||||||
|
|
||||||
use obikidxcache::index_cache::IndexCache;
|
use obikidxcache::index_cache::IndexCache;
|
||||||
use obikindex::OKIResult;
|
|
||||||
use rkyv::{Archive, Deserialize, Serialize};
|
use rkyv::{Archive, Deserialize, Serialize};
|
||||||
|
|
||||||
use super::masking::{iupac_code, masked_state};
|
use super::masking::{iupac_code, masked_state};
|
||||||
use super::subsample::{EntropyBias, SurvivingFamily, sample_index};
|
use super::scan::LayerReducer;
|
||||||
|
use super::subsample::SurvivingFamily;
|
||||||
|
|
||||||
/// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the
|
/// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the
|
||||||
/// same length (one byte per sampled family, in the order [`sample_index`]
|
/// same length (one byte per sampled family, in the order
|
||||||
/// produced them: layer by layer, `family_idx` order within each). An
|
/// [`super::scan::sampled_scan`] produced them: layer by layer,
|
||||||
/// excluded genome (`excluded[g]` in
|
/// `family_idx` order within each). An excluded genome (`excluded[g]` in
|
||||||
/// [`snp_pseudo_alignment`]) has no row at all, not a blanked one — its
|
/// [`AlignmentReducer::new`]) has no row at all, not a blanked one — its
|
||||||
/// index simply doesn't appear in `genome_indices`. `pub`, not
|
/// index simply doesn't appear in `genome_indices`. `pub`, not
|
||||||
/// `pub(crate)`: part of the public signature of
|
/// `pub(crate)`: part of the public signature of
|
||||||
/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`].
|
/// [`crate::siblings::extensions::SiblingExt`]'s sampling-based methods.
|
||||||
///
|
#[derive(Clone, Archive, Serialize, Deserialize)]
|
||||||
/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): `--session` persistence
|
|
||||||
/// for `SankoffBundle` — same reasoning as `PairwiseTally`'s own derive.
|
|
||||||
#[derive(Archive, Serialize, Deserialize)]
|
|
||||||
pub struct SnpAlignment {
|
pub struct SnpAlignment {
|
||||||
pub sequences: Vec<Vec<u8>>,
|
pub sequences: Vec<Vec<u8>>,
|
||||||
/// This index's own genome numbering (matching `IndexCache::meta().genomes()`'s
|
/// This index's own genome numbering (matching `IndexCache::meta().genomes()`'s
|
||||||
@@ -31,50 +29,43 @@ pub struct SnpAlignment {
|
|||||||
pub genome_indices: Vec<usize>,
|
pub genome_indices: Vec<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// See [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`]
|
/// Accumulates a [`SnpAlignment`] over a [`super::scan::sampled_scan`]
|
||||||
/// for the public-facing docs — this is its implementation. A single
|
/// pass — a thin [`LayerReducer`] wrapper around [`reduce_alignment`], so
|
||||||
/// "reduce" ([`reduce_alignment`]) over [`sample_index`]'s per-layer
|
/// it can run alongside any other reducer (e.g.
|
||||||
/// batches — the only consumer today, but structured so a future sibling
|
/// `algorithms::pairwise::PairwiseReducer`) over the exact same batches in
|
||||||
/// reduce (e.g. the Sankoff cost-matrix calibration's cardinality/
|
/// one shared scan, instead of each needing its own `sample_index` call.
|
||||||
/// composition tallies) can run over the exact same batches, each cloning
|
pub struct AlignmentReducer {
|
||||||
/// its own `Arc`, without this function changing at all.
|
genome_indices: Vec<usize>,
|
||||||
pub(crate) fn snp_pseudo_alignment(
|
sequences: Vec<Vec<u8>>,
|
||||||
cache: &IndexCache,
|
|
||||||
n: usize,
|
|
||||||
free_loss: bool,
|
free_loss: bool,
|
||||||
no_ambiguity: bool,
|
no_ambiguity: bool,
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
) -> OKIResult<SnpAlignment> {
|
|
||||||
let n_genomes = cache.meta().genomes().len();
|
|
||||||
let genome_indices: Vec<usize> = (0..n_genomes)
|
|
||||||
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
|
|
||||||
.collect();
|
|
||||||
let mut sequences: Vec<Vec<u8>> = vec![Vec::new(); genome_indices.len()];
|
|
||||||
|
|
||||||
sample_index(
|
|
||||||
cache,
|
|
||||||
n,
|
|
||||||
free_loss,
|
|
||||||
no_ambiguity,
|
|
||||||
excluded,
|
|
||||||
entropy_bias,
|
|
||||||
0,
|
|
||||||
|_partition, _layer, _raw_index, survivors| {
|
|
||||||
reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences);
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
Ok(SnpAlignment { sequences, genome_indices })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Appends one layer's worth of columns onto `sequences` — a "reduce" over
|
impl AlignmentReducer {
|
||||||
/// [`sample_index`]'s per-layer batch, self-contained (no sampling/masking
|
pub fn new(cache: &IndexCache, excluded: &[bool], free_loss: bool, no_ambiguity: bool) -> Self {
|
||||||
/// logic of its own, just character encoding) so it can be called
|
let n_genomes = cache.meta().genomes().len();
|
||||||
/// independently of, and alongside, any other reduce over the same batch —
|
let genome_indices: Vec<usize> = (0..n_genomes)
|
||||||
/// `pub(super)`: `algorithms::sankoff::sankoff_bundle` runs this one
|
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
|
||||||
/// alongside `pairwise::reduce_pairwise` over the same `on_layer` call.
|
.collect();
|
||||||
pub(super) fn reduce_alignment(
|
let sequences = vec![Vec::new(); genome_indices.len()];
|
||||||
|
Self { genome_indices, sequences, free_loss, no_ambiguity }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn finish(self) -> SnpAlignment {
|
||||||
|
SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LayerReducer for AlignmentReducer {
|
||||||
|
fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
|
||||||
|
reduce_alignment(survivors, &self.genome_indices, self.free_loss, self.no_ambiguity, &mut self.sequences);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Appends one layer's worth of columns onto `sequences` — self-contained
|
||||||
|
/// (no sampling/masking logic of its own, just character encoding), the
|
||||||
|
/// actual reduce [`AlignmentReducer`] wraps.
|
||||||
|
fn reduce_alignment(
|
||||||
survivors: &[SurvivingFamily],
|
survivors: &[SurvivingFamily],
|
||||||
genome_indices: &[usize],
|
genome_indices: &[usize],
|
||||||
free_loss: bool,
|
free_loss: bool,
|
||||||
|
|||||||
@@ -22,29 +22,27 @@ mod masking;
|
|||||||
mod minorant_selection;
|
mod minorant_selection;
|
||||||
mod pairwise;
|
mod pairwise;
|
||||||
mod sankoff;
|
mod sankoff;
|
||||||
mod session_cache;
|
mod scan;
|
||||||
mod snp_distance;
|
mod snp_distance;
|
||||||
mod stats;
|
mod stats;
|
||||||
mod subsample;
|
mod subsample;
|
||||||
|
|
||||||
use obikidxcache::index_cache::IndexCache;
|
use obikidxcache::index_cache::IndexCache;
|
||||||
|
|
||||||
pub use alignment::SnpAlignment;
|
pub use alignment::{AlignmentReducer, SnpAlignment};
|
||||||
pub use cardcomp::{
|
pub use cardcomp::{
|
||||||
cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix,
|
cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix,
|
||||||
};
|
};
|
||||||
pub use pairwise::{BasePairTally, CardinalityTally, RawSnpDistanceOutput};
|
pub use pairwise::{BasePairTally, CardinalityTally, PairwiseReducer, RawSnpDistanceOutput};
|
||||||
pub use sankoff::SankoffBundle;
|
pub use sankoff::SankoffBundle;
|
||||||
pub use snp_distance::{GammaShape, SnpDistanceKind};
|
pub use scan::{LayerReducer, sampled_scan};
|
||||||
|
pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
||||||
pub use stats::SiblingAnnexStats;
|
pub use stats::SiblingAnnexStats;
|
||||||
pub use subsample::{EntropyBias, SurvivingFamily};
|
pub use subsample::{EntropyBias, SurvivingFamily};
|
||||||
|
|
||||||
pub(crate) use alignment::snp_pseudo_alignment;
|
|
||||||
pub(crate) use annex::build_layer_sibling_annex;
|
pub(crate) use annex::build_layer_sibling_annex;
|
||||||
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4};
|
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4};
|
||||||
pub(crate) use family_scan::{Selection, scan_layer_families};
|
pub(crate) use family_scan::{Selection, scan_layer_families};
|
||||||
pub(crate) use sankoff::sankoff_bundle;
|
|
||||||
pub(crate) use snp_distance::snp_distance;
|
|
||||||
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
|
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
|
||||||
|
|
||||||
/// Whether every layer number in `cache` fits in a `FamilyMask` field
|
/// Whether every layer number in `cache` fits in a `FamilyMask` field
|
||||||
|
|||||||
@@ -44,16 +44,15 @@ struct PairStats {
|
|||||||
cardinality: [[u64; 5]; 5],
|
cardinality: [[u64; 5]; 5],
|
||||||
}
|
}
|
||||||
|
|
||||||
/// See the module docs. `pub(crate)`: consumed by
|
/// See the module docs. `pub`: returned by [`PairwiseReducer::finish`].
|
||||||
/// `algorithms::sankoff`'s orchestration, not yet exposed past this crate.
|
|
||||||
///
|
///
|
||||||
/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): lets `--session` dump/
|
/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): kept from when this type
|
||||||
/// restore a tally across separate CLI invocations as opaque bytes handed
|
/// was dumped to `--session` as opaque bytes via `obiksession::Session`;
|
||||||
/// to `obiksession::Session::store`/`restore` — `obiksession` itself knows
|
/// no active user right now (`--session` support was removed pending a
|
||||||
/// nothing about this type, it only stores/checksums/mmaps the bytes this
|
/// rebuild on top of `algorithms::scan::sampled_scan`), but cheap to keep
|
||||||
/// crate produces (see `obiksession`'s own crate docs on that split).
|
/// derived for when it returns.
|
||||||
#[derive(Archive, Serialize, Deserialize)]
|
#[derive(Archive, Serialize, Deserialize)]
|
||||||
pub(crate) struct PairwiseTally {
|
pub struct PairwiseTally {
|
||||||
n_genomes: usize,
|
n_genomes: usize,
|
||||||
/// Upper triangle only (`i < j`), flat-indexed via [`Self::flat_index`].
|
/// Upper triangle only (`i < j`), flat-indexed via [`Self::flat_index`].
|
||||||
pairs: Vec<PairStats>,
|
pairs: Vec<PairStats>,
|
||||||
@@ -314,7 +313,7 @@ pub struct CardinalityTally {
|
|||||||
/// (`PairStats` accumulates over every genome pair unconditionally;
|
/// (`PairStats` accumulates over every genome pair unconditionally;
|
||||||
/// exclusion is a filter applied only in derived, post-hoc views).
|
/// exclusion is a filter applied only in derived, post-hoc views).
|
||||||
#[derive(Default, Clone, Archive, Serialize, Deserialize)]
|
#[derive(Default, Clone, Archive, Serialize, Deserialize)]
|
||||||
pub(crate) struct PartitionDispersion {
|
pub struct PartitionDispersion {
|
||||||
/// One entry per partition id seen so far (grows lazily — partitions
|
/// One entry per partition id seen so far (grows lazily — partitions
|
||||||
/// are visited in increasing order in practice, but nothing here
|
/// are visited in increasing order in practice, but nothing here
|
||||||
/// assumes that).
|
/// assumes that).
|
||||||
@@ -462,6 +461,34 @@ pub(crate) fn reduce_pairwise(
|
|||||||
dispersion.record(partition, partition_subst, partition_eligible);
|
dispersion.record(partition, partition_subst, partition_eligible);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Accumulates a [`PairwiseTally`]/[`PartitionDispersion`] pair over a
|
||||||
|
/// [`super::scan::sampled_scan`] pass — a thin
|
||||||
|
/// [`super::scan::LayerReducer`] wrapper around [`reduce_pairwise`], so it
|
||||||
|
/// can run alongside any other reducer (e.g.
|
||||||
|
/// `algorithms::alignment::AlignmentReducer`) over the exact same batches
|
||||||
|
/// in one shared scan, instead of each needing its own `sample_index`
|
||||||
|
/// call.
|
||||||
|
pub struct PairwiseReducer {
|
||||||
|
tally: PairwiseTally,
|
||||||
|
dispersion: PartitionDispersion,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PairwiseReducer {
|
||||||
|
pub fn new(n_genomes: usize) -> Self {
|
||||||
|
Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn finish(self) -> (PairwiseTally, PartitionDispersion) {
|
||||||
|
(self.tally, self.dispersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::scan::LayerReducer for PairwiseReducer {
|
||||||
|
fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
|
||||||
|
reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod dispersion_tests {
|
mod dispersion_tests {
|
||||||
use super::PartitionDispersion;
|
use super::PartitionDispersion;
|
||||||
|
|||||||
@@ -1,62 +1,36 @@
|
|||||||
//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree`
|
//! [`SankoffBundle`]: the pure post-processing glue for the
|
||||||
//! pipeline: one [`sample_index`] call (one Bernoulli draw, shared by
|
//! `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline — turns an already-run
|
||||||
//! everyone — the same family selection feeds the alignment and every
|
//! [`super::scan::sampled_scan`] pass's [`AlignmentReducer`]/
|
||||||
//! tally, never two independent samples of the same index) driving two
|
//! [`PairwiseReducer`] output into every value that pipeline needs
|
||||||
//! reduces over the same `on_layer` batches — [`reduce_alignment`] and
|
//! ([`SnpAlignment`], [`RawSnpDistanceOutput`], [`BasePairTally`],
|
||||||
//! [`reduce_pairwise`] — so [`SnpAlignment`], [`RawSnpDistanceOutput`],
|
//! [`CardinalityTally`]). No sampling of its own: the caller (`cmd/phylo`)
|
||||||
//! [`BasePairTally`] and [`CardinalityTally`] all come out of a single
|
//! runs the shared scan itself, with whichever reducers a given CLI
|
||||||
//! scan. The latter three are cheap `O(n²)` post-processing over
|
//! invocation actually needs (alignment only, pairwise only, or both when
|
||||||
//! [`PairwiseTally`] once the scan is done (see its own module docs for why
|
//! `--sankoff` and a `snp-*` `--distance` are requested together) — see
|
||||||
//! that removes the old two-pass `raw_snp_distance`-before-`base_pair_tally`
|
//! `algorithms::scan`'s own module docs for why that single shared call,
|
||||||
//! dependency entirely).
|
//! not a per-output driver, is what guarantees every output in one
|
||||||
|
//! invocation comes from the same site selection.
|
||||||
|
|
||||||
use ndarray::Array2;
|
use ndarray::Array2;
|
||||||
use obikidxcache::index_cache::IndexCache;
|
use obikindex::OKIResult;
|
||||||
use obikindex::{OKIError, OKIResult};
|
|
||||||
use obiksession::Session;
|
|
||||||
|
|
||||||
use super::alignment::{SnpAlignment, reduce_alignment};
|
use super::alignment::SnpAlignment;
|
||||||
use super::pairwise::{
|
use super::pairwise::{
|
||||||
BasePairTally, CardinalityTally, PairwiseTally, PartitionDispersion, RawSnpDistanceOutput,
|
BasePairTally, CardinalityTally, PairwiseReducer, PairwiseTally, PartitionDispersion,
|
||||||
reduce_pairwise,
|
RawSnpDistanceOutput,
|
||||||
};
|
|
||||||
use super::session_cache::{
|
|
||||||
CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint,
|
|
||||||
store_tally_checkpoint,
|
|
||||||
};
|
};
|
||||||
use super::snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
use super::snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
||||||
use super::subsample::{EntropyBias, sample_index};
|
|
||||||
|
|
||||||
/// The alignment's own `--session` artifact — kept in lockstep with the
|
/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline
|
||||||
/// shared `pairwise_tally`/`partition_dispersion`/`progress` artifacts
|
/// needs, built together from one already-scanned
|
||||||
/// (`session_cache`): both are folded from the *same* `on_layer` call, so
|
/// [`AlignmentReducer`]/[`PairwiseReducer`] pair. `pub`: part of the
|
||||||
/// checkpointing them together, under the same `progress` marker, is what
|
/// public signature of [`crate::siblings::extensions::SiblingExt`]'s
|
||||||
/// keeps a resumed alignment's column count consistent with how many
|
/// sampling-based callers.
|
||||||
/// layers the resumed tally actually reflects.
|
|
||||||
const ALIGNMENT_ARTIFACT: &str = "snp_alignment";
|
|
||||||
|
|
||||||
fn restore_alignment(session: &Session) -> Option<SnpAlignment> {
|
|
||||||
let bytes = session.restore(ALIGNMENT_ARTIFACT).ok().flatten()?;
|
|
||||||
rkyv::from_bytes::<SnpAlignment, rkyv::rancor::Error>(&bytes[..]).ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn store_alignment(session: &Session, alignment: &SnpAlignment) -> OKIResult<()> {
|
|
||||||
let bytes = rkyv::to_bytes::<rkyv::rancor::Error>(alignment)
|
|
||||||
.map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?;
|
|
||||||
session.store(ALIGNMENT_ARTIFACT, &bytes).map_err(OKIError::Io)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs,
|
|
||||||
/// computed together from one shared, possibly-subsampled/entropy-biased
|
|
||||||
/// selection — see the module docs. `pub`: part of the public signature of
|
|
||||||
/// [`crate::siblings::extensions::SiblingExt::sankoff_bundle`].
|
|
||||||
///
|
///
|
||||||
/// `tally`/`dispersion` are kept (not just consumed into `raw`/
|
/// `tally`/`dispersion` are kept (not just consumed into `raw`/
|
||||||
/// `base_pair_tally`/`cardinality_tally`) so [`Self::snp_distance`] can
|
/// `base_pair_tally`/`cardinality_tally`) so [`Self::snp_distance`] can
|
||||||
/// compute a `snp-*` `--distance` matrix from this *exact* sample — see
|
/// compute a `snp-*` `--distance` matrix from this *exact* sample without
|
||||||
/// that method's own docs for why reusing it, rather than a fresh
|
/// a second scan.
|
||||||
/// `SiblingExt::snp_distance` call, is a correctness requirement here, not
|
|
||||||
/// just a performance nicety.
|
|
||||||
pub struct SankoffBundle {
|
pub struct SankoffBundle {
|
||||||
pub alignment: SnpAlignment,
|
pub alignment: SnpAlignment,
|
||||||
pub raw: RawSnpDistanceOutput,
|
pub raw: RawSnpDistanceOutput,
|
||||||
@@ -67,124 +41,44 @@ pub struct SankoffBundle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SankoffBundle {
|
impl SankoffBundle {
|
||||||
|
/// Assemble a bundle from an already-finished [`SnpAlignment`] (see
|
||||||
|
/// `AlignmentReducer::finish`) plus a [`PairwiseReducer`] that both ran
|
||||||
|
/// through the same [`super::scan::sampled_scan`] call — pure
|
||||||
|
/// post-processing, `O(n²)` in genome count, never a second scan.
|
||||||
|
/// `alignment` is taken already-finished, not as an `AlignmentReducer`,
|
||||||
|
/// because a caller that *also* needs the alignment on its own (e.g.
|
||||||
|
/// `--pseudo-alignment` combined with `--sankoff` in one invocation)
|
||||||
|
/// has to read it before/alongside building this bundle — `SnpAlignment`
|
||||||
|
/// derives `Clone` for exactly that case. `ratio_ceiling`
|
||||||
|
/// (`--sankoff-ratio-ceiling`) gates [`Self::base_pair_tally`] only
|
||||||
|
/// (see [`PairwiseTally::cardinality_tally`]'s own docs for why
|
||||||
|
/// [`CardinalityTally`] uses a different — `excluded`-only — inclusion
|
||||||
|
/// rule).
|
||||||
|
pub fn finish(
|
||||||
|
alignment: SnpAlignment,
|
||||||
|
pairwise: PairwiseReducer,
|
||||||
|
ratio_ceiling: f64,
|
||||||
|
excluded: &[bool],
|
||||||
|
) -> Self {
|
||||||
|
let (tally, dispersion) = pairwise.finish();
|
||||||
|
let raw = tally.raw_snp_distance();
|
||||||
|
let included = tally.included(ratio_ceiling, excluded);
|
||||||
|
let base_pair_tally = tally.base_pair_tally(&included);
|
||||||
|
let cardinality_tally = tally.cardinality_tally(excluded);
|
||||||
|
Self { alignment, raw, base_pair_tally, cardinality_tally, tally, dispersion }
|
||||||
|
}
|
||||||
|
|
||||||
/// A `snp-*` `--distance` matrix computed from this bundle's own
|
/// A `snp-*` `--distance` matrix computed from this bundle's own
|
||||||
/// already-sampled tally — no new [`sample_index`] call.
|
/// already-sampled tally — no new scan. Exists specifically for the
|
||||||
///
|
/// case where `--sankoff`/`--tnt`/`--phyg`/`--iqtree` and a `snp-*`
|
||||||
/// Exists specifically for the case where `--sankoff`/`--tnt`/`--phyg`/
|
/// `--distance` are requested in the same CLI invocation: both consume
|
||||||
/// `--iqtree` and a `snp-*` `--distance` are requested in the same CLI
|
/// exactly the same selection parameters, so recomputing from a second
|
||||||
/// invocation: both consume exactly the same selection parameters
|
/// [`super::scan::sampled_scan`] call would silently draw a
|
||||||
/// (`n`/`free_loss`/`no_ambiguity`/excluded set/`entropy_bias` — the
|
/// **different** sample (`rand::rng()` is a thread-local generator
|
||||||
/// CLI has no way to give them different values in one run), so a
|
/// that advances across calls, not reseeded each time) — defeating the
|
||||||
/// second independent `SiblingExt::snp_distance` call wouldn't just
|
/// actual point of running several algorithms together, which is
|
||||||
/// waste the sampling work, it would silently sample a **different**
|
/// comparing them on one identical site selection.
|
||||||
/// set of sites (`sample_layer`'s `rand::rng()` is a thread-local
|
|
||||||
/// generator that advances across calls, not reseeded each time) —
|
|
||||||
/// defeating the actual point of running several algorithms together,
|
|
||||||
/// which is comparing them on one identical site selection.
|
|
||||||
pub fn snp_distance(&self, kind: SnpDistanceKind, gamma_shape: GammaShape) -> OKIResult<Array2<f64>> {
|
pub fn snp_distance(&self, kind: SnpDistanceKind, gamma_shape: GammaShape) -> OKIResult<Array2<f64>> {
|
||||||
distance_matrix(&self.tally, &self.dispersion, kind, gamma_shape)
|
distance_matrix(&self.tally, &self.dispersion, kind, gamma_shape)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// See [`crate::siblings::extensions::SiblingExt::sankoff_bundle`] for the
|
|
||||||
/// public-facing docs — this is its implementation. `ratio_ceiling` gates
|
|
||||||
/// [`BasePairTally`] only (see [`PairwiseTally::cardinality_tally`]'s own
|
|
||||||
/// docs for why [`CardinalityTally`] uses a different — `excluded`-only —
|
|
||||||
/// inclusion rule).
|
|
||||||
///
|
|
||||||
/// `session`: `--session`, `None` means no persistence (always resample).
|
|
||||||
/// `Some(session)` must already be open under the exact selection
|
|
||||||
/// parameters this call is about to use — the caller's job, same contract
|
|
||||||
/// as [`super::snp_distance::snp_distance`]'s own `session` parameter,
|
|
||||||
/// which this shares an artifact namespace with (see `session_cache`'s
|
|
||||||
/// module docs): a tally checkpointed here is restorable by a later
|
|
||||||
/// `--distance snp-*` call against the same session, and vice versa.
|
|
||||||
pub(crate) fn sankoff_bundle(
|
|
||||||
cache: &IndexCache,
|
|
||||||
n: usize,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
ratio_ceiling: f64,
|
|
||||||
session: Option<&Session>,
|
|
||||||
) -> OKIResult<SankoffBundle> {
|
|
||||||
let n_genomes = cache.meta().genomes().len();
|
|
||||||
let genome_indices: Vec<usize> = (0..n_genomes)
|
|
||||||
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
let restored = session.and_then(|s| Some((restore_tally_checkpoint(s)?, restore_alignment(s)?)));
|
|
||||||
let (mut tally, mut dispersion, mut sequences, skip_layers) = match restored {
|
|
||||||
Some(((tally, dispersion, skip_layers), alignment)) => {
|
|
||||||
(tally, dispersion, alignment.sequences, skip_layers)
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
let sequences = vec![Vec::new(); genome_indices.len()];
|
|
||||||
(PairwiseTally::new(n_genomes), PartitionDispersion::default(), sequences, 0)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if skip_layers != usize::MAX {
|
|
||||||
// Same checkpoint cadence and rationale as `snp_distance`'s own
|
|
||||||
// resumable loop (see `session_cache::CHECKPOINT_INTERVAL*`'s own
|
|
||||||
// docs) — the alignment's `sequences` are folded in lockstep with
|
|
||||||
// the tally so a resumed run's column count always matches how
|
|
||||||
// many layers the restored tally actually reflects.
|
|
||||||
let mut last_checkpoint_layer = skip_layers;
|
|
||||||
let mut last_checkpoint_at = std::time::Instant::now();
|
|
||||||
sample_index(
|
|
||||||
cache,
|
|
||||||
n,
|
|
||||||
free_loss,
|
|
||||||
no_ambiguity,
|
|
||||||
excluded,
|
|
||||||
entropy_bias,
|
|
||||||
skip_layers,
|
|
||||||
|partition, _layer, raw_index, survivors| {
|
|
||||||
reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences);
|
|
||||||
reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion);
|
|
||||||
let Some(session) = session else { return };
|
|
||||||
let layers_done = raw_index + 1;
|
|
||||||
let due_by_count =
|
|
||||||
layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS;
|
|
||||||
let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL;
|
|
||||||
if !(due_by_count || due_by_time) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() };
|
|
||||||
let stored = store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64)
|
|
||||||
.and_then(|()| store_alignment(session, &alignment));
|
|
||||||
match stored {
|
|
||||||
Ok(()) => {
|
|
||||||
last_checkpoint_layer = layers_done;
|
|
||||||
last_checkpoint_at = std::time::Instant::now();
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
|
|
||||||
if let Some(session) = session {
|
|
||||||
let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() };
|
|
||||||
store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?;
|
|
||||||
store_alignment(session, &alignment)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let raw = tally.raw_snp_distance();
|
|
||||||
let included = tally.included(ratio_ceiling, excluded);
|
|
||||||
let base_pair_tally = tally.base_pair_tally(&included);
|
|
||||||
let cardinality_tally = tally.cardinality_tally(excluded);
|
|
||||||
|
|
||||||
Ok(SankoffBundle {
|
|
||||||
alignment: SnpAlignment { sequences, genome_indices },
|
|
||||||
raw,
|
|
||||||
base_pair_tally,
|
|
||||||
cardinality_tally,
|
|
||||||
tally,
|
|
||||||
dispersion,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
//! The single trunk every sampled sibling-family computation runs
|
||||||
|
//! through: one [`sample_index`] pass, fanned out to whichever
|
||||||
|
//! [`LayerReducer`]s the caller actually needs. Replaces three previously
|
||||||
|
//! independent drivers (a pseudo-alignment-only one, a tally-only one, and
|
||||||
|
//! a hand-fused alignment+tally one for `--sankoff`) that each ran their
|
||||||
|
//! own `sample_index` call — meaning two of them combined in the same CLI
|
||||||
|
//! invocation used to draw two different random samples for what was
|
||||||
|
//! supposed to be one shared site selection. A caller that needs several
|
||||||
|
//! outputs from the same sample now passes several reducers to one
|
||||||
|
//! [`sampled_scan`] call instead of calling several higher-level functions
|
||||||
|
//! and hoping they agree.
|
||||||
|
|
||||||
|
use obikidxcache::index_cache::IndexCache;
|
||||||
|
use obikindex::OKIResult;
|
||||||
|
|
||||||
|
use super::stats::sibling_family_size_histogram;
|
||||||
|
use super::subsample::{EntropyBias, SurvivingFamily, sample_index};
|
||||||
|
|
||||||
|
/// One accumulator fed every layer's [`SurvivingFamily`] batch during a
|
||||||
|
/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of
|
||||||
|
/// its own), so any number of these can run side by side over the exact
|
||||||
|
/// same batches without knowing about each other. Implement this once per
|
||||||
|
/// kind of output the trunk should be able to produce (see
|
||||||
|
/// `algorithms::alignment::AlignmentReducer`, `algorithms::pairwise::PairwiseReducer`).
|
||||||
|
pub trait LayerReducer {
|
||||||
|
fn reduce(&mut self, partition: usize, layer: usize, survivors: &[SurvivingFamily]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `n`: `Some(target)` samples proportionally per layer, up to `target`
|
||||||
|
/// sites index-wide (`--subsample`); `None` means exhaustive — every
|
||||||
|
/// non-monomorphic minorant of the whole index, driven through the exact
|
||||||
|
/// same [`sample_index`] machinery at a quota equal to the total eligible
|
||||||
|
/// count (so every family is drawn with certainty) rather than a separate
|
||||||
|
/// full-scan code path.
|
||||||
|
fn resolve_target(cache: &IndexCache, n: Option<usize>) -> OKIResult<usize> {
|
||||||
|
match n {
|
||||||
|
Some(target) => Ok(target),
|
||||||
|
None => {
|
||||||
|
let counts = sibling_family_size_histogram(cache)?;
|
||||||
|
Ok((counts[1] + counts[2] + counts[3]) as usize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run one shared, possibly-subsampled/entropy-biased selection over
|
||||||
|
/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see
|
||||||
|
/// [`sample_index`]'s own docs for what each controls), feeding every
|
||||||
|
/// surviving family of every visited layer to every reducer in
|
||||||
|
/// `reducers`, in the same order, before moving to the next layer. A
|
||||||
|
/// no-op (not an error) if the resolved target is `0`.
|
||||||
|
pub fn sampled_scan(
|
||||||
|
cache: &IndexCache,
|
||||||
|
n: Option<usize>,
|
||||||
|
free_loss: bool,
|
||||||
|
no_ambiguity: bool,
|
||||||
|
excluded: &[bool],
|
||||||
|
entropy_bias: Option<EntropyBias>,
|
||||||
|
reducers: &mut [&mut dyn LayerReducer],
|
||||||
|
) -> OKIResult<()> {
|
||||||
|
let target = resolve_target(cache, n)?;
|
||||||
|
if target == 0 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| {
|
||||||
|
for reducer in reducers.iter_mut() {
|
||||||
|
reducer.reduce(partition, layer, &survivors);
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
//! Shared `--session` checkpoint plumbing for [`super::snp_distance`] and
|
|
||||||
//! [`super::sankoff`] — both fold [`super::subsample::sample_index`]'s
|
|
||||||
//! layer-by-layer output into a [`PairwiseTally`]/[`PartitionDispersion`]
|
|
||||||
//! pair, and both persist it under the *same* artifact names on purpose:
|
|
||||||
//! a tally checkpointed by one is restorable by the other. Concretely,
|
|
||||||
//! `--sankoff --session DIR` today followed by `--distance snp-k2p
|
|
||||||
//! --session DIR` tomorrow restores the first run's tally instead of
|
|
||||||
//! resampling, even though only `--sankoff` originally built it —
|
|
||||||
//! `snp_distance`'s own selection-parameter conflict check (via
|
|
||||||
//! `obiksession::Session::open`, the caller's responsibility) is what
|
|
||||||
//! guarantees that's actually safe to do.
|
|
||||||
|
|
||||||
use obikindex::{OKIError, OKIResult};
|
|
||||||
use obiksession::Session;
|
|
||||||
|
|
||||||
use super::pairwise::{PairwiseTally, PartitionDispersion};
|
|
||||||
|
|
||||||
pub(crate) const TALLY_ARTIFACT: &str = "pairwise_tally";
|
|
||||||
pub(crate) const DISPERSION_ARTIFACT: &str = "partition_dispersion";
|
|
||||||
pub(crate) const PROGRESS_ARTIFACT: &str = "progress";
|
|
||||||
/// Sentinel `progress` value meaning "every layer was processed — this
|
|
||||||
/// tally is complete," distinct from any real layer count so a resumed run
|
|
||||||
/// can tell "finished" apart from "stopped after N layers" without relying
|
|
||||||
/// on N happening to equal the index's actual layer count.
|
|
||||||
pub(crate) const PROGRESS_COMPLETE: u64 = u64::MAX;
|
|
||||||
|
|
||||||
/// How much in-progress sampling work a checkpoint can lose: at most
|
|
||||||
/// `CHECKPOINT_INTERVAL_LAYERS` layers *or* `CHECKPOINT_INTERVAL` of wall
|
|
||||||
/// time, whichever comes first — not "checkpoint every layer" (rewriting
|
|
||||||
/// the whole, potentially large, tally/dispersion after every single one of
|
|
||||||
/// what can be thousands of layers would make the checkpointing overhead
|
|
||||||
/// itself dominate), and not "only at the end" either (which would defeat
|
|
||||||
/// the point: on a long run against a real production index, losing 90% of
|
|
||||||
/// an already-completed scan to a crash is exactly the expensive case
|
|
||||||
/// resumability exists to avoid — the actual justification, not "the tally
|
|
||||||
/// is cheap to rebuild," which was an earlier, wrong framing of this
|
|
||||||
/// tradeoff caught on review).
|
|
||||||
pub(crate) const CHECKPOINT_INTERVAL_LAYERS: usize = 8;
|
|
||||||
pub(crate) const CHECKPOINT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
|
|
||||||
|
|
||||||
/// Restore a checkpointed `(PairwiseTally, PartitionDispersion, resume-from
|
|
||||||
/// layer index)` triple from `session` — `None` on any miss (an artifact
|
|
||||||
/// not cached, an I/O error, or a deserialization failure), all treated
|
|
||||||
/// identically: something's missing or unusable, fall back to starting
|
|
||||||
/// from scratch (`skip_layers = 0`) rather than surfacing a hard error over
|
|
||||||
/// what's meant to be a transparent cache. The resume index is
|
|
||||||
/// [`PROGRESS_COMPLETE`] when a prior run finished every layer. Layers
|
|
||||||
/// processed after a resume draw a fresh, independent random sample (see
|
|
||||||
/// `subsample::sample_index`'s own `skip_layers` docs) — deliberately not
|
|
||||||
/// bit-identical to what an uninterrupted run would have produced,
|
|
||||||
/// consistent with wanting run-to-run sampling variance rather than fixed
|
|
||||||
/// reproducibility.
|
|
||||||
///
|
|
||||||
/// Full owned deserialization (`rkyv::from_bytes`) for the tally/dispersion,
|
|
||||||
/// not zero-copy (`rkyv::access`) — simpler to integrate with
|
|
||||||
/// `PairwiseTally`'s existing query methods, which all take `&self` by
|
|
||||||
/// value semantics, not an `Archived<PairwiseTally>`. Still skips the
|
|
||||||
/// expensive part (`sample_index` re-scanning the sibling annex for every
|
|
||||||
/// already-completed layer); the deserialize of an already-in-memory-sized
|
|
||||||
/// byte buffer is comparatively cheap. Making this genuinely zero-copy
|
|
||||||
/// later would mean every `PairwiseTally` accessor (`categories`,
|
|
||||||
/// `base_freq`, ...) working generically over `Archived<PairwiseTally>`
|
|
||||||
/// too — a bigger, separate change.
|
|
||||||
pub(crate) fn restore_tally_checkpoint(session: &Session) -> Option<(PairwiseTally, PartitionDispersion, usize)> {
|
|
||||||
let tally_bytes = session.restore(TALLY_ARTIFACT).ok().flatten()?;
|
|
||||||
let dispersion_bytes = session.restore(DISPERSION_ARTIFACT).ok().flatten()?;
|
|
||||||
let progress_bytes = session.restore(PROGRESS_ARTIFACT).ok().flatten()?;
|
|
||||||
let tally = rkyv::from_bytes::<PairwiseTally, rkyv::rancor::Error>(&tally_bytes[..]).ok()?;
|
|
||||||
let dispersion =
|
|
||||||
rkyv::from_bytes::<PartitionDispersion, rkyv::rancor::Error>(&dispersion_bytes[..]).ok()?;
|
|
||||||
let progress_raw: [u8; 8] = progress_bytes.get(..8)?.try_into().ok()?;
|
|
||||||
let progress = u64::from_le_bytes(progress_raw);
|
|
||||||
let skip_layers = if progress == PROGRESS_COMPLETE { usize::MAX } else { progress as usize };
|
|
||||||
if skip_layers == usize::MAX {
|
|
||||||
tracing::info!("--session: restored a complete cached SNP tally — skipping resampling");
|
|
||||||
} else {
|
|
||||||
tracing::info!("--session: resuming a cached SNP tally from layer {skip_layers} onward");
|
|
||||||
}
|
|
||||||
Some((tally, dispersion, skip_layers))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Store `tally`/`dispersion` plus how far sampling has gotten
|
|
||||||
/// (`layers_done`, or [`PROGRESS_COMPLETE`] once every layer has been
|
|
||||||
/// processed) — a checkpoint [`restore_tally_checkpoint`] can resume from
|
|
||||||
/// later, whether that "later" is after a crash or a clean, separate
|
|
||||||
/// invocation.
|
|
||||||
pub(crate) fn store_tally_checkpoint(
|
|
||||||
session: &Session,
|
|
||||||
tally: &PairwiseTally,
|
|
||||||
dispersion: &PartitionDispersion,
|
|
||||||
progress: u64,
|
|
||||||
) -> OKIResult<()> {
|
|
||||||
let tally_bytes = rkyv::to_bytes::<rkyv::rancor::Error>(tally)
|
|
||||||
.map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?;
|
|
||||||
session.store(TALLY_ARTIFACT, &tally_bytes).map_err(OKIError::Io)?;
|
|
||||||
let dispersion_bytes = rkyv::to_bytes::<rkyv::rancor::Error>(dispersion)
|
|
||||||
.map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?;
|
|
||||||
session.store(DISPERSION_ARTIFACT, &dispersion_bytes).map_err(OKIError::Io)?;
|
|
||||||
session.store(PROGRESS_ARTIFACT, &progress.to_le_bytes()).map_err(OKIError::Io)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
@@ -22,21 +22,13 @@
|
|||||||
//! change.
|
//! change.
|
||||||
|
|
||||||
use ndarray::Array2;
|
use ndarray::Array2;
|
||||||
use obikidxcache::index_cache::IndexCache;
|
|
||||||
use obikindex::OKIResult;
|
use obikindex::OKIResult;
|
||||||
use obiksession::Session;
|
|
||||||
|
|
||||||
use super::pairwise::{PairwiseTally, PartitionDispersion};
|
use super::pairwise::{PairwiseTally, PartitionDispersion};
|
||||||
use super::session_cache::{
|
|
||||||
CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint,
|
|
||||||
store_tally_checkpoint,
|
|
||||||
};
|
|
||||||
use super::sibling_family_size_histogram;
|
|
||||||
use super::subsample::{EntropyBias, sample_index};
|
|
||||||
|
|
||||||
/// One `snp-*` `--distance` value. `pub`: part of
|
/// One `snp-*` `--distance` value. `pub`: part of the public signature of
|
||||||
/// [`crate::siblings::extensions::SiblingExt::snp_distance`]'s public
|
/// [`crate::siblings::extensions::SiblingExt`]'s distance-computing
|
||||||
/// signature.
|
/// callers.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum SnpDistanceKind {
|
pub enum SnpDistanceKind {
|
||||||
/// Uncorrected p-distance (`snp / (snp + shared)`).
|
/// Uncorrected p-distance (`snp / (snp + shared)`).
|
||||||
@@ -213,140 +205,18 @@ fn formula(kind: SnpDistanceKind) -> fn(&PairwiseTally, usize, usize, Option<f64
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build (or reuse) a [`PairwiseTally`] and reduce it to one `n×n` distance
|
/// Everything downstream of an already-built
|
||||||
/// matrix under `kind`. `n`: `Some(target)` samples proportionally per
|
/// [`PairwiseTally`]/[`PartitionDispersion`]: the `--gamma-shape` support
|
||||||
/// layer exactly like `--sankoff`/`--pseudo-alignment` (see
|
/// check, `alpha` resolution (fixed/auto/disabled), and the final `n×n`
|
||||||
/// `algorithms::subsample::sample_index`'s own docs); `None` means
|
/// matrix build — `O(n²)` post-processing, never a scan of its own. The
|
||||||
/// exhaustive — every non-monomorphic minorant of the whole index, achieved
|
/// tally/dispersion themselves come from a
|
||||||
/// by handing `sample_index` a quota equal to the index-wide total (from
|
/// [`super::pairwise::PairwiseReducer`] that already ran through
|
||||||
/// [`sibling_family_size_histogram`], annex-bits-only, no cross-partition
|
/// [`super::scan::sampled_scan`] — see [`super::sankoff::SankoffBundle::snp_distance`]
|
||||||
/// resolution), which makes every per-layer quota equal to that layer's own
|
/// for why reusing an already-scanned reducer, rather than a second
|
||||||
/// full eligible count, i.e. Bernoulli `p = 1` everywhere — no separate
|
/// [`sampled_scan`](super::scan::sampled_scan) call, is a correctness
|
||||||
/// exhaustive-only driver needed. Entropy-biased sampling
|
/// requirement (not just a performance nicety) whenever `--sankoff` and a
|
||||||
/// (`entropy_bias`) only makes sense for a genuine subsample, so it's
|
/// `snp-*` `--distance` are requested together.
|
||||||
/// ignored (forced to `None`) when `n` is `None`, regardless of what the
|
pub fn distance_matrix(
|
||||||
/// caller passes.
|
|
||||||
///
|
|
||||||
/// `gamma_shape`: `--gamma-shape`, [`GammaShape::Disabled`] leaves the
|
|
||||||
/// correction off. Rejected with [`obikindex::OKIError::InvalidInput`] if
|
|
||||||
/// anything but `Disabled` is given alongside a `kind` that doesn't support
|
|
||||||
/// it ([`SnpDistanceKind::supports_gamma`]) — checked here rather than left
|
|
||||||
/// to silently no-op. [`GammaShape::Auto`] estimates `alpha` once, from the
|
|
||||||
/// same sampling pass, via [`PartitionDispersion::estimate_alpha`]; if that
|
|
||||||
/// estimation finds no measurable signal (see its own docs for when), the
|
|
||||||
/// correction is silently left off rather than applied with a fabricated
|
|
||||||
/// value — logged either way.
|
|
||||||
///
|
|
||||||
/// `session`: `--session`, `None` means no persistence (today's default
|
|
||||||
/// behavior — always resample). `Some(session)` is expected to already be
|
|
||||||
/// open under the *exact* selection parameters this call is about to use
|
|
||||||
/// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias`) — checking
|
|
||||||
/// that is the caller's job (`obiksession::Session::open`'s own
|
|
||||||
/// params-conflict mechanism), not re-validated here. If a cached tally is
|
|
||||||
/// found it's restored and `sample_index` is skipped entirely; otherwise
|
|
||||||
/// the fresh tally is stored back into the session before this returns, so
|
|
||||||
/// a later call under the same session/params restores instead of
|
|
||||||
/// resampling.
|
|
||||||
pub(crate) fn snp_distance(
|
|
||||||
cache: &IndexCache,
|
|
||||||
kind: SnpDistanceKind,
|
|
||||||
n: Option<usize>,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
gamma_shape: GammaShape,
|
|
||||||
session: Option<&Session>,
|
|
||||||
) -> OKIResult<Array2<f64>> {
|
|
||||||
// Fail fast, before paying for `sample_index`, not just inside
|
|
||||||
// `distance_matrix` (which runs after sampling either way — the right
|
|
||||||
// place for the check when called from `SankoffBundle::snp_distance`,
|
|
||||||
// where sampling already happened as part of `--sankoff` itself).
|
|
||||||
if gamma_shape != GammaShape::Disabled && !kind.supports_gamma() {
|
|
||||||
return Err(obikindex::OKIError::InvalidInput(
|
|
||||||
"--gamma-shape has no effect on this --distance value".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let (mut tally, mut dispersion, skip_layers) = match session.and_then(restore_tally_checkpoint) {
|
|
||||||
Some(triple) => triple,
|
|
||||||
None => {
|
|
||||||
let n_genomes = cache.meta().genomes().len();
|
|
||||||
(PairwiseTally::new(n_genomes), PartitionDispersion::default(), 0)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if skip_layers != usize::MAX {
|
|
||||||
let (target, entropy_bias) = match n {
|
|
||||||
Some(target) => (target, entropy_bias),
|
|
||||||
None => {
|
|
||||||
let counts = sibling_family_size_histogram(cache)?;
|
|
||||||
let total_eligible = (counts[1] + counts[2] + counts[3]) as usize;
|
|
||||||
(total_eligible, None)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if target > 0 {
|
|
||||||
// Checkpoint every `CHECKPOINT_INTERVAL_LAYERS` layers or
|
|
||||||
// `CHECKPOINT_INTERVAL` of wall time, whichever comes first —
|
|
||||||
// see those constants' own docs for why "just recompute
|
|
||||||
// everything" isn't the right call to make on a long run
|
|
||||||
// against a real production index.
|
|
||||||
let mut last_checkpoint_layer = skip_layers;
|
|
||||||
let mut last_checkpoint_at = std::time::Instant::now();
|
|
||||||
sample_index(
|
|
||||||
cache,
|
|
||||||
target,
|
|
||||||
free_loss,
|
|
||||||
no_ambiguity,
|
|
||||||
excluded,
|
|
||||||
entropy_bias,
|
|
||||||
skip_layers,
|
|
||||||
|partition, _layer, raw_index, survivors| {
|
|
||||||
super::pairwise::reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion);
|
|
||||||
let Some(session) = session else { return };
|
|
||||||
let layers_done = raw_index + 1;
|
|
||||||
let due_by_count =
|
|
||||||
layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS;
|
|
||||||
let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL;
|
|
||||||
if !(due_by_count || due_by_time) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
match store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64) {
|
|
||||||
Ok(()) => {
|
|
||||||
last_checkpoint_layer = layers_done;
|
|
||||||
last_checkpoint_at = std::time::Instant::now();
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(session) = session {
|
|
||||||
store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
distance_matrix(&tally, &dispersion, kind, gamma_shape)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The post-sampling half of [`snp_distance`] — everything downstream of
|
|
||||||
/// an already-built [`PairwiseTally`]/[`PartitionDispersion`]: the
|
|
||||||
/// `--gamma-shape` support check, `alpha` resolution (fixed/auto/disabled),
|
|
||||||
/// and the final `n×n` matrix build. Factored out so
|
|
||||||
/// [`super::sankoff::SankoffBundle::snp_distance`] can reuse a tally it
|
|
||||||
/// already has (from `--sankoff`/`--tnt`/`--phyg`/`--iqtree` sharing the
|
|
||||||
/// exact same selection parameters as a `snp-*` `--distance` in the same
|
|
||||||
/// CLI invocation) instead of re-running [`sample_index`] — which, beyond
|
|
||||||
/// the wasted work, would also draw a **different** random sample the
|
|
||||||
/// second time (`rand::rng()` is a thread-local generator that advances
|
|
||||||
/// across calls, never reseeded per call), silently breaking the
|
|
||||||
/// "same site selection for every algorithm in this run" guarantee that's
|
|
||||||
/// the actual point of combining these flags in one invocation.
|
|
||||||
pub(crate) fn distance_matrix(
|
|
||||||
tally: &PairwiseTally,
|
tally: &PairwiseTally,
|
||||||
dispersion: &PartitionDispersion,
|
dispersion: &PartitionDispersion,
|
||||||
kind: SnpDistanceKind,
|
kind: SnpDistanceKind,
|
||||||
|
|||||||
@@ -9,16 +9,13 @@
|
|||||||
use std::io::{BufWriter, Write};
|
use std::io::{BufWriter, Write};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
use ndarray::Array2;
|
|
||||||
use obikidxcache::index_cache::IndexCache;
|
use obikidxcache::index_cache::IndexCache;
|
||||||
use obikindex::{OKIError, OKIResult};
|
use obikindex::{OKIError, OKIResult};
|
||||||
use obisys::progress_bar;
|
use obisys::progress_bar;
|
||||||
|
|
||||||
use crate::siblings::algorithms::{
|
use crate::siblings::algorithms::{
|
||||||
EntropyBias, GammaShape, SankoffBundle, Selection, SiblingAnnexStats, SnpAlignment,
|
Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4,
|
||||||
SnpDistanceKind, build_layer_sibling_annex, family_entropy, family_entropy_4, is_fast_mode,
|
is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram,
|
||||||
sankoff_bundle, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram,
|
|
||||||
snp_distance, snp_pseudo_alignment,
|
|
||||||
};
|
};
|
||||||
use crate::siblings::extensions::SiblingBuilder;
|
use crate::siblings::extensions::SiblingBuilder;
|
||||||
use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap};
|
use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap};
|
||||||
@@ -58,28 +55,6 @@ pub trait SiblingExt {
|
|||||||
/// `--subsample`/`--entropy-bias` yet (see the project memory on this).
|
/// `--subsample`/`--entropy-bias` yet (see the project memory on this).
|
||||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>;
|
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>;
|
||||||
|
|
||||||
/// Build a SNP-only pseudo-alignment (`algorithms::subsample::sample_index`
|
|
||||||
/// — proportional per-layer sampling, `n` sites index-wide, target not
|
|
||||||
/// guarantee) from an already-built sibling annex (run
|
|
||||||
/// [`build_sibling_annex`](Self::build_sibling_annex) first). `free_loss`
|
|
||||||
/// (`--free-loss`), `no_ambiguity` (`--no-ambiguity`) and `excluded`
|
|
||||||
/// (`--exclude-genome`, `excluded[g]` true to drop genome `g`) all feed
|
|
||||||
/// straight into the sampling itself, not a later filter — see
|
|
||||||
/// `algorithms::masking`'s docs: a family whose polymorphism only
|
|
||||||
/// existed via states/genomes these exclude is caught and discarded
|
|
||||||
/// during sampling, not left in the output as a false site. Every
|
|
||||||
/// column of the result has already survived that check, and an
|
|
||||||
/// excluded genome has no row at all in the result (see
|
|
||||||
/// `SnpAlignment::genome_indices`).
|
|
||||||
fn snp_pseudo_alignment(
|
|
||||||
&self,
|
|
||||||
n: usize,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
) -> OKIResult<SnpAlignment>;
|
|
||||||
|
|
||||||
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
|
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
|
||||||
/// no `per_genome`) — reads only the already-built annex
|
/// no `per_genome`) — reads only the already-built annex
|
||||||
/// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits,
|
/// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits,
|
||||||
@@ -97,84 +72,6 @@ pub trait SiblingExt {
|
|||||||
/// instead when only the global histogram is needed.
|
/// instead when only the global histogram is needed.
|
||||||
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
|
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
|
||||||
|
|
||||||
/// Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree`
|
|
||||||
/// pipeline: one shared, possibly-subsampled/entropy-biased selection
|
|
||||||
/// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same
|
|
||||||
/// meaning as [`snp_pseudo_alignment`](Self::snp_pseudo_alignment)'s
|
|
||||||
/// own) drives a single scan producing the pseudo-alignment *and* every
|
|
||||||
/// tally the Sankoff cost-matrix calibration
|
|
||||||
/// (`crate::siblings::cardinality_transition_probs`/
|
|
||||||
/// `composition_transition_probs`/`pairwise_cost_matrix`) needs —
|
|
||||||
/// never two independent draws of the same index. `ratio_ceiling`
|
|
||||||
/// (`--sankoff-ratio-ceiling`) excludes genome pairs too close to
|
|
||||||
/// substitution saturation from `SankoffBundle::base_pair_tally`'s
|
|
||||||
/// pool (a saturated pair's base composition is noise, not signal) —
|
|
||||||
/// `SankoffBundle::cardinality_tally` is *not* gated by it (see
|
|
||||||
/// `crate::siblings::CardinalityTally`'s own docs for why that
|
|
||||||
/// wouldn't make sense: cardinality reflects each genome's own
|
|
||||||
/// coverage/duplication structure, not the pair's mutual divergence).
|
|
||||||
/// `session`: `--session`, `None` disables persistence (always
|
|
||||||
/// resample). `Some` must already be open under these exact selection
|
|
||||||
/// parameters (the caller's responsibility) — restores a cached tally/
|
|
||||||
/// alignment when present instead of resampling, and checkpoints a
|
|
||||||
/// fresh one back as sampling progresses otherwise. Shares its
|
|
||||||
/// artifact namespace with [`Self::snp_distance`]'s own `session`
|
|
||||||
/// parameter internally: a tally checkpointed by one is restorable by
|
|
||||||
/// the other, given the same session and selection parameters.
|
|
||||||
fn sankoff_bundle(
|
|
||||||
&self,
|
|
||||||
n: usize,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
ratio_ceiling: f64,
|
|
||||||
session: Option<&obiksession::Session>,
|
|
||||||
) -> OKIResult<SankoffBundle>;
|
|
||||||
|
|
||||||
/// A `snp-*` `--distance` matrix — one of the closed-form corrections
|
|
||||||
/// in [`SnpDistanceKind`], over the central-position SNP substitution
|
|
||||||
/// spectrum (requires an already-built sibling annex, run
|
|
||||||
/// [`build_sibling_annex`](Self::build_sibling_annex) first).
|
|
||||||
///
|
|
||||||
/// `n`: `Some(target)` samples proportionally per layer exactly like
|
|
||||||
/// [`sankoff_bundle`](Self::sankoff_bundle)/
|
|
||||||
/// [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) (`--subsample`);
|
|
||||||
/// `None` means exhaustive — every non-monomorphic minorant of the
|
|
||||||
/// whole index, not an approximation (see
|
|
||||||
/// `algorithms::snp_distance::snp_distance`'s own docs for how this
|
|
||||||
/// reuses the same sampling machinery at `p = 1` rather than a second
|
|
||||||
/// driver). `free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same
|
|
||||||
/// meaning as `snp_pseudo_alignment`'s own (the last forced to `None`
|
|
||||||
/// when `n` is `None`: entropy-biasing only makes sense for a genuine
|
|
||||||
/// subsample).
|
|
||||||
///
|
|
||||||
/// `gamma_shape`: `--gamma-shape`, the Jin-Nei rate-heterogeneity
|
|
||||||
/// correction — [`GammaShape::Disabled`] turns it off,
|
|
||||||
/// [`GammaShape::Fixed`]/[`GammaShape::Auto`] are rejected with
|
|
||||||
/// [`OKIError::InvalidInput`] for a `kind` that doesn't support it
|
|
||||||
/// ([`SnpDistanceKind::supports_gamma`]). `Auto` estimates `alpha` from
|
|
||||||
/// the data itself (method-of-moments over per-partition substitution
|
|
||||||
/// rate dispersion) rather than requiring a user-supplied value.
|
|
||||||
///
|
|
||||||
/// `session`: `--session`, `None` disables persistence (always
|
|
||||||
/// resample). `Some` must already be open under these exact selection
|
|
||||||
/// parameters (the caller's responsibility, via
|
|
||||||
/// `obiksession::Session::open`'s params-conflict check) — restores a
|
|
||||||
/// cached tally when present instead of resampling, and stores a
|
|
||||||
/// freshly computed one back for next time otherwise.
|
|
||||||
fn snp_distance(
|
|
||||||
&self,
|
|
||||||
kind: SnpDistanceKind,
|
|
||||||
n: Option<usize>,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
gamma_shape: GammaShape,
|
|
||||||
session: Option<&obiksession::Session>,
|
|
||||||
) -> OKIResult<Array2<f64>>;
|
|
||||||
|
|
||||||
/// The Family Overlap annex — number of shared *variable* families per
|
/// The Family Overlap annex — number of shared *variable* families per
|
||||||
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs).
|
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs).
|
||||||
/// Built (alongside the entropy annex) if missing, same as every other
|
/// Built (alongside the entropy annex) if missing, same as every other
|
||||||
@@ -183,8 +80,8 @@ pub trait SiblingExt {
|
|||||||
/// (`FamilyOverlap::mean_row(g) < threshold` — same statistic
|
/// (`FamilyOverlap::mean_row(g) < threshold` — same statistic
|
||||||
/// `--family-overlap`'s matrix shows, just averaged per row —
|
/// `--family-overlap`'s matrix shows, just averaged per row —
|
||||||
/// triggers adding `g` to the excluded-genome set consumed by
|
/// triggers adding `g` to the excluded-genome set consumed by
|
||||||
/// `snp_pseudo_alignment`/`sankoff_bundle`/`snp_distance`; this needs
|
/// `algorithms::scan::sampled_scan`'s callers; this needs no
|
||||||
/// no computation of its own beyond that check).
|
/// computation of its own beyond that check).
|
||||||
fn family_overlap(&self) -> OKIResult<FamilyOverlap>;
|
fn family_overlap(&self) -> OKIResult<FamilyOverlap>;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -312,17 +209,6 @@ impl SiblingExt for IndexCache {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snp_pseudo_alignment(
|
|
||||||
&self,
|
|
||||||
n: usize,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
) -> OKIResult<SnpAlignment> {
|
|
||||||
snp_pseudo_alignment(self, n, free_loss, no_ambiguity, excluded, entropy_bias)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> {
|
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> {
|
||||||
sibling_family_size_histogram(self)
|
sibling_family_size_histogram(self)
|
||||||
}
|
}
|
||||||
@@ -331,33 +217,6 @@ impl SiblingExt for IndexCache {
|
|||||||
sibling_annex_stats(self)
|
sibling_annex_stats(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sankoff_bundle(
|
|
||||||
&self,
|
|
||||||
n: usize,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
ratio_ceiling: f64,
|
|
||||||
session: Option<&obiksession::Session>,
|
|
||||||
) -> OKIResult<SankoffBundle> {
|
|
||||||
sankoff_bundle(self, n, free_loss, no_ambiguity, excluded, entropy_bias, ratio_ceiling, session)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn snp_distance(
|
|
||||||
&self,
|
|
||||||
kind: SnpDistanceKind,
|
|
||||||
n: Option<usize>,
|
|
||||||
free_loss: bool,
|
|
||||||
no_ambiguity: bool,
|
|
||||||
excluded: &[bool],
|
|
||||||
entropy_bias: Option<EntropyBias>,
|
|
||||||
gamma_shape: GammaShape,
|
|
||||||
session: Option<&obiksession::Session>,
|
|
||||||
) -> OKIResult<Array2<f64>> {
|
|
||||||
snp_distance(self, kind, n, free_loss, no_ambiguity, excluded, entropy_bias, gamma_shape, session)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn family_overlap(&self) -> OKIResult<FamilyOverlap> {
|
fn family_overlap(&self) -> OKIResult<FamilyOverlap> {
|
||||||
self.ensure_entropy_and_overlap_annexes()?;
|
self.ensure_entropy_and_overlap_annexes()?;
|
||||||
let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
|
let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
|
||||||
|
|||||||
@@ -37,9 +37,10 @@ pub use iter::{
|
|||||||
pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||||
|
|
||||||
pub use algorithms::{
|
pub use algorithms::{
|
||||||
BasePairTally, CardinalityTally, EntropyBias, GammaShape, RawSnpDistanceOutput, SankoffBundle,
|
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
|
||||||
SiblingAnnexStats, SnpAlignment, SnpDistanceKind, cardinality_transition_probs,
|
PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment,
|
||||||
composition_transition_probs, pairwise_cost_matrix,
|
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
|
||||||
|
pairwise_cost_matrix, sampled_scan,
|
||||||
};
|
};
|
||||||
pub use extensions::SiblingExt;
|
pub use extensions::SiblingExt;
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user