Refactor algorithms for unified sampling and reduction
Introduces a unified sampling trunk, new reducer traits, and refined logic for SNP distance calculation and session handling. Enforces strict parameter sharing across all algorithms to ensure consistent sampling.
This commit is contained in:
1 parent
0d1ecac233
commit
1702addbf9
12 files changed
+320
-760
No files matched your search
Generated
-1
@@ -1594,7 +1594,6 @@ dependencies = [
|
||||
"obikindex",
|
||||
"obikindexer",
|
||||
"obikseq",
|
||||
"obiksession",
|
||||
"obipipeline",
|
||||
"obiread",
|
||||
"obiskbuilder",
|
||||
|
||||
@@ -24,43 +24,6 @@ use phylip::write_phylip_relaxed;
|
||||
use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
|
||||
use tnt::write_sankoff_tnt;
|
||||
|
||||
/// Deterministic byte encoding of the `snp-*` `--distance` selection
|
||||
/// parameters, for `obiksession::Session::open`'s params-conflict check.
|
||||
/// `obiksession` treats this as an opaque blob (see its own crate docs) —
|
||||
/// only this module needs to know the layout, and only well enough to
|
||||
/// produce/compare it, never to pretty-print a per-field diff (v1 keeps
|
||||
/// the conflict message generic rather than decoding both sides field by
|
||||
/// field — see its use in `run` below).
|
||||
fn session_params_blob(
|
||||
n: Option<usize>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
match n {
|
||||
Some(v) => {
|
||||
buf.push(1);
|
||||
buf.extend_from_slice(&(v as u64).to_le_bytes());
|
||||
}
|
||||
None => buf.push(0),
|
||||
}
|
||||
buf.push(free_loss as u8);
|
||||
buf.push(no_ambiguity as u8);
|
||||
buf.extend_from_slice(&(excluded.len() as u64).to_le_bytes());
|
||||
buf.extend(excluded.iter().map(|&b| b as u8));
|
||||
match entropy_bias {
|
||||
Some(EntropyBias { mu, sigma }) => {
|
||||
buf.push(1);
|
||||
buf.extend_from_slice(&mu.to_le_bytes());
|
||||
buf.extend_from_slice(&sigma.to_le_bytes());
|
||||
}
|
||||
None => buf.push(0),
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
pub use args::PhyloArgs;
|
||||
|
||||
pub fn run(args: PhyloArgs) {
|
||||
@@ -273,63 +236,83 @@ pub fn run(args: PhyloArgs) {
|
||||
None
|
||||
};
|
||||
|
||||
// ── `--session`: persist the snp-* sample across separate invocations ──
|
||||
// Only covers `snp-*` `--distance` today (see `--session`'s own docs);
|
||||
// `None` when `--session` isn't given, matching today's always-resample
|
||||
// default exactly.
|
||||
let session = args.session.as_ref().map(|dir| {
|
||||
let params = session_params_blob(
|
||||
args.subsample,
|
||||
args.free_loss,
|
||||
args.no_ambiguity,
|
||||
&snp_exclude_mask,
|
||||
entropy_bias,
|
||||
// ── `--session` ──────────────────────────────────────────────────────────
|
||||
// Temporarily unavailable: the per-driver checkpoint/resume plumbing
|
||||
// it relied on (`algorithms::session_cache`) was removed along with
|
||||
// the three independent sampling drivers it patched over (see
|
||||
// `algorithms::scan`'s own module docs) — persistence needs to be
|
||||
// rebuilt on top of the new single `sampled_scan` trunk instead of
|
||||
// silently becoming a no-op.
|
||||
if args.session.is_some() {
|
||||
eprintln!(
|
||||
"error: --session is temporarily unavailable (being rebuilt on top of the unified \
|
||||
sampling trunk) — rerun without --session/--session-force"
|
||||
);
|
||||
match obiksession::Session::open(dir, ¶ms, args.session_force) {
|
||||
Ok(Ok(session)) => {
|
||||
match session.outcome() {
|
||||
obiksession::SessionOutcome::Created => {
|
||||
info!("--session {}: starting fresh", dir.display());
|
||||
}
|
||||
obiksession::SessionOutcome::Reused => {
|
||||
info!("--session {}: found, will reuse its cached sample if present", dir.display());
|
||||
}
|
||||
}
|
||||
session
|
||||
}
|
||||
Ok(Err(_conflict)) => {
|
||||
eprintln!(
|
||||
"error: --session {} was built with different selection parameters \
|
||||
(--subsample/--free-loss/--no-ambiguity/--exclude-genome/--min-shared-family/\
|
||||
--entropy/--entropy-sd) than this run — pass --session-force to discard its \
|
||||
cached sample and start fresh, or point --session at a different directory",
|
||||
dir.display()
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("error opening --session {}: {e}", dir.display());
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ──
|
||||
// All three consume exactly the same selection parameters
|
||||
// (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/
|
||||
// `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from
|
||||
// the *same* random sample when combined in one invocation — so
|
||||
// whichever of them are requested here share exactly one
|
||||
// `sampled_scan` call, each contributing only the reducer(s) it needs,
|
||||
// instead of each running its own scan (which used to silently draw a
|
||||
// different sample per flag — see `obikphylo::siblings::LayerReducer`'s
|
||||
// own module docs).
|
||||
let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree;
|
||||
let snp_kind = args.distance.as_snp();
|
||||
let need_alignment = args.pseudo_alignment || sankoff_requested;
|
||||
let need_pairwise = sankoff_requested || snp_kind.is_some();
|
||||
|
||||
if args.pseudo_alignment && args.subsample.is_none() {
|
||||
eprintln!("error: --pseudo-alignment requires --subsample <N>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
if sankoff_requested && args.subsample.is_none() {
|
||||
eprintln!("error: --sankoff requires --subsample <N>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let mut alignment_reducer = need_alignment.then(|| {
|
||||
obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity)
|
||||
});
|
||||
let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n));
|
||||
|
||||
// ── SNP pseudo-alignment (`--pseudo-alignment`) ─────────────────────────────
|
||||
if args.pseudo_alignment {
|
||||
let Some(subsample_n) = args.subsample else {
|
||||
eprintln!("error: --pseudo-alignment requires --subsample <N>");
|
||||
std::process::exit(1);
|
||||
};
|
||||
if need_alignment || need_pairwise {
|
||||
let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new();
|
||||
if let Some(r) = alignment_reducer.as_mut() {
|
||||
reducers.push(r);
|
||||
}
|
||||
if let Some(r) = pairwise_reducer.as_mut() {
|
||||
reducers.push(r);
|
||||
}
|
||||
|
||||
info!("sampling SNP pseudo-alignment (target {subsample_n} site(s))");
|
||||
let t = Stage::start("pseudo_alignment");
|
||||
let alignment = cache
|
||||
.snp_pseudo_alignment(subsample_n, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias)
|
||||
info!(
|
||||
"sampling SNP sites{}",
|
||||
match args.subsample {
|
||||
Some(n) => format!(" (target {n} site(s))"),
|
||||
None => " (exhaustive)".into(),
|
||||
}
|
||||
);
|
||||
let t = Stage::start("sampled_scan");
|
||||
obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("error building pseudo-alignment: {e}");
|
||||
eprintln!("error sampling SNP sites: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
rep.push(t.stop());
|
||||
}
|
||||
|
||||
// Finished once here (not left as reducers) so `--pseudo-alignment`
|
||||
// and the Sankoff group can each read it below without fighting over
|
||||
// which one consumes it — see `SankoffBundle::finish`'s own docs.
|
||||
let alignment = alignment_reducer.map(obikphylo::siblings::AlignmentReducer::finish);
|
||||
|
||||
// ── SNP pseudo-alignment (`--pseudo-alignment`) ─────────────────────────────
|
||||
if args.pseudo_alignment {
|
||||
let alignment = alignment.as_ref().expect("scanned above: need_alignment");
|
||||
|
||||
let path = args.output.as_ref()
|
||||
.map(|p| format!("{}_alignment.fasta", p.display()))
|
||||
@@ -350,33 +333,17 @@ pub fn run(args: PhyloArgs) {
|
||||
// `snp-*` `--distance` branch below can reuse its already-sampled
|
||||
// tally instead of resampling — see `SankoffBundle::snp_distance`'s own
|
||||
// docs for why that's required for correctness, not just speed, when
|
||||
// both are requested in the same invocation (they always share the
|
||||
// same `n`/`free_loss`/`no_ambiguity`/excluded-set/`entropy_bias`: the
|
||||
// CLI has no way to give them different values in one run).
|
||||
// both are requested in the same invocation.
|
||||
let mut bundle = None;
|
||||
if args.sankoff || args.tnt || args.phyg || args.iqtree {
|
||||
let Some(subsample_n) = args.subsample else {
|
||||
eprintln!("error: --sankoff requires --subsample <N>");
|
||||
std::process::exit(1);
|
||||
};
|
||||
|
||||
info!("sampling Sankoff calibration bundle (target {subsample_n} site(s))");
|
||||
let t = Stage::start("sankoff_bundle");
|
||||
let b = cache
|
||||
.sankoff_bundle(
|
||||
subsample_n,
|
||||
args.free_loss,
|
||||
args.no_ambiguity,
|
||||
&snp_exclude_mask,
|
||||
entropy_bias,
|
||||
args.sankoff_ratio_ceiling,
|
||||
session.as_ref(),
|
||||
)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("error computing Sankoff calibration bundle: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
rep.push(t.stop());
|
||||
if sankoff_requested {
|
||||
let alignment = alignment.clone().expect("scanned above: need_alignment");
|
||||
let pairwise = pairwise_reducer.take().expect("scanned above: need_pairwise");
|
||||
let b = obikphylo::siblings::SankoffBundle::finish(
|
||||
alignment,
|
||||
pairwise,
|
||||
args.sankoff_ratio_ceiling,
|
||||
&snp_exclude_mask,
|
||||
);
|
||||
|
||||
let p_card = cardinality_transition_probs(&b.cardinality_tally);
|
||||
let p_comp = composition_transition_probs(&b.base_pair_tally);
|
||||
@@ -441,7 +408,7 @@ pub fn run(args: PhyloArgs) {
|
||||
eprintln!("error: --shared-kmers has no meaning for a snp-* --distance value");
|
||||
std::process::exit(1);
|
||||
}
|
||||
let kind = args.distance.as_snp().expect("DistanceArg is always classic or snp");
|
||||
let kind = snp_kind.expect("DistanceArg is always classic or snp");
|
||||
let gamma_shape = args.gamma_shape.map_or(GammaShape::Disabled, Into::into);
|
||||
let t = Stage::start("snp_distance");
|
||||
let matrix = match &bundle {
|
||||
@@ -453,23 +420,9 @@ pub fn run(args: PhyloArgs) {
|
||||
b.snp_distance(kind, gamma_shape)
|
||||
}
|
||||
None => {
|
||||
info!(
|
||||
"computing {kind:?} SNP distance for {n} genome(s){}",
|
||||
match args.subsample {
|
||||
Some(n) => format!(" (subsampled, target {n} site(s))"),
|
||||
None => " (exhaustive)".into(),
|
||||
}
|
||||
);
|
||||
cache.snp_distance(
|
||||
kind,
|
||||
args.subsample,
|
||||
args.free_loss,
|
||||
args.no_ambiguity,
|
||||
&snp_exclude_mask,
|
||||
entropy_bias,
|
||||
gamma_shape,
|
||||
session.as_ref(),
|
||||
)
|
||||
info!("computing {kind:?} SNP distance for {n} genome(s) (reusing the sampled scan above)");
|
||||
let (tally, dispersion) = pairwise_reducer.take().expect("scanned above: need_pairwise").finish();
|
||||
obikphylo::siblings::distance_matrix(&tally, &dispersion, kind, gamma_shape)
|
||||
}
|
||||
}
|
||||
.unwrap_or_else(|e| {
|
||||
|
||||
@@ -12,7 +12,6 @@ obicompactvec = { path = "../obicompactvec" }
|
||||
obikidxcache = { path = "../obikidxcache" }
|
||||
obiskbuilder = { path = "../obiskbuilder" }
|
||||
obipipeline = { path = "../obipipeline" }
|
||||
obiksession = { path = "../obiksession" }
|
||||
kodama = "0.3.0"
|
||||
memmap2 = "0.9"
|
||||
ndarray = "0.17"
|
||||
|
||||
@@ -1,27 +1,25 @@
|
||||
//! SNP-only pseudo-alignment: one row (IUPAC-coded) per genome, one column
|
||||
//! per sampled family — built directly on [`sample_index`], so every family
|
||||
//! this produces has already survived both its Bernoulli draw and post-hoc
|
||||
//! per sampled family — built as a [`LayerReducer`] over
|
||||
//! [`super::scan::sampled_scan`]'s per-layer batches, so every family this
|
||||
//! produces has already survived both its Bernoulli draw and post-hoc
|
||||
//! masking ([`super::masking::survives_masking`]).
|
||||
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
use rkyv::{Archive, Deserialize, Serialize};
|
||||
|
||||
use super::masking::{iupac_code, masked_state};
|
||||
use super::subsample::{EntropyBias, SurvivingFamily, sample_index};
|
||||
use super::scan::LayerReducer;
|
||||
use super::subsample::SurvivingFamily;
|
||||
|
||||
/// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the
|
||||
/// same length (one byte per sampled family, in the order [`sample_index`]
|
||||
/// produced them: layer by layer, `family_idx` order within each). An
|
||||
/// excluded genome (`excluded[g]` in
|
||||
/// [`snp_pseudo_alignment`]) has no row at all, not a blanked one — its
|
||||
/// same length (one byte per sampled family, in the order
|
||||
/// [`super::scan::sampled_scan`] produced them: layer by layer,
|
||||
/// `family_idx` order within each). An excluded genome (`excluded[g]` in
|
||||
/// [`AlignmentReducer::new`]) has no row at all, not a blanked one — its
|
||||
/// index simply doesn't appear in `genome_indices`. `pub`, not
|
||||
/// `pub(crate)`: part of the public signature of
|
||||
/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`].
|
||||
///
|
||||
/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): `--session` persistence
|
||||
/// for `SankoffBundle` — same reasoning as `PairwiseTally`'s own derive.
|
||||
#[derive(Archive, Serialize, Deserialize)]
|
||||
/// [`crate::siblings::extensions::SiblingExt`]'s sampling-based methods.
|
||||
#[derive(Clone, Archive, Serialize, Deserialize)]
|
||||
pub struct SnpAlignment {
|
||||
pub sequences: Vec<Vec<u8>>,
|
||||
/// This index's own genome numbering (matching `IndexCache::meta().genomes()`'s
|
||||
@@ -31,50 +29,43 @@ pub struct SnpAlignment {
|
||||
pub genome_indices: Vec<usize>,
|
||||
}
|
||||
|
||||
/// See [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`]
|
||||
/// for the public-facing docs — this is its implementation. A single
|
||||
/// "reduce" ([`reduce_alignment`]) over [`sample_index`]'s per-layer
|
||||
/// batches — the only consumer today, but structured so a future sibling
|
||||
/// reduce (e.g. the Sankoff cost-matrix calibration's cardinality/
|
||||
/// composition tallies) can run over the exact same batches, each cloning
|
||||
/// its own `Arc`, without this function changing at all.
|
||||
pub(crate) fn snp_pseudo_alignment(
|
||||
cache: &IndexCache,
|
||||
n: usize,
|
||||
/// Accumulates a [`SnpAlignment`] over a [`super::scan::sampled_scan`]
|
||||
/// pass — a thin [`LayerReducer`] wrapper around [`reduce_alignment`], so
|
||||
/// it can run alongside any other reducer (e.g.
|
||||
/// `algorithms::pairwise::PairwiseReducer`) over the exact same batches in
|
||||
/// one shared scan, instead of each needing its own `sample_index` call.
|
||||
pub struct AlignmentReducer {
|
||||
genome_indices: Vec<usize>,
|
||||
sequences: Vec<Vec<u8>>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<SnpAlignment> {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
let genome_indices: Vec<usize> = (0..n_genomes)
|
||||
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
|
||||
.collect();
|
||||
let mut sequences: Vec<Vec<u8>> = vec![Vec::new(); genome_indices.len()];
|
||||
|
||||
sample_index(
|
||||
cache,
|
||||
n,
|
||||
free_loss,
|
||||
no_ambiguity,
|
||||
excluded,
|
||||
entropy_bias,
|
||||
0,
|
||||
|_partition, _layer, _raw_index, survivors| {
|
||||
reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences);
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(SnpAlignment { sequences, genome_indices })
|
||||
}
|
||||
|
||||
/// Appends one layer's worth of columns onto `sequences` — a "reduce" over
|
||||
/// [`sample_index`]'s per-layer batch, self-contained (no sampling/masking
|
||||
/// logic of its own, just character encoding) so it can be called
|
||||
/// independently of, and alongside, any other reduce over the same batch —
|
||||
/// `pub(super)`: `algorithms::sankoff::sankoff_bundle` runs this one
|
||||
/// alongside `pairwise::reduce_pairwise` over the same `on_layer` call.
|
||||
pub(super) fn reduce_alignment(
|
||||
impl AlignmentReducer {
|
||||
pub fn new(cache: &IndexCache, excluded: &[bool], free_loss: bool, no_ambiguity: bool) -> Self {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
let genome_indices: Vec<usize> = (0..n_genomes)
|
||||
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
|
||||
.collect();
|
||||
let sequences = vec![Vec::new(); genome_indices.len()];
|
||||
Self { genome_indices, sequences, free_loss, no_ambiguity }
|
||||
}
|
||||
|
||||
pub fn finish(self) -> SnpAlignment {
|
||||
SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices }
|
||||
}
|
||||
}
|
||||
|
||||
impl LayerReducer for AlignmentReducer {
|
||||
fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
|
||||
reduce_alignment(survivors, &self.genome_indices, self.free_loss, self.no_ambiguity, &mut self.sequences);
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends one layer's worth of columns onto `sequences` — self-contained
|
||||
/// (no sampling/masking logic of its own, just character encoding), the
|
||||
/// actual reduce [`AlignmentReducer`] wraps.
|
||||
fn reduce_alignment(
|
||||
survivors: &[SurvivingFamily],
|
||||
genome_indices: &[usize],
|
||||
free_loss: bool,
|
||||
|
||||
@@ -22,29 +22,27 @@ mod masking;
|
||||
mod minorant_selection;
|
||||
mod pairwise;
|
||||
mod sankoff;
|
||||
mod session_cache;
|
||||
mod scan;
|
||||
mod snp_distance;
|
||||
mod stats;
|
||||
mod subsample;
|
||||
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
|
||||
pub use alignment::SnpAlignment;
|
||||
pub use alignment::{AlignmentReducer, SnpAlignment};
|
||||
pub use cardcomp::{
|
||||
cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix,
|
||||
};
|
||||
pub use pairwise::{BasePairTally, CardinalityTally, RawSnpDistanceOutput};
|
||||
pub use pairwise::{BasePairTally, CardinalityTally, PairwiseReducer, RawSnpDistanceOutput};
|
||||
pub use sankoff::SankoffBundle;
|
||||
pub use snp_distance::{GammaShape, SnpDistanceKind};
|
||||
pub use scan::{LayerReducer, sampled_scan};
|
||||
pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
||||
pub use stats::SiblingAnnexStats;
|
||||
pub use subsample::{EntropyBias, SurvivingFamily};
|
||||
|
||||
pub(crate) use alignment::snp_pseudo_alignment;
|
||||
pub(crate) use annex::build_layer_sibling_annex;
|
||||
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4};
|
||||
pub(crate) use family_scan::{Selection, scan_layer_families};
|
||||
pub(crate) use sankoff::sankoff_bundle;
|
||||
pub(crate) use snp_distance::snp_distance;
|
||||
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
|
||||
|
||||
/// Whether every layer number in `cache` fits in a `FamilyMask` field
|
||||
|
||||
@@ -44,16 +44,15 @@ struct PairStats {
|
||||
cardinality: [[u64; 5]; 5],
|
||||
}
|
||||
|
||||
/// See the module docs. `pub(crate)`: consumed by
|
||||
/// `algorithms::sankoff`'s orchestration, not yet exposed past this crate.
|
||||
/// See the module docs. `pub`: returned by [`PairwiseReducer::finish`].
|
||||
///
|
||||
/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): lets `--session` dump/
|
||||
/// restore a tally across separate CLI invocations as opaque bytes handed
|
||||
/// to `obiksession::Session::store`/`restore` — `obiksession` itself knows
|
||||
/// nothing about this type, it only stores/checksums/mmaps the bytes this
|
||||
/// crate produces (see `obiksession`'s own crate docs on that split).
|
||||
/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): kept from when this type
|
||||
/// was dumped to `--session` as opaque bytes via `obiksession::Session`;
|
||||
/// no active user right now (`--session` support was removed pending a
|
||||
/// rebuild on top of `algorithms::scan::sampled_scan`), but cheap to keep
|
||||
/// derived for when it returns.
|
||||
#[derive(Archive, Serialize, Deserialize)]
|
||||
pub(crate) struct PairwiseTally {
|
||||
pub struct PairwiseTally {
|
||||
n_genomes: usize,
|
||||
/// Upper triangle only (`i < j`), flat-indexed via [`Self::flat_index`].
|
||||
pairs: Vec<PairStats>,
|
||||
@@ -314,7 +313,7 @@ pub struct CardinalityTally {
|
||||
/// (`PairStats` accumulates over every genome pair unconditionally;
|
||||
/// exclusion is a filter applied only in derived, post-hoc views).
|
||||
#[derive(Default, Clone, Archive, Serialize, Deserialize)]
|
||||
pub(crate) struct PartitionDispersion {
|
||||
pub struct PartitionDispersion {
|
||||
/// One entry per partition id seen so far (grows lazily — partitions
|
||||
/// are visited in increasing order in practice, but nothing here
|
||||
/// assumes that).
|
||||
@@ -462,6 +461,34 @@ pub(crate) fn reduce_pairwise(
|
||||
dispersion.record(partition, partition_subst, partition_eligible);
|
||||
}
|
||||
|
||||
/// Accumulates a [`PairwiseTally`]/[`PartitionDispersion`] pair over a
|
||||
/// [`super::scan::sampled_scan`] pass — a thin
|
||||
/// [`super::scan::LayerReducer`] wrapper around [`reduce_pairwise`], so it
|
||||
/// can run alongside any other reducer (e.g.
|
||||
/// `algorithms::alignment::AlignmentReducer`) over the exact same batches
|
||||
/// in one shared scan, instead of each needing its own `sample_index`
|
||||
/// call.
|
||||
pub struct PairwiseReducer {
|
||||
tally: PairwiseTally,
|
||||
dispersion: PartitionDispersion,
|
||||
}
|
||||
|
||||
impl PairwiseReducer {
|
||||
pub fn new(n_genomes: usize) -> Self {
|
||||
Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() }
|
||||
}
|
||||
|
||||
pub fn finish(self) -> (PairwiseTally, PartitionDispersion) {
|
||||
(self.tally, self.dispersion)
|
||||
}
|
||||
}
|
||||
|
||||
impl super::scan::LayerReducer for PairwiseReducer {
|
||||
fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
|
||||
reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod dispersion_tests {
|
||||
use super::PartitionDispersion;
|
||||
|
||||
@@ -1,62 +1,36 @@
|
||||
//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree`
|
||||
//! pipeline: one [`sample_index`] call (one Bernoulli draw, shared by
|
||||
//! everyone — the same family selection feeds the alignment and every
|
||||
//! tally, never two independent samples of the same index) driving two
|
||||
//! reduces over the same `on_layer` batches — [`reduce_alignment`] and
|
||||
//! [`reduce_pairwise`] — so [`SnpAlignment`], [`RawSnpDistanceOutput`],
|
||||
//! [`BasePairTally`] and [`CardinalityTally`] all come out of a single
|
||||
//! scan. The latter three are cheap `O(n²)` post-processing over
|
||||
//! [`PairwiseTally`] once the scan is done (see its own module docs for why
|
||||
//! that removes the old two-pass `raw_snp_distance`-before-`base_pair_tally`
|
||||
//! dependency entirely).
|
||||
//! [`SankoffBundle`]: the pure post-processing glue for the
|
||||
//! `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline — turns an already-run
|
||||
//! [`super::scan::sampled_scan`] pass's [`AlignmentReducer`]/
|
||||
//! [`PairwiseReducer`] output into every value that pipeline needs
|
||||
//! ([`SnpAlignment`], [`RawSnpDistanceOutput`], [`BasePairTally`],
|
||||
//! [`CardinalityTally`]). No sampling of its own: the caller (`cmd/phylo`)
|
||||
//! runs the shared scan itself, with whichever reducers a given CLI
|
||||
//! invocation actually needs (alignment only, pairwise only, or both when
|
||||
//! `--sankoff` and a `snp-*` `--distance` are requested together) — see
|
||||
//! `algorithms::scan`'s own module docs for why that single shared call,
|
||||
//! not a per-output driver, is what guarantees every output in one
|
||||
//! invocation comes from the same site selection.
|
||||
|
||||
use ndarray::Array2;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obiksession::Session;
|
||||
use obikindex::OKIResult;
|
||||
|
||||
use super::alignment::{SnpAlignment, reduce_alignment};
|
||||
use super::alignment::SnpAlignment;
|
||||
use super::pairwise::{
|
||||
BasePairTally, CardinalityTally, PairwiseTally, PartitionDispersion, RawSnpDistanceOutput,
|
||||
reduce_pairwise,
|
||||
};
|
||||
use super::session_cache::{
|
||||
CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint,
|
||||
store_tally_checkpoint,
|
||||
BasePairTally, CardinalityTally, PairwiseReducer, PairwiseTally, PartitionDispersion,
|
||||
RawSnpDistanceOutput,
|
||||
};
|
||||
use super::snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
||||
use super::subsample::{EntropyBias, sample_index};
|
||||
|
||||
/// The alignment's own `--session` artifact — kept in lockstep with the
|
||||
/// shared `pairwise_tally`/`partition_dispersion`/`progress` artifacts
|
||||
/// (`session_cache`): both are folded from the *same* `on_layer` call, so
|
||||
/// checkpointing them together, under the same `progress` marker, is what
|
||||
/// keeps a resumed alignment's column count consistent with how many
|
||||
/// layers the resumed tally actually reflects.
|
||||
const ALIGNMENT_ARTIFACT: &str = "snp_alignment";
|
||||
|
||||
fn restore_alignment(session: &Session) -> Option<SnpAlignment> {
|
||||
let bytes = session.restore(ALIGNMENT_ARTIFACT).ok().flatten()?;
|
||||
rkyv::from_bytes::<SnpAlignment, rkyv::rancor::Error>(&bytes[..]).ok()
|
||||
}
|
||||
|
||||
fn store_alignment(session: &Session, alignment: &SnpAlignment) -> OKIResult<()> {
|
||||
let bytes = rkyv::to_bytes::<rkyv::rancor::Error>(alignment)
|
||||
.map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?;
|
||||
session.store(ALIGNMENT_ARTIFACT, &bytes).map_err(OKIError::Io)
|
||||
}
|
||||
|
||||
/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs,
|
||||
/// computed together from one shared, possibly-subsampled/entropy-biased
|
||||
/// selection — see the module docs. `pub`: part of the public signature of
|
||||
/// [`crate::siblings::extensions::SiblingExt::sankoff_bundle`].
|
||||
/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline
|
||||
/// needs, built together from one already-scanned
|
||||
/// [`AlignmentReducer`]/[`PairwiseReducer`] pair. `pub`: part of the
|
||||
/// public signature of [`crate::siblings::extensions::SiblingExt`]'s
|
||||
/// sampling-based callers.
|
||||
///
|
||||
/// `tally`/`dispersion` are kept (not just consumed into `raw`/
|
||||
/// `base_pair_tally`/`cardinality_tally`) so [`Self::snp_distance`] can
|
||||
/// compute a `snp-*` `--distance` matrix from this *exact* sample — see
|
||||
/// that method's own docs for why reusing it, rather than a fresh
|
||||
/// `SiblingExt::snp_distance` call, is a correctness requirement here, not
|
||||
/// just a performance nicety.
|
||||
/// compute a `snp-*` `--distance` matrix from this *exact* sample without
|
||||
/// a second scan.
|
||||
pub struct SankoffBundle {
|
||||
pub alignment: SnpAlignment,
|
||||
pub raw: RawSnpDistanceOutput,
|
||||
@@ -67,124 +41,44 @@ pub struct SankoffBundle {
|
||||
}
|
||||
|
||||
impl SankoffBundle {
|
||||
/// Assemble a bundle from an already-finished [`SnpAlignment`] (see
|
||||
/// `AlignmentReducer::finish`) plus a [`PairwiseReducer`] that both ran
|
||||
/// through the same [`super::scan::sampled_scan`] call — pure
|
||||
/// post-processing, `O(n²)` in genome count, never a second scan.
|
||||
/// `alignment` is taken already-finished, not as an `AlignmentReducer`,
|
||||
/// because a caller that *also* needs the alignment on its own (e.g.
|
||||
/// `--pseudo-alignment` combined with `--sankoff` in one invocation)
|
||||
/// has to read it before/alongside building this bundle — `SnpAlignment`
|
||||
/// derives `Clone` for exactly that case. `ratio_ceiling`
|
||||
/// (`--sankoff-ratio-ceiling`) gates [`Self::base_pair_tally`] only
|
||||
/// (see [`PairwiseTally::cardinality_tally`]'s own docs for why
|
||||
/// [`CardinalityTally`] uses a different — `excluded`-only — inclusion
|
||||
/// rule).
|
||||
pub fn finish(
|
||||
alignment: SnpAlignment,
|
||||
pairwise: PairwiseReducer,
|
||||
ratio_ceiling: f64,
|
||||
excluded: &[bool],
|
||||
) -> Self {
|
||||
let (tally, dispersion) = pairwise.finish();
|
||||
let raw = tally.raw_snp_distance();
|
||||
let included = tally.included(ratio_ceiling, excluded);
|
||||
let base_pair_tally = tally.base_pair_tally(&included);
|
||||
let cardinality_tally = tally.cardinality_tally(excluded);
|
||||
Self { alignment, raw, base_pair_tally, cardinality_tally, tally, dispersion }
|
||||
}
|
||||
|
||||
/// A `snp-*` `--distance` matrix computed from this bundle's own
|
||||
/// already-sampled tally — no new [`sample_index`] call.
|
||||
///
|
||||
/// Exists specifically for the case where `--sankoff`/`--tnt`/`--phyg`/
|
||||
/// `--iqtree` and a `snp-*` `--distance` are requested in the same CLI
|
||||
/// invocation: both consume exactly the same selection parameters
|
||||
/// (`n`/`free_loss`/`no_ambiguity`/excluded set/`entropy_bias` — the
|
||||
/// CLI has no way to give them different values in one run), so a
|
||||
/// second independent `SiblingExt::snp_distance` call wouldn't just
|
||||
/// waste the sampling work, it would silently sample a **different**
|
||||
/// set of sites (`sample_layer`'s `rand::rng()` is a thread-local
|
||||
/// generator that advances across calls, not reseeded each time) —
|
||||
/// defeating the actual point of running several algorithms together,
|
||||
/// which is comparing them on one identical site selection.
|
||||
/// already-sampled tally — no new scan. Exists specifically for the
|
||||
/// case where `--sankoff`/`--tnt`/`--phyg`/`--iqtree` and a `snp-*`
|
||||
/// `--distance` are requested in the same CLI invocation: both consume
|
||||
/// exactly the same selection parameters, so recomputing from a second
|
||||
/// [`super::scan::sampled_scan`] call would silently draw a
|
||||
/// **different** sample (`rand::rng()` is a thread-local generator
|
||||
/// that advances across calls, not reseeded each time) — defeating the
|
||||
/// actual point of running several algorithms together, which is
|
||||
/// comparing them on one identical site selection.
|
||||
pub fn snp_distance(&self, kind: SnpDistanceKind, gamma_shape: GammaShape) -> OKIResult<Array2<f64>> {
|
||||
distance_matrix(&self.tally, &self.dispersion, kind, gamma_shape)
|
||||
}
|
||||
}
|
||||
|
||||
/// See [`crate::siblings::extensions::SiblingExt::sankoff_bundle`] for the
|
||||
/// public-facing docs — this is its implementation. `ratio_ceiling` gates
|
||||
/// [`BasePairTally`] only (see [`PairwiseTally::cardinality_tally`]'s own
|
||||
/// docs for why [`CardinalityTally`] uses a different — `excluded`-only —
|
||||
/// inclusion rule).
|
||||
///
|
||||
/// `session`: `--session`, `None` means no persistence (always resample).
|
||||
/// `Some(session)` must already be open under the exact selection
|
||||
/// parameters this call is about to use — the caller's job, same contract
|
||||
/// as [`super::snp_distance::snp_distance`]'s own `session` parameter,
|
||||
/// which this shares an artifact namespace with (see `session_cache`'s
|
||||
/// module docs): a tally checkpointed here is restorable by a later
|
||||
/// `--distance snp-*` call against the same session, and vice versa.
|
||||
pub(crate) fn sankoff_bundle(
|
||||
cache: &IndexCache,
|
||||
n: usize,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
ratio_ceiling: f64,
|
||||
session: Option<&Session>,
|
||||
) -> OKIResult<SankoffBundle> {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
let genome_indices: Vec<usize> = (0..n_genomes)
|
||||
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
|
||||
.collect();
|
||||
|
||||
let restored = session.and_then(|s| Some((restore_tally_checkpoint(s)?, restore_alignment(s)?)));
|
||||
let (mut tally, mut dispersion, mut sequences, skip_layers) = match restored {
|
||||
Some(((tally, dispersion, skip_layers), alignment)) => {
|
||||
(tally, dispersion, alignment.sequences, skip_layers)
|
||||
}
|
||||
None => {
|
||||
let sequences = vec![Vec::new(); genome_indices.len()];
|
||||
(PairwiseTally::new(n_genomes), PartitionDispersion::default(), sequences, 0)
|
||||
}
|
||||
};
|
||||
|
||||
if skip_layers != usize::MAX {
|
||||
// Same checkpoint cadence and rationale as `snp_distance`'s own
|
||||
// resumable loop (see `session_cache::CHECKPOINT_INTERVAL*`'s own
|
||||
// docs) — the alignment's `sequences` are folded in lockstep with
|
||||
// the tally so a resumed run's column count always matches how
|
||||
// many layers the restored tally actually reflects.
|
||||
let mut last_checkpoint_layer = skip_layers;
|
||||
let mut last_checkpoint_at = std::time::Instant::now();
|
||||
sample_index(
|
||||
cache,
|
||||
n,
|
||||
free_loss,
|
||||
no_ambiguity,
|
||||
excluded,
|
||||
entropy_bias,
|
||||
skip_layers,
|
||||
|partition, _layer, raw_index, survivors| {
|
||||
reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences);
|
||||
reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion);
|
||||
let Some(session) = session else { return };
|
||||
let layers_done = raw_index + 1;
|
||||
let due_by_count =
|
||||
layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS;
|
||||
let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL;
|
||||
if !(due_by_count || due_by_time) {
|
||||
return;
|
||||
}
|
||||
let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() };
|
||||
let stored = store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64)
|
||||
.and_then(|()| store_alignment(session, &alignment));
|
||||
match stored {
|
||||
Ok(()) => {
|
||||
last_checkpoint_layer = layers_done;
|
||||
last_checkpoint_at = std::time::Instant::now();
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}");
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
|
||||
if let Some(session) = session {
|
||||
let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() };
|
||||
store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?;
|
||||
store_alignment(session, &alignment)?;
|
||||
}
|
||||
}
|
||||
|
||||
let raw = tally.raw_snp_distance();
|
||||
let included = tally.included(ratio_ceiling, excluded);
|
||||
let base_pair_tally = tally.base_pair_tally(&included);
|
||||
let cardinality_tally = tally.cardinality_tally(excluded);
|
||||
|
||||
Ok(SankoffBundle {
|
||||
alignment: SnpAlignment { sequences, genome_indices },
|
||||
raw,
|
||||
base_pair_tally,
|
||||
cardinality_tally,
|
||||
tally,
|
||||
dispersion,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
//! The single trunk every sampled sibling-family computation runs
|
||||
//! through: one [`sample_index`] pass, fanned out to whichever
|
||||
//! [`LayerReducer`]s the caller actually needs. Replaces three previously
|
||||
//! independent drivers (a pseudo-alignment-only one, a tally-only one, and
|
||||
//! a hand-fused alignment+tally one for `--sankoff`) that each ran their
|
||||
//! own `sample_index` call — meaning two of them combined in the same CLI
|
||||
//! invocation used to draw two different random samples for what was
|
||||
//! supposed to be one shared site selection. A caller that needs several
|
||||
//! outputs from the same sample now passes several reducers to one
|
||||
//! [`sampled_scan`] call instead of calling several higher-level functions
|
||||
//! and hoping they agree.
|
||||
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
|
||||
use super::stats::sibling_family_size_histogram;
|
||||
use super::subsample::{EntropyBias, SurvivingFamily, sample_index};
|
||||
|
||||
/// One accumulator fed every layer's [`SurvivingFamily`] batch during a
|
||||
/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of
|
||||
/// its own), so any number of these can run side by side over the exact
|
||||
/// same batches without knowing about each other. Implement this once per
|
||||
/// kind of output the trunk should be able to produce (see
|
||||
/// `algorithms::alignment::AlignmentReducer`, `algorithms::pairwise::PairwiseReducer`).
|
||||
pub trait LayerReducer {
|
||||
fn reduce(&mut self, partition: usize, layer: usize, survivors: &[SurvivingFamily]);
|
||||
}
|
||||
|
||||
/// `n`: `Some(target)` samples proportionally per layer, up to `target`
|
||||
/// sites index-wide (`--subsample`); `None` means exhaustive — every
|
||||
/// non-monomorphic minorant of the whole index, driven through the exact
|
||||
/// same [`sample_index`] machinery at a quota equal to the total eligible
|
||||
/// count (so every family is drawn with certainty) rather than a separate
|
||||
/// full-scan code path.
|
||||
fn resolve_target(cache: &IndexCache, n: Option<usize>) -> OKIResult<usize> {
|
||||
match n {
|
||||
Some(target) => Ok(target),
|
||||
None => {
|
||||
let counts = sibling_family_size_histogram(cache)?;
|
||||
Ok((counts[1] + counts[2] + counts[3]) as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one shared, possibly-subsampled/entropy-biased selection over
|
||||
/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see
|
||||
/// [`sample_index`]'s own docs for what each controls), feeding every
|
||||
/// surviving family of every visited layer to every reducer in
|
||||
/// `reducers`, in the same order, before moving to the next layer. A
|
||||
/// no-op (not an error) if the resolved target is `0`.
|
||||
pub fn sampled_scan(
|
||||
cache: &IndexCache,
|
||||
n: Option<usize>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
reducers: &mut [&mut dyn LayerReducer],
|
||||
) -> OKIResult<()> {
|
||||
let target = resolve_target(cache, n)?;
|
||||
if target == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| {
|
||||
for reducer in reducers.iter_mut() {
|
||||
reducer.reduce(partition, layer, &survivors);
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
//! Shared `--session` checkpoint plumbing for [`super::snp_distance`] and
|
||||
//! [`super::sankoff`] — both fold [`super::subsample::sample_index`]'s
|
||||
//! layer-by-layer output into a [`PairwiseTally`]/[`PartitionDispersion`]
|
||||
//! pair, and both persist it under the *same* artifact names on purpose:
|
||||
//! a tally checkpointed by one is restorable by the other. Concretely,
|
||||
//! `--sankoff --session DIR` today followed by `--distance snp-k2p
|
||||
//! --session DIR` tomorrow restores the first run's tally instead of
|
||||
//! resampling, even though only `--sankoff` originally built it —
|
||||
//! `snp_distance`'s own selection-parameter conflict check (via
|
||||
//! `obiksession::Session::open`, the caller's responsibility) is what
|
||||
//! guarantees that's actually safe to do.
|
||||
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obiksession::Session;
|
||||
|
||||
use super::pairwise::{PairwiseTally, PartitionDispersion};
|
||||
|
||||
pub(crate) const TALLY_ARTIFACT: &str = "pairwise_tally";
|
||||
pub(crate) const DISPERSION_ARTIFACT: &str = "partition_dispersion";
|
||||
pub(crate) const PROGRESS_ARTIFACT: &str = "progress";
|
||||
/// Sentinel `progress` value meaning "every layer was processed — this
|
||||
/// tally is complete," distinct from any real layer count so a resumed run
|
||||
/// can tell "finished" apart from "stopped after N layers" without relying
|
||||
/// on N happening to equal the index's actual layer count.
|
||||
pub(crate) const PROGRESS_COMPLETE: u64 = u64::MAX;
|
||||
|
||||
/// How much in-progress sampling work a checkpoint can lose: at most
|
||||
/// `CHECKPOINT_INTERVAL_LAYERS` layers *or* `CHECKPOINT_INTERVAL` of wall
|
||||
/// time, whichever comes first — not "checkpoint every layer" (rewriting
|
||||
/// the whole, potentially large, tally/dispersion after every single one of
|
||||
/// what can be thousands of layers would make the checkpointing overhead
|
||||
/// itself dominate), and not "only at the end" either (which would defeat
|
||||
/// the point: on a long run against a real production index, losing 90% of
|
||||
/// an already-completed scan to a crash is exactly the expensive case
|
||||
/// resumability exists to avoid — the actual justification, not "the tally
|
||||
/// is cheap to rebuild," which was an earlier, wrong framing of this
|
||||
/// tradeoff caught on review).
|
||||
pub(crate) const CHECKPOINT_INTERVAL_LAYERS: usize = 8;
|
||||
pub(crate) const CHECKPOINT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
/// Restore a checkpointed `(PairwiseTally, PartitionDispersion, resume-from
|
||||
/// layer index)` triple from `session` — `None` on any miss (an artifact
|
||||
/// not cached, an I/O error, or a deserialization failure), all treated
|
||||
/// identically: something's missing or unusable, fall back to starting
|
||||
/// from scratch (`skip_layers = 0`) rather than surfacing a hard error over
|
||||
/// what's meant to be a transparent cache. The resume index is
|
||||
/// [`PROGRESS_COMPLETE`] when a prior run finished every layer. Layers
|
||||
/// processed after a resume draw a fresh, independent random sample (see
|
||||
/// `subsample::sample_index`'s own `skip_layers` docs) — deliberately not
|
||||
/// bit-identical to what an uninterrupted run would have produced,
|
||||
/// consistent with wanting run-to-run sampling variance rather than fixed
|
||||
/// reproducibility.
|
||||
///
|
||||
/// Full owned deserialization (`rkyv::from_bytes`) for the tally/dispersion,
|
||||
/// not zero-copy (`rkyv::access`) — simpler to integrate with
|
||||
/// `PairwiseTally`'s existing query methods, which all take `&self` by
|
||||
/// value semantics, not an `Archived<PairwiseTally>`. Still skips the
|
||||
/// expensive part (`sample_index` re-scanning the sibling annex for every
|
||||
/// already-completed layer); the deserialize of an already-in-memory-sized
|
||||
/// byte buffer is comparatively cheap. Making this genuinely zero-copy
|
||||
/// later would mean every `PairwiseTally` accessor (`categories`,
|
||||
/// `base_freq`, ...) working generically over `Archived<PairwiseTally>`
|
||||
/// too — a bigger, separate change.
|
||||
pub(crate) fn restore_tally_checkpoint(session: &Session) -> Option<(PairwiseTally, PartitionDispersion, usize)> {
|
||||
let tally_bytes = session.restore(TALLY_ARTIFACT).ok().flatten()?;
|
||||
let dispersion_bytes = session.restore(DISPERSION_ARTIFACT).ok().flatten()?;
|
||||
let progress_bytes = session.restore(PROGRESS_ARTIFACT).ok().flatten()?;
|
||||
let tally = rkyv::from_bytes::<PairwiseTally, rkyv::rancor::Error>(&tally_bytes[..]).ok()?;
|
||||
let dispersion =
|
||||
rkyv::from_bytes::<PartitionDispersion, rkyv::rancor::Error>(&dispersion_bytes[..]).ok()?;
|
||||
let progress_raw: [u8; 8] = progress_bytes.get(..8)?.try_into().ok()?;
|
||||
let progress = u64::from_le_bytes(progress_raw);
|
||||
let skip_layers = if progress == PROGRESS_COMPLETE { usize::MAX } else { progress as usize };
|
||||
if skip_layers == usize::MAX {
|
||||
tracing::info!("--session: restored a complete cached SNP tally — skipping resampling");
|
||||
} else {
|
||||
tracing::info!("--session: resuming a cached SNP tally from layer {skip_layers} onward");
|
||||
}
|
||||
Some((tally, dispersion, skip_layers))
|
||||
}
|
||||
|
||||
/// Store `tally`/`dispersion` plus how far sampling has gotten
|
||||
/// (`layers_done`, or [`PROGRESS_COMPLETE`] once every layer has been
|
||||
/// processed) — a checkpoint [`restore_tally_checkpoint`] can resume from
|
||||
/// later, whether that "later" is after a crash or a clean, separate
|
||||
/// invocation.
|
||||
pub(crate) fn store_tally_checkpoint(
|
||||
session: &Session,
|
||||
tally: &PairwiseTally,
|
||||
dispersion: &PartitionDispersion,
|
||||
progress: u64,
|
||||
) -> OKIResult<()> {
|
||||
let tally_bytes = rkyv::to_bytes::<rkyv::rancor::Error>(tally)
|
||||
.map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?;
|
||||
session.store(TALLY_ARTIFACT, &tally_bytes).map_err(OKIError::Io)?;
|
||||
let dispersion_bytes = rkyv::to_bytes::<rkyv::rancor::Error>(dispersion)
|
||||
.map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?;
|
||||
session.store(DISPERSION_ARTIFACT, &dispersion_bytes).map_err(OKIError::Io)?;
|
||||
session.store(PROGRESS_ARTIFACT, &progress.to_le_bytes()).map_err(OKIError::Io)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -22,21 +22,13 @@
|
||||
//! change.
|
||||
|
||||
use ndarray::Array2;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
use obiksession::Session;
|
||||
|
||||
use super::pairwise::{PairwiseTally, PartitionDispersion};
|
||||
use super::session_cache::{
|
||||
CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint,
|
||||
store_tally_checkpoint,
|
||||
};
|
||||
use super::sibling_family_size_histogram;
|
||||
use super::subsample::{EntropyBias, sample_index};
|
||||
|
||||
/// One `snp-*` `--distance` value. `pub`: part of
|
||||
/// [`crate::siblings::extensions::SiblingExt::snp_distance`]'s public
|
||||
/// signature.
|
||||
/// One `snp-*` `--distance` value. `pub`: part of the public signature of
|
||||
/// [`crate::siblings::extensions::SiblingExt`]'s distance-computing
|
||||
/// callers.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SnpDistanceKind {
|
||||
/// Uncorrected p-distance (`snp / (snp + shared)`).
|
||||
@@ -213,140 +205,18 @@ fn formula(kind: SnpDistanceKind) -> fn(&PairwiseTally, usize, usize, Option<f64
|
||||
}
|
||||
}
|
||||
|
||||
/// Build (or reuse) a [`PairwiseTally`] and reduce it to one `n×n` distance
|
||||
/// matrix under `kind`. `n`: `Some(target)` samples proportionally per
|
||||
/// layer exactly like `--sankoff`/`--pseudo-alignment` (see
|
||||
/// `algorithms::subsample::sample_index`'s own docs); `None` means
|
||||
/// exhaustive — every non-monomorphic minorant of the whole index, achieved
|
||||
/// by handing `sample_index` a quota equal to the index-wide total (from
|
||||
/// [`sibling_family_size_histogram`], annex-bits-only, no cross-partition
|
||||
/// resolution), which makes every per-layer quota equal to that layer's own
|
||||
/// full eligible count, i.e. Bernoulli `p = 1` everywhere — no separate
|
||||
/// exhaustive-only driver needed. Entropy-biased sampling
|
||||
/// (`entropy_bias`) only makes sense for a genuine subsample, so it's
|
||||
/// ignored (forced to `None`) when `n` is `None`, regardless of what the
|
||||
/// caller passes.
|
||||
///
|
||||
/// `gamma_shape`: `--gamma-shape`, [`GammaShape::Disabled`] leaves the
|
||||
/// correction off. Rejected with [`obikindex::OKIError::InvalidInput`] if
|
||||
/// anything but `Disabled` is given alongside a `kind` that doesn't support
|
||||
/// it ([`SnpDistanceKind::supports_gamma`]) — checked here rather than left
|
||||
/// to silently no-op. [`GammaShape::Auto`] estimates `alpha` once, from the
|
||||
/// same sampling pass, via [`PartitionDispersion::estimate_alpha`]; if that
|
||||
/// estimation finds no measurable signal (see its own docs for when), the
|
||||
/// correction is silently left off rather than applied with a fabricated
|
||||
/// value — logged either way.
|
||||
///
|
||||
/// `session`: `--session`, `None` means no persistence (today's default
|
||||
/// behavior — always resample). `Some(session)` is expected to already be
|
||||
/// open under the *exact* selection parameters this call is about to use
|
||||
/// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias`) — checking
|
||||
/// that is the caller's job (`obiksession::Session::open`'s own
|
||||
/// params-conflict mechanism), not re-validated here. If a cached tally is
|
||||
/// found it's restored and `sample_index` is skipped entirely; otherwise
|
||||
/// the fresh tally is stored back into the session before this returns, so
|
||||
/// a later call under the same session/params restores instead of
|
||||
/// resampling.
|
||||
pub(crate) fn snp_distance(
|
||||
cache: &IndexCache,
|
||||
kind: SnpDistanceKind,
|
||||
n: Option<usize>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
gamma_shape: GammaShape,
|
||||
session: Option<&Session>,
|
||||
) -> OKIResult<Array2<f64>> {
|
||||
// Fail fast, before paying for `sample_index`, not just inside
|
||||
// `distance_matrix` (which runs after sampling either way — the right
|
||||
// place for the check when called from `SankoffBundle::snp_distance`,
|
||||
// where sampling already happened as part of `--sankoff` itself).
|
||||
if gamma_shape != GammaShape::Disabled && !kind.supports_gamma() {
|
||||
return Err(obikindex::OKIError::InvalidInput(
|
||||
"--gamma-shape has no effect on this --distance value".into(),
|
||||
));
|
||||
}
|
||||
|
||||
let (mut tally, mut dispersion, skip_layers) = match session.and_then(restore_tally_checkpoint) {
|
||||
Some(triple) => triple,
|
||||
None => {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
(PairwiseTally::new(n_genomes), PartitionDispersion::default(), 0)
|
||||
}
|
||||
};
|
||||
|
||||
if skip_layers != usize::MAX {
|
||||
let (target, entropy_bias) = match n {
|
||||
Some(target) => (target, entropy_bias),
|
||||
None => {
|
||||
let counts = sibling_family_size_histogram(cache)?;
|
||||
let total_eligible = (counts[1] + counts[2] + counts[3]) as usize;
|
||||
(total_eligible, None)
|
||||
}
|
||||
};
|
||||
|
||||
if target > 0 {
|
||||
// Checkpoint every `CHECKPOINT_INTERVAL_LAYERS` layers or
|
||||
// `CHECKPOINT_INTERVAL` of wall time, whichever comes first —
|
||||
// see those constants' own docs for why "just recompute
|
||||
// everything" isn't the right call to make on a long run
|
||||
// against a real production index.
|
||||
let mut last_checkpoint_layer = skip_layers;
|
||||
let mut last_checkpoint_at = std::time::Instant::now();
|
||||
sample_index(
|
||||
cache,
|
||||
target,
|
||||
free_loss,
|
||||
no_ambiguity,
|
||||
excluded,
|
||||
entropy_bias,
|
||||
skip_layers,
|
||||
|partition, _layer, raw_index, survivors| {
|
||||
super::pairwise::reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion);
|
||||
let Some(session) = session else { return };
|
||||
let layers_done = raw_index + 1;
|
||||
let due_by_count =
|
||||
layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS;
|
||||
let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL;
|
||||
if !(due_by_count || due_by_time) {
|
||||
return;
|
||||
}
|
||||
match store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64) {
|
||||
Ok(()) => {
|
||||
last_checkpoint_layer = layers_done;
|
||||
last_checkpoint_at = std::time::Instant::now();
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}");
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(session) = session {
|
||||
store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?;
|
||||
}
|
||||
}
|
||||
|
||||
distance_matrix(&tally, &dispersion, kind, gamma_shape)
|
||||
}
|
||||
|
||||
/// The post-sampling half of [`snp_distance`] — everything downstream of
|
||||
/// an already-built [`PairwiseTally`]/[`PartitionDispersion`]: the
|
||||
/// `--gamma-shape` support check, `alpha` resolution (fixed/auto/disabled),
|
||||
/// and the final `n×n` matrix build. Factored out so
|
||||
/// [`super::sankoff::SankoffBundle::snp_distance`] can reuse a tally it
|
||||
/// already has (from `--sankoff`/`--tnt`/`--phyg`/`--iqtree` sharing the
|
||||
/// exact same selection parameters as a `snp-*` `--distance` in the same
|
||||
/// CLI invocation) instead of re-running [`sample_index`] — which, beyond
|
||||
/// the wasted work, would also draw a **different** random sample the
|
||||
/// second time (`rand::rng()` is a thread-local generator that advances
|
||||
/// across calls, never reseeded per call), silently breaking the
|
||||
/// "same site selection for every algorithm in this run" guarantee that's
|
||||
/// the actual point of combining these flags in one invocation.
|
||||
pub(crate) fn distance_matrix(
|
||||
/// Everything downstream of an already-built
|
||||
/// [`PairwiseTally`]/[`PartitionDispersion`]: the `--gamma-shape` support
|
||||
/// check, `alpha` resolution (fixed/auto/disabled), and the final `n×n`
|
||||
/// matrix build — `O(n²)` post-processing, never a scan of its own. The
|
||||
/// tally/dispersion themselves come from a
|
||||
/// [`super::pairwise::PairwiseReducer`] that already ran through
|
||||
/// [`super::scan::sampled_scan`] — see [`super::sankoff::SankoffBundle::snp_distance`]
|
||||
/// for why reusing an already-scanned reducer, rather than a second
|
||||
/// [`sampled_scan`](super::scan::sampled_scan) call, is a correctness
|
||||
/// requirement (not just a performance nicety) whenever `--sankoff` and a
|
||||
/// `snp-*` `--distance` are requested together.
|
||||
pub fn distance_matrix(
|
||||
tally: &PairwiseTally,
|
||||
dispersion: &PartitionDispersion,
|
||||
kind: SnpDistanceKind,
|
||||
|
||||
@@ -9,16 +9,13 @@
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use ndarray::Array2;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obisys::progress_bar;
|
||||
|
||||
use crate::siblings::algorithms::{
|
||||
EntropyBias, GammaShape, SankoffBundle, Selection, SiblingAnnexStats, SnpAlignment,
|
||||
SnpDistanceKind, build_layer_sibling_annex, family_entropy, family_entropy_4, is_fast_mode,
|
||||
sankoff_bundle, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram,
|
||||
snp_distance, snp_pseudo_alignment,
|
||||
Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4,
|
||||
is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram,
|
||||
};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap};
|
||||
@@ -58,28 +55,6 @@ pub trait SiblingExt {
|
||||
/// `--subsample`/`--entropy-bias` yet (see the project memory on this).
|
||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>;
|
||||
|
||||
/// Build a SNP-only pseudo-alignment (`algorithms::subsample::sample_index`
|
||||
/// — proportional per-layer sampling, `n` sites index-wide, target not
|
||||
/// guarantee) from an already-built sibling annex (run
|
||||
/// [`build_sibling_annex`](Self::build_sibling_annex) first). `free_loss`
|
||||
/// (`--free-loss`), `no_ambiguity` (`--no-ambiguity`) and `excluded`
|
||||
/// (`--exclude-genome`, `excluded[g]` true to drop genome `g`) all feed
|
||||
/// straight into the sampling itself, not a later filter — see
|
||||
/// `algorithms::masking`'s docs: a family whose polymorphism only
|
||||
/// existed via states/genomes these exclude is caught and discarded
|
||||
/// during sampling, not left in the output as a false site. Every
|
||||
/// column of the result has already survived that check, and an
|
||||
/// excluded genome has no row at all in the result (see
|
||||
/// `SnpAlignment::genome_indices`).
|
||||
fn snp_pseudo_alignment(
|
||||
&self,
|
||||
n: usize,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<SnpAlignment>;
|
||||
|
||||
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
|
||||
/// no `per_genome`) — reads only the already-built annex
|
||||
/// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits,
|
||||
@@ -97,84 +72,6 @@ pub trait SiblingExt {
|
||||
/// instead when only the global histogram is needed.
|
||||
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
|
||||
|
||||
/// Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree`
|
||||
/// pipeline: one shared, possibly-subsampled/entropy-biased selection
|
||||
/// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same
|
||||
/// meaning as [`snp_pseudo_alignment`](Self::snp_pseudo_alignment)'s
|
||||
/// own) drives a single scan producing the pseudo-alignment *and* every
|
||||
/// tally the Sankoff cost-matrix calibration
|
||||
/// (`crate::siblings::cardinality_transition_probs`/
|
||||
/// `composition_transition_probs`/`pairwise_cost_matrix`) needs —
|
||||
/// never two independent draws of the same index. `ratio_ceiling`
|
||||
/// (`--sankoff-ratio-ceiling`) excludes genome pairs too close to
|
||||
/// substitution saturation from `SankoffBundle::base_pair_tally`'s
|
||||
/// pool (a saturated pair's base composition is noise, not signal) —
|
||||
/// `SankoffBundle::cardinality_tally` is *not* gated by it (see
|
||||
/// `crate::siblings::CardinalityTally`'s own docs for why that
|
||||
/// wouldn't make sense: cardinality reflects each genome's own
|
||||
/// coverage/duplication structure, not the pair's mutual divergence).
|
||||
/// `session`: `--session`, `None` disables persistence (always
|
||||
/// resample). `Some` must already be open under these exact selection
|
||||
/// parameters (the caller's responsibility) — restores a cached tally/
|
||||
/// alignment when present instead of resampling, and checkpoints a
|
||||
/// fresh one back as sampling progresses otherwise. Shares its
|
||||
/// artifact namespace with [`Self::snp_distance`]'s own `session`
|
||||
/// parameter internally: a tally checkpointed by one is restorable by
|
||||
/// the other, given the same session and selection parameters.
|
||||
fn sankoff_bundle(
|
||||
&self,
|
||||
n: usize,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
ratio_ceiling: f64,
|
||||
session: Option<&obiksession::Session>,
|
||||
) -> OKIResult<SankoffBundle>;
|
||||
|
||||
/// A `snp-*` `--distance` matrix — one of the closed-form corrections
|
||||
/// in [`SnpDistanceKind`], over the central-position SNP substitution
|
||||
/// spectrum (requires an already-built sibling annex, run
|
||||
/// [`build_sibling_annex`](Self::build_sibling_annex) first).
|
||||
///
|
||||
/// `n`: `Some(target)` samples proportionally per layer exactly like
|
||||
/// [`sankoff_bundle`](Self::sankoff_bundle)/
|
||||
/// [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) (`--subsample`);
|
||||
/// `None` means exhaustive — every non-monomorphic minorant of the
|
||||
/// whole index, not an approximation (see
|
||||
/// `algorithms::snp_distance::snp_distance`'s own docs for how this
|
||||
/// reuses the same sampling machinery at `p = 1` rather than a second
|
||||
/// driver). `free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same
|
||||
/// meaning as `snp_pseudo_alignment`'s own (the last forced to `None`
|
||||
/// when `n` is `None`: entropy-biasing only makes sense for a genuine
|
||||
/// subsample).
|
||||
///
|
||||
/// `gamma_shape`: `--gamma-shape`, the Jin-Nei rate-heterogeneity
|
||||
/// correction — [`GammaShape::Disabled`] turns it off,
|
||||
/// [`GammaShape::Fixed`]/[`GammaShape::Auto`] are rejected with
|
||||
/// [`OKIError::InvalidInput`] for a `kind` that doesn't support it
|
||||
/// ([`SnpDistanceKind::supports_gamma`]). `Auto` estimates `alpha` from
|
||||
/// the data itself (method-of-moments over per-partition substitution
|
||||
/// rate dispersion) rather than requiring a user-supplied value.
|
||||
///
|
||||
/// `session`: `--session`, `None` disables persistence (always
|
||||
/// resample). `Some` must already be open under these exact selection
|
||||
/// parameters (the caller's responsibility, via
|
||||
/// `obiksession::Session::open`'s params-conflict check) — restores a
|
||||
/// cached tally when present instead of resampling, and stores a
|
||||
/// freshly computed one back for next time otherwise.
|
||||
fn snp_distance(
|
||||
&self,
|
||||
kind: SnpDistanceKind,
|
||||
n: Option<usize>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
gamma_shape: GammaShape,
|
||||
session: Option<&obiksession::Session>,
|
||||
) -> OKIResult<Array2<f64>>;
|
||||
|
||||
/// The Family Overlap annex — number of shared *variable* families per
|
||||
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs).
|
||||
/// Built (alongside the entropy annex) if missing, same as every other
|
||||
@@ -183,8 +80,8 @@ pub trait SiblingExt {
|
||||
/// (`FamilyOverlap::mean_row(g) < threshold` — same statistic
|
||||
/// `--family-overlap`'s matrix shows, just averaged per row —
|
||||
/// triggers adding `g` to the excluded-genome set consumed by
|
||||
/// `snp_pseudo_alignment`/`sankoff_bundle`/`snp_distance`; this needs
|
||||
/// no computation of its own beyond that check).
|
||||
/// `algorithms::scan::sampled_scan`'s callers; this needs no
|
||||
/// computation of its own beyond that check).
|
||||
fn family_overlap(&self) -> OKIResult<FamilyOverlap>;
|
||||
}
|
||||
|
||||
@@ -312,17 +209,6 @@ impl SiblingExt for IndexCache {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn snp_pseudo_alignment(
|
||||
&self,
|
||||
n: usize,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<SnpAlignment> {
|
||||
snp_pseudo_alignment(self, n, free_loss, no_ambiguity, excluded, entropy_bias)
|
||||
}
|
||||
|
||||
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> {
|
||||
sibling_family_size_histogram(self)
|
||||
}
|
||||
@@ -331,33 +217,6 @@ impl SiblingExt for IndexCache {
|
||||
sibling_annex_stats(self)
|
||||
}
|
||||
|
||||
fn sankoff_bundle(
|
||||
&self,
|
||||
n: usize,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
ratio_ceiling: f64,
|
||||
session: Option<&obiksession::Session>,
|
||||
) -> OKIResult<SankoffBundle> {
|
||||
sankoff_bundle(self, n, free_loss, no_ambiguity, excluded, entropy_bias, ratio_ceiling, session)
|
||||
}
|
||||
|
||||
fn snp_distance(
|
||||
&self,
|
||||
kind: SnpDistanceKind,
|
||||
n: Option<usize>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
gamma_shape: GammaShape,
|
||||
session: Option<&obiksession::Session>,
|
||||
) -> OKIResult<Array2<f64>> {
|
||||
snp_distance(self, kind, n, free_loss, no_ambiguity, excluded, entropy_bias, gamma_shape, session)
|
||||
}
|
||||
|
||||
fn family_overlap(&self) -> OKIResult<FamilyOverlap> {
|
||||
self.ensure_entropy_and_overlap_annexes()?;
|
||||
let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
|
||||
|
||||
@@ -37,9 +37,10 @@ pub use iter::{
|
||||
pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
|
||||
pub use algorithms::{
|
||||
BasePairTally, CardinalityTally, EntropyBias, GammaShape, RawSnpDistanceOutput, SankoffBundle,
|
||||
SiblingAnnexStats, SnpAlignment, SnpDistanceKind, cardinality_transition_probs,
|
||||
composition_transition_probs, pairwise_cost_matrix,
|
||||
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
|
||||
PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment,
|
||||
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
|
||||
pairwise_cost_matrix, sampled_scan,
|
||||
};
|
||||
pub use extensions::SiblingExt;
|
||||
|
||||
|
||||
Reference in new issue
Block a user