Refactor sampling logic and introduce diagnostic outputs

This commit introduces new features for sampling, including detailed `SamplePlan` generation, Shannon entropy summaries, and site statistics reporting. It formalizes conditional distance matrix computation and clarifies the logic for family weighting and site selection.
This commit is contained in:
Eric Coissac committed 2026-10-15 12:00:44 +02:00
1 parent 2232628e58
commit cb5fc202b6
13 files changed
+799 -122

No files matched your search

+78 -3
View File
@@ -101,6 +101,51 @@ pub enum DistanceArg {
SnpTv,
}
/// The distance matrix this invocation must compute, if any: the one asked for
/// with `--distance`, else `jaccard` when `--nj`, `--upgma` or
/// `--shared-kmers` needs a matrix to work from, else none — a matrix is never
/// computed (nor written) just because nothing said otherwise.
pub fn distance_to_compute(
distance: Option<DistanceArg>,
nj: bool,
upgma: bool,
shared_kmers: bool,
) -> Option<DistanceArg> {
distance.or_else(|| (nj || upgma || shared_kmers).then_some(DistanceArg::Jaccard))
}
#[cfg(test)]
mod distance_tests {
use super::*;
#[test]
fn nothing_asked_nothing_computed() {
assert!(distance_to_compute(None, false, false, false).is_none());
}
#[test]
fn tree_and_shared_kmer_outputs_imply_jaccard() {
for (nj, upgma, shared) in [(true, false, false), (false, true, false), (false, false, true)] {
assert!(matches!(
distance_to_compute(None, nj, upgma, shared),
Some(DistanceArg::Jaccard)
));
}
}
#[test]
fn an_explicit_distance_wins() {
assert!(matches!(
distance_to_compute(Some(DistanceArg::Mash), true, true, true),
Some(DistanceArg::Mash)
));
assert!(matches!(
distance_to_compute(Some(DistanceArg::SnpJc), false, false, false),
Some(DistanceArg::SnpJc)
));
}
}
impl DistanceArg {
/// `Some` for the whole-index metrics, `None` for `snp-*` values.
pub fn as_classic(self) -> Option<DistanceMetric> {
@@ -215,6 +260,15 @@ pub struct PhyloArgs {
#[arg(long)]
pub shannon: bool,
/// Write the same entropy scan as `--shannon` as a distribution
/// (`<prefix>_shannon_summary.csv`): the number of families per entropy
/// kind, number of genomes carrying the family, number of those without
/// ambiguity and 0.05-bit entropy bin — a few hundred thousand lines at
/// most, whatever the size of the index, instead of one line per family.
/// Requires an already-built sibling annex, like `--shannon`.
#[arg(long)]
pub shannon_summary: bool,
/// Write a SNP-only pseudo-alignment (FASTA) — requires
/// `--n-sites <N|all>` and an already-built sibling annex
/// (`--sibling-annex` first, in this invocation or an earlier one).
@@ -260,6 +314,24 @@ pub struct PhyloArgs {
#[arg(long)]
pub pair_balance: bool,
/// Write what the draw contains, from the very sites it kept
/// (requires `--pseudo-alignment`, `--sankoff` or a `snp-*`
/// `--distance`): `<prefix>_sample_genomes.csv` (per genome, the sites it
/// has a call in and the informative sites it takes part in),
/// `<prefix>_sample_carriers.csv` (sites and informative sites per number
/// of carriers) and `<prefix>_sample_overlap.csv` (the sample's own
/// genome × genome overlap matrix, to compare with `--family-overlap`).
#[arg(long)]
pub sample_stats: bool,
/// Write how the draw was spread over the layers of the index
/// (requires `--pseudo-alignment`, `--sankoff` or a `snp-*`
/// `--distance`): `<prefix>_sample_plan.csv`, one line per layer with the
/// families eligible, their total weight, the layer's quota, the families
/// drawn and the ones kept after masking.
#[arg(long)]
pub sample_plan: bool,
/// Entropy-biased sampling target (Gaussian kernel mean) for
/// `--pseudo-alignment` — activates biasing as soon as this or
/// `--entropy-sd` is given; the other defaults to 1.0/0.5.
@@ -371,7 +443,10 @@ pub struct PhyloArgs {
#[arg(long, default_value = "100")]
pub sankoff_cost_scale: f64,
/// Distance to compute — either a whole-index metric (`jaccard`,
/// Distance matrix to compute and write — none is computed unless this
/// is given, or `--nj`/`--upgma`/`--shared-kmers` is (they need a matrix
/// and use `jaccard` then, see [`distance_to_compute`]). Either a
/// whole-index metric (`jaccard`,
/// `mash`, `hamming`, `bray-curtis`, ...) or a `snp-*` correction over
/// the central-position SNP substitution spectrum (`snp-raw`, `snp-jc`,
/// `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv`) —
@@ -381,8 +456,8 @@ pub struct PhyloArgs {
/// `DevDocMD/theory/evolutionary_distances.md`, "`--distance`
/// unification" for the full catalog and why LogDet/Tajima-Nei/F84/
/// HKY85 aren't offered yet).
#[arg(long, value_enum, default_value = "jaccard")]
pub distance: DistanceArg,
#[arg(long, value_enum)]
pub distance: Option<DistanceArg>,
/// Rate-heterogeneity correction (Jin-Nei gamma shape parameter `α`)
/// for `snp-*` `--distance` values that support it
+57 -8
View File
@@ -3,6 +3,7 @@ mod iqtree;
mod phyg;
mod phylip;
mod sankoff;
mod stats;
mod tnt;
use std::io::{self, BufWriter, Write};
@@ -18,9 +19,10 @@ use obikphylo::{Metrics, neighbor_joining, upgma};
use obisys::{Reporter, Stage};
use tracing::info;
use args::NSitesArg;
use args::{DistanceArg, NSitesArg, distance_to_compute};
use iqtree::write_iqtree;
use phyg::write_sankoff_phyg;
use stats::{write_sample_plan, write_sample_stats};
use phylip::write_phylip_relaxed;
use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
use tnt::write_sankoff_tnt;
@@ -194,6 +196,21 @@ pub fn run(args: PhyloArgs) {
info!("entropy report → {path}");
}
// ── Shannon entropy distribution (`--shannon-summary`) ──────────────────────
if args.shannon_summary {
let path = args.output.as_ref()
.map(|p| format!("{}_shannon_summary.csv", p.display()))
.unwrap_or_else(|| "shannon_summary.csv".into());
info!("summarising per-family Shannon entropy");
let t = Stage::start("shannon_summary");
cache.shannon_entropy_summary(std::path::Path::new(&path)).unwrap_or_else(|e| {
eprintln!("error summarising Shannon entropy: {e}");
std::process::exit(1);
});
rep.push(t.stop());
info!("entropy summary → {path}");
}
// ── `--min-shared-family` auto-exclusion ────────────────────────────────────
// Layered on top of `--exclude-genome`, not instead of it — a separate
// mask (not folded into `exclude_mask` itself) since it must reach
@@ -270,7 +287,8 @@ pub fn run(args: PhyloArgs) {
// different sample per flag — see `obikphylo::siblings::LayerReducer`'s
// own module docs).
let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree;
let snp_kind = args.distance.as_snp();
let distance = distance_to_compute(args.distance, args.nj, args.upgma, args.shared_kmers);
let snp_kind = distance.and_then(DistanceArg::as_snp);
let need_alignment = args.pseudo_alignment || sankoff_requested;
let need_pairwise = sankoff_requested || snp_kind.is_some();
// `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional)
@@ -284,10 +302,20 @@ pub fn run(args: PhyloArgs) {
eprintln!("error: --sankoff requires --n-sites <N|all>");
std::process::exit(1);
}
if (args.sample_stats || args.sample_plan) && !(need_alignment || need_pairwise) {
eprintln!(
"error: --sample-stats/--sample-plan describe a draw: they need --pseudo-alignment, \
--sankoff or a snp-* --distance"
);
std::process::exit(1);
}
let mut alignment_reducer =
need_alignment.then(|| obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask));
let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n));
let mut stats_reducer =
args.sample_stats.then(|| obikphylo::siblings::SampleStatsReducer::new(n, &snp_exclude_mask));
let mut sample_plan = None;
if need_alignment || need_pairwise {
let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new();
@@ -297,6 +325,9 @@ pub fn run(args: PhyloArgs) {
if let Some(r) = pairwise_reducer.as_mut() {
reducers.push(r);
}
if let Some(r) = stats_reducer.as_mut() {
reducers.push(r);
}
info!(
"sampling SNP sites{}",
@@ -306,14 +337,24 @@ pub fn run(args: PhyloArgs) {
}
);
let t = Stage::start("sampled_scan");
obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers)
.unwrap_or_else(|e| {
eprintln!("error sampling SNP sites: {e}");
std::process::exit(1);
});
sample_plan = Some(
obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers)
.unwrap_or_else(|e| {
eprintln!("error sampling SNP sites: {e}");
std::process::exit(1);
}),
);
rep.push(t.stop());
}
// ── What the draw contains and how it was spread (`--sample-stats`/`--sample-plan`) ──
if let Some(reducer) = stats_reducer {
write_sample_stats(&reducer.finish(), &labels, &snp_exclude_mask, &args.output);
}
if args.sample_plan {
write_sample_plan(&sample_plan.expect("scanned above: need_alignment || need_pairwise"), &args.output);
}
// Finished once here (not left as reducers) so `--pseudo-alignment`
// and the Sankoff group can each read it below without fighting over
// which one consumes it — see `SankoffBundle::finish`'s own docs.
@@ -396,9 +437,17 @@ pub fn run(args: PhyloArgs) {
}
// ── Distance computation: classic whole-index metric vs. `snp-*` ───────────
// Nothing below (matrix, shared-kmer matrix, NJ/UPGMA trees) is wanted
// unless a distance was asked for (`--distance`) or implied
// (`--nj`/`--upgma`/`--shared-kmers`): stop here rather than compute and
// write a matrix nobody requested.
let Some(distance) = distance else {
rep.print();
return;
};
// Two genuinely different code paths behind one `--distance` value — see
// `args::DistanceArg`'s own docs.
let (matrix, shared_kmers) = match args.distance.as_classic() {
let (matrix, shared_kmers) = match distance.as_classic() {
Some(metric) => {
info!("computing {metric:?} distances for {n} genome(s)");
let need_shared = args.shared_kmers || args.nj || args.upgma;
+116
View File
@@ -0,0 +1,116 @@
// Diagnostic CSV outputs of a draw — `--sample-stats`, `--sample-plan` — kept
// out of `mod.rs` (already long) the way `tnt.rs`/`phyg.rs` hold their own
// writers. Each writer only formats what `obikphylo` computed; none of them
// decides anything.
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use obikphylo::siblings::{SamplePlan, SampleStats};
use tracing::info;
/// `<prefix>_<suffix>`, or `<suffix>` alone without `-o` — the convention of
/// every other file `phylo` writes.
fn output_path(output: &Option<PathBuf>, suffix: &str) -> String {
match output {
Some(prefix) => format!("{}_{suffix}", prefix.display()),
None => suffix.to_string(),
}
}
fn create(path: &str) -> BufWriter<std::fs::File> {
BufWriter::new(std::fs::File::create(path).unwrap_or_else(|e| {
eprintln!("error creating {path}: {e}");
std::process::exit(1);
}))
}
/// `--sample-stats`: three CSV files describing what the draw contains.
///
/// - `_sample_genomes.csv`: per genome, the sampled sites it has a call in,
/// their share of all sampled sites, and the informative sites in which it
/// takes part in the bipartition; `excluded` genomes are listed with zeros.
/// - `_sample_carriers.csv`: per number of carriers, the sampled sites and how
/// many of them are informative.
/// - `_sample_overlap.csv`: the sample's own genome × genome overlap matrix
/// (sampled sites both genomes have a call in), same layout as
/// `--family-overlap`, to set against the index's.
pub fn write_sample_stats(
stats: &SampleStats,
labels: &[String],
excluded: &[bool],
output: &Option<PathBuf>,
) {
let n = stats.n_genomes;
let path = output_path(output, "sample_genomes.csv");
let mut f = create(&path);
writeln!(f, "genome,excluded,sites_with_call,share_of_sites,informative_sites").unwrap();
for g in 0..n {
let share = if stats.n_sites == 0 { 0.0 } else { stats.calls[g] as f64 / stats.n_sites as f64 };
writeln!(
f,
"{},{},{},{:.6},{}",
labels[g],
excluded.get(g).copied().unwrap_or(false),
stats.calls[g],
share,
stats.informative_calls[g]
)
.unwrap();
}
info!("sample genomes → {path}");
let path = output_path(output, "sample_carriers.csv");
let mut f = create(&path);
writeln!(f, "n_carriers,sites,informative_sites").unwrap();
for (carriers, (&sites, &informative)) in
stats.sites_by_carriers.iter().zip(&stats.informative_by_carriers).enumerate()
{
if sites > 0 {
writeln!(f, "{carriers},{sites},{informative}").unwrap();
}
}
info!("sample carriers → {path}");
let path = output_path(output, "sample_overlap.csv");
let mut f = create(&path);
write!(f, "genome").unwrap();
for label in labels.iter().take(n) {
write!(f, ",{label}").unwrap();
}
writeln!(f).unwrap();
for i in 0..n {
write!(f, "{}", labels[i]).unwrap();
for j in 0..n {
write!(f, ",{}", stats.overlap(i, j)).unwrap();
}
writeln!(f).unwrap();
}
info!("sample overlap matrix → {path}");
}
/// `--sample-plan`: `_sample_plan.csv`, one line per layer in the order the
/// draw visited them — the families eligible, their total weight, the layer's
/// quota, how many families were drawn and resolved, and how many survived
/// masking. Only layer lines (a column sum is one `awk` away); the target and
/// the total kept are logged.
pub fn write_sample_plan(plan: &SamplePlan, output: &Option<PathBuf>) {
let path = output_path(output, "sample_plan.csv");
let mut f = create(&path);
writeln!(f, "partition,layer,eligible,mass,quota,resolved,kept").unwrap();
for layer in &plan.layers {
writeln!(
f,
"{},{},{},{:.6},{},{},{}",
layer.partition, layer.layer, layer.eligible, layer.mass, layer.quota, layer.resolved, layer.kept
)
.unwrap();
}
info!(
"sample plan ({} layer(s), target {}, kept {}) → {path}",
plan.layers.len(),
plan.target,
plan.kept()
);
}
@@ -0,0 +1,205 @@
//! The full, unsampled entropy scan behind `--shannon` and
//! `--shannon-summary`: every non-monomorphic minorant of every cached layer,
//! resolved once, with both entropies and the genome counts they were computed
//! over. [`scan_family_entropies`] is the one traversal; the two outputs differ
//! only in what they do with each [`FamilyEntropyRow`] — one row per family
//! (huge on a large index), or counts per bin ([`EntropySummary`], a few
//! hundred thousand rows at most whatever the index size).
use std::collections::{BTreeMap, HashSet};
use std::io::{self, Write};
use obikidxcache::index_cache::IndexCache;
use obikindex::{OKIError, OKIResult};
use obisys::progress_bar;
use super::entropy::{family_entropy, family_entropy_4};
use super::family_scan::{Selection, scan_layer_families};
use super::is_fast_mode;
use super::minorant_selection::non_monomorphic_reindex_layer;
/// One variable family, as `--shannon` reports it.
pub(crate) struct FamilyEntropyRow {
pub partition: usize,
pub layer: usize,
pub family_idx: usize,
pub family_size: usize,
/// Entropy over the 15 non-empty states, every genome carrying the family.
pub entropy15: f64,
/// Entropy over the 4 bases, genomes carrying exactly one member only;
/// `0.0` (and `n_unambiguous == 0`) when there is none.
pub entropy4: f64,
pub n_present: usize,
pub n_unambiguous: usize,
}
/// Calls `visit` once per non-monomorphic family of the index, layer by layer.
/// Needs only the sibling annex (`non_monomorphic_reindex_layer`): it
/// recomputes every value, so it neither reads nor requires the entropy,
/// quartet, overlap or pair-weight annexes. Bounded to non-monomorphic
/// minorants — monomorphism is already known from the annex bits alone (~98%
/// of a layer's minorants, measured on real data), so there is no reason to
/// pay `scan_layer_families`'s expensive cross-partition resolution for
/// families that would be skipped right away. A failing `visit` stops the scan.
pub(crate) fn scan_family_entropies(
cache: &IndexCache,
mut visit: impl FnMut(&FamilyEntropyRow) -> io::Result<()>,
) -> OKIResult<()> {
let n_genomes = cache.meta().genomes().len();
let fast_mode = is_fast_mode(cache);
let pb = progress_bar("shannon_entropy", cache.n_partition() as u64, "partitions");
for partition in cache.partitions() {
let n_layer = cache.n_layer(partition).unwrap_or(0);
for layer_idx in 0..n_layer {
let layer = cache
.get_layer(partition, layer_idx)
.expect("cache-consistent: n_layer(partition) already bounds layer_idx");
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
let eligible: HashSet<usize> = reindex.keys().copied().collect();
let mut failure = None;
scan_layer_families(
cache,
partition,
layer_idx,
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, mask, genome_mask| {
if failure.is_some() {
return;
}
debug_assert!(
mask.family_size() >= 2,
"Selection::Some(eligible) must only visit non-monomorphic minorants"
);
let Some((entropy15, n_present)) = family_entropy(genome_mask) else {
return;
};
let (entropy4, n_unambiguous) =
family_entropy_4(genome_mask).unwrap_or((0.0, 0));
let row = FamilyEntropyRow {
partition,
layer: layer_idx,
family_idx,
family_size: mask.family_size() as usize,
entropy15,
entropy4,
n_present,
n_unambiguous,
};
if let Err(e) = visit(&row) {
failure = Some(e);
}
},
)?;
if let Some(e) = failure {
return Err(OKIError::Io(e));
}
}
pb.inc(1);
}
pb.finish_and_clear();
Ok(())
}
/// Width, in bits, of an entropy bin of [`EntropySummary`].
pub(crate) const ENTROPY_BIN_WIDTH: f64 = 0.05;
/// Which entropy a summary line is about.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
enum Which {
Entropy15,
Entropy4,
}
/// Number of families per (entropy, genomes present, genomes without
/// ambiguity, entropy bin) — what `--shannon` rows reduce to once the
/// question is a distribution (entropy against the number of carriers, how
/// many carriers `--no-ambiguity` loses) and not a particular family.
#[derive(Default)]
pub(crate) struct EntropySummary {
counts: BTreeMap<(Which, usize, usize, usize), u64>,
}
impl EntropySummary {
fn bin(entropy: f64) -> usize {
(entropy / ENTROPY_BIN_WIDTH).floor() as usize
}
pub(crate) fn add(&mut self, row: &FamilyEntropyRow) {
*self
.counts
.entry((Which::Entropy15, row.n_present, row.n_unambiguous, Self::bin(row.entropy15)))
.or_default() += 1;
// `entropy4` is not defined without a genome carrying exactly one member
if row.n_unambiguous > 0 {
*self
.counts
.entry((Which::Entropy4, row.n_present, row.n_unambiguous, Self::bin(row.entropy4)))
.or_default() += 1;
}
}
pub(crate) fn write(&self, w: &mut impl Write) -> io::Result<()> {
writeln!(w, "entropy,n_genomes_present,n_genomes_unambiguous,bin_start,n_families")?;
for (&(which, present, unambiguous, bin), &count) in &self.counts {
let name = match which {
Which::Entropy15 => "entropy15",
Which::Entropy4 => "entropy4",
};
writeln!(
w,
"{name},{present},{unambiguous},{:.2},{count}",
bin as f64 * ENTROPY_BIN_WIDTH
)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn row(entropy15: f64, entropy4: f64, n_present: usize, n_unambiguous: usize) -> FamilyEntropyRow {
FamilyEntropyRow {
partition: 0,
layer: 0,
family_idx: 0,
family_size: 2,
entropy15,
entropy4,
n_present,
n_unambiguous,
}
}
fn written(summary: &EntropySummary) -> Vec<String> {
let mut buf = Vec::new();
summary.write(&mut buf).unwrap();
String::from_utf8(buf).unwrap().lines().map(str::to_owned).collect()
}
#[test]
fn families_are_counted_per_carrier_numbers_and_entropy_bin() {
let mut s = EntropySummary::default();
s.add(&row(1.0, 1.0, 4, 4));
s.add(&row(1.02, 1.02, 4, 4)); // same 0.05 bin as 1.0
s.add(&row(0.2, 0.2, 4, 4));
let lines = written(&s);
assert_eq!(lines[0], "entropy,n_genomes_present,n_genomes_unambiguous,bin_start,n_families");
assert!(lines.contains(&"entropy15,4,4,1.00,2".to_string()));
assert!(lines.contains(&"entropy15,4,4,0.20,1".to_string()));
assert!(lines.contains(&"entropy4,4,4,1.00,2".to_string()));
}
#[test]
fn entropy4_is_left_out_when_no_genome_has_a_single_member() {
let mut s = EntropySummary::default();
s.add(&row(0.0, 0.0, 1, 0));
let lines = written(&s);
assert!(lines.contains(&"entropy15,1,0,0.00,1".to_string()));
assert!(!lines.iter().any(|l| l.starts_with("entropy4")));
}
}
@@ -72,7 +72,7 @@ pub(crate) fn survives_masking(masked: &[Option<u8>], excluded: &[bool]) -> bool
/// genomes. That is the pattern that can separate two pairs of genomes from
/// each other (`AB|CD`); a singleton variant, or a family carried by fewer
/// than four genomes, fits every topology and informs none.
fn has_bipartition(counts: &[u32; 16]) -> bool {
pub(crate) fn has_bipartition(counts: &[u32; 16]) -> bool {
counts[1..].iter().filter(|&&c| c >= 2).count() >= 2
}
+5 -4
View File
@@ -17,6 +17,7 @@ mod alignment;
mod annex;
mod cardcomp;
mod entropy;
mod entropy_scan;
mod family_annexes;
mod family_scan;
mod masking;
@@ -24,6 +25,7 @@ mod minorant_selection;
mod pair_weight;
mod pairwise;
mod quartet;
mod sample_stats;
mod sankoff;
mod scan;
mod snp_distance;
@@ -41,17 +43,16 @@ pub use sankoff::SankoffBundle;
pub use scan::{LayerReducer, sampled_scan};
pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
pub use stats::SiblingAnnexStats;
pub use subsample::{EntropyBias, SamplingSpec, SurvivingFamily};
pub use sample_stats::{SampleStats, SampleStatsReducer};
pub use subsample::{EntropyBias, LayerReport, SamplePlan, SamplingSpec, SurvivingFamily};
pub(crate) use annex::build_layer_sibling_annex;
pub(crate) use entropy::{family_entropy, family_entropy_4};
pub(crate) use entropy_scan::{EntropySummary, scan_family_entropies};
pub(crate) use family_annexes::{
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
};
pub(crate) use pair_weight::inverse_overlap_matrix;
pub(crate) use quartet::restrict_to_quartet;
pub(crate) use family_scan::{Selection, scan_layer_families};
pub(crate) use minorant_selection::non_monomorphic_reindex_layer;
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
/// Whether every layer number in `cache` fits in a `FamilyMask` field
@@ -0,0 +1,164 @@
//! `--sample-stats`: what a draw actually contains, accumulated as a
//! [`LayerReducer`] over [`super::scan::sampled_scan`]'s batches, side by side
//! with the alignment, from the very same families — per genome (how many
//! sampled sites it has a call in, how many of them can resolve a
//! bipartition), per number of carriers (how many sampled sites, how many
//! informative), and per pair of genomes (how many sampled sites both have a
//! call in: the sample's own family-overlap matrix, to set against the index's).
//!
//! A *call* is a masked state other than the empty one (`0`): absence is never
//! a call, with or without `--free-loss`. A site is *informative* when its
//! carriers show a bipartition (`masking::has_bipartition`).
use super::masking::has_bipartition;
use super::scan::LayerReducer;
use super::subsample::SurvivingFamily;
use crate::siblings::helpers::triangle_index;
/// The statistics of one draw. `pub`: part of the public signature of the
/// `--sample-stats` output.
#[derive(Clone, Debug)]
pub struct SampleStats {
pub n_genomes: usize,
/// Number of sampled sites.
pub n_sites: u64,
/// Per genome: sampled sites it has a call in.
pub calls: Vec<u64>,
/// Per genome: informative sampled sites in which its state is one of the
/// two or more carried by at least two genomes — those where it takes
/// part in the bipartition.
pub informative_calls: Vec<u64>,
/// Index = number of carriers (`0..=n_genomes`): sampled sites.
pub sites_by_carriers: Vec<u64>,
/// Same, informative sites only.
pub informative_by_carriers: Vec<u64>,
/// Sampled sites both genomes of a pair have a call in, upper triangle in
/// the family-overlap annex's own order — read it with
/// [`overlap`](Self::overlap).
overlap: Vec<u64>,
}
impl SampleStats {
/// Sampled sites genomes `i` and `j` both have a call in (`0` for `i == j`,
/// as in `FamilyOverlap::get`).
pub fn overlap(&self, i: usize, j: usize) -> u64 {
if i == j {
0
} else {
self.overlap[triangle_index(self.n_genomes, i, j)]
}
}
}
/// Accumulates a [`SampleStats`] over a [`super::scan::sampled_scan`] pass.
/// `excluded[g]` genomes are not counted anywhere, as in the alignment.
pub struct SampleStatsReducer {
excluded: Vec<bool>,
stats: SampleStats,
}
impl SampleStatsReducer {
pub fn new(n_genomes: usize, excluded: &[bool]) -> Self {
let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2;
Self {
excluded: (0..n_genomes).map(|g| excluded.get(g).copied().unwrap_or(false)).collect(),
stats: SampleStats {
n_genomes,
n_sites: 0,
calls: vec![0; n_genomes],
informative_calls: vec![0; n_genomes],
sites_by_carriers: vec![0; n_genomes + 1],
informative_by_carriers: vec![0; n_genomes + 1],
overlap: vec![0; n_pairs],
},
}
}
pub fn finish(self) -> SampleStats {
self.stats
}
/// One sampled site, from its per-genome masked states.
fn add_site(&mut self, masked: &[Option<u8>]) {
let n = self.stats.n_genomes;
let mut counts = [0u32; 16];
let mut carriers: Vec<usize> = Vec::new();
for (g, &state) in masked.iter().enumerate() {
if let Some(state) = state.filter(|&s| s != 0 && !self.excluded[g]) {
counts[(state & 0b1111) as usize] += 1;
carriers.push(g);
}
}
let informative = has_bipartition(&counts);
self.stats.n_sites += 1;
self.stats.sites_by_carriers[carriers.len()] += 1;
if informative {
self.stats.informative_by_carriers[carriers.len()] += 1;
}
for (a, &i) in carriers.iter().enumerate() {
self.stats.calls[i] += 1;
if informative && counts[(masked[i].unwrap() & 0b1111) as usize] >= 2 {
self.stats.informative_calls[i] += 1;
}
for &j in &carriers[a + 1..] {
self.stats.overlap[triangle_index(n, i, j)] += 1;
}
}
}
}
impl LayerReducer for SampleStatsReducer {
fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
for family in survivors {
self.add_site(&family.masked);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn counts_calls_carriers_and_pairs() {
let mut r = SampleStatsReducer::new(5, &[]);
// informative: A A G G, genome 4 absent (state 0)
r.add_site(&[Some(1), Some(1), Some(4), Some(4), Some(0)]);
// not informative: three A and one lone G, genome 4 missing
r.add_site(&[Some(1), Some(1), Some(1), Some(4), None]);
let s = r.finish();
assert_eq!(s.n_sites, 2);
assert_eq!(s.calls, vec![2, 2, 2, 2, 0]);
assert_eq!(s.sites_by_carriers[4], 2);
assert_eq!(s.informative_by_carriers[4], 1);
// genome 3 is in the G pair of the informative site only
assert_eq!(s.informative_calls, vec![1, 1, 1, 1, 0]);
assert_eq!(s.overlap(0, 1), 2);
assert_eq!(s.overlap(2, 3), 2);
assert_eq!(s.overlap(0, 4), 0);
assert_eq!(s.overlap(1, 1), 0);
assert_eq!(s.overlap(3, 0), s.overlap(0, 3));
}
#[test]
fn excluded_genomes_count_nowhere() {
let mut r = SampleStatsReducer::new(4, &[false, false, false, true]);
r.add_site(&[Some(1), Some(1), Some(4), Some(4)]);
let s = r.finish();
// without genome 3 the G state has a single carrier: not informative
assert_eq!(s.calls, vec![1, 1, 1, 0]);
assert_eq!(s.sites_by_carriers[3], 1);
assert_eq!(s.informative_by_carriers[3], 0);
assert_eq!(s.overlap(2, 3), 0);
}
#[test]
fn genome_with_a_lone_state_takes_no_part_in_the_bipartition() {
let mut r = SampleStatsReducer::new(5, &[]);
// A A C C G: genome 4 is the lone G
r.add_site(&[Some(1), Some(1), Some(2), Some(2), Some(4)]);
let s = r.finish();
assert_eq!(s.informative_calls, vec![1, 1, 1, 1, 0]);
assert_eq!(s.calls[4], 1);
}
}
@@ -14,7 +14,7 @@ use obikidxcache::index_cache::IndexCache;
use obikindex::OKIResult;
use super::stats::sibling_family_size_histogram;
use super::subsample::{SamplingSpec, SurvivingFamily, sample_index};
use super::subsample::{SamplePlan, SamplingSpec, SurvivingFamily, sample_index};
/// One accumulator fed every layer's [`SurvivingFamily`] batch during a
/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of
@@ -47,22 +47,22 @@ fn resolve_target(cache: &IndexCache, n: Option<usize>) -> OKIResult<usize> {
/// own docs and [`SamplingSpec`] for what each controls), feeding every
/// surviving family of every visited layer to every reducer in
/// `reducers`, in the same order, before moving to the next layer. A
/// no-op (not an error) if the resolved target is `0`.
/// no-op (not an error) if the resolved target is `0`. Returns how the draw
/// was spread over the layers ([`SamplePlan`]).
pub fn sampled_scan(
cache: &IndexCache,
n: Option<usize>,
spec: SamplingSpec,
excluded: &[bool],
reducers: &mut [&mut dyn LayerReducer],
) -> OKIResult<()> {
) -> OKIResult<SamplePlan> {
let target = resolve_target(cache, n)?;
if target == 0 {
return Ok(());
return Ok(SamplePlan::default());
}
sample_index(cache, target, spec, excluded, 0, |partition, layer, _raw_index, survivors| {
for reducer in reducers.iter_mut() {
reducer.reduce(partition, layer, &survivors);
}
})?;
Ok(())
})
}
@@ -115,6 +115,41 @@ pub struct SamplingSpec {
pub entropy_bias: Option<EntropyBias>,
}
/// What one layer contributed to a draw — one row of `--sample-plan`.
#[derive(Clone, Debug)]
pub struct LayerReport {
pub partition: usize,
pub layer: usize,
/// Families eligible when the draw starts (non-monomorphic, and
/// quartet-informative with `--quartet`).
pub eligible: u64,
/// Their total sampling weight: their number, or with `--pair-balance` the
/// sum of their pair weights.
pub mass: f64,
/// The layer's share of the target.
pub quota: usize,
/// Families drawn and resolved.
pub resolved: usize,
/// Of those, the ones that survived masking (and `--quartet`'s exact check).
pub kept: usize,
}
/// How a draw was spread over the layers of the index.
#[derive(Clone, Debug, Default)]
pub struct SamplePlan {
/// The number of sites asked for.
pub target: usize,
/// One report per layer, in the order the draw visited them.
pub layers: Vec<LayerReport>,
}
impl SamplePlan {
/// Sites actually kept, all layers together.
pub fn kept(&self) -> usize {
self.layers.iter().map(|layer| layer.kept).sum()
}
}
/// [`iter_full_scalar`] of `kind` (entropy, pair weight), rotated to start at
/// `start` and wrap once back to it — the same `family_idx` order
/// [`sample_layer`]'s own circular index sequence walks, so the streams can be
@@ -143,6 +178,8 @@ struct LayerPlan {
/// One bit per minorant (`family_idx` order): still eligible to be drawn.
eligible: TempBitVecBuilder,
n_minorants: usize,
/// How many of those bits are set when the draw starts.
eligible_count: u64,
/// Total sampling weight of the eligible families: their number, or, with
/// `--pair-balance`, the sum of their pair weights. A layer's quota is its
/// share of this mass over every layer, so a layer rich in families that
@@ -191,10 +228,10 @@ struct LayerPlan {
/// the seam a future resume mechanism would hook into, rather than being
/// ripped out and re-added later.
///
/// Returns the actual number of *newly processed* sites kept (not
/// counting whatever `skip_layers` layers already contributed on an
/// earlier, interrupted call) — may be less than `n` (see the module
/// docs).
/// Returns how the draw was spread over the layers ([`SamplePlan`]): one
/// report per layer, with its quota and what it actually kept (nothing for the
/// `skip_layers` layers an earlier, interrupted call already covered). The
/// sum of what was kept may be less than `n` (see the module docs).
pub(crate) fn sample_index(
cache: &IndexCache,
n: usize,
@@ -202,7 +239,7 @@ pub(crate) fn sample_index(
excluded: &[bool],
skip_layers: usize,
mut on_layer: impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
) -> OKIResult<usize> {
) -> OKIResult<SamplePlan> {
let n_genomes = cache.meta().genomes().len();
let fast_mode = is_fast_mode(cache);
@@ -237,6 +274,7 @@ pub(crate) fn sample_index(
restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?;
}
let n_minorants = eligible.view().len();
let eligible_count = eligible.view().count_ones();
let mass = if spec.pair_balance {
iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)?
.enumerate()
@@ -244,15 +282,23 @@ pub(crate) fn sample_index(
.map(|(_, weight)| f64::from(weight))
.sum()
} else {
eligible.view().count_ones() as f64
eligible_count as f64
};
plans.push(LayerPlan { partition, layer: layer_idx, eligible, n_minorants, mass });
plans.push(LayerPlan {
partition,
layer: layer_idx,
eligible,
n_minorants,
eligible_count,
mass,
});
}
}
let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum();
let mut report = SamplePlan { target: n, layers: Vec::with_capacity(plans.len()) };
if total_mass <= 0.0 {
return Ok(0);
return Ok(report);
}
// Quotas by cumulative rounding: layer `i` is allotted
@@ -260,7 +306,6 @@ pub(crate) fn sample_index(
// were allotted`, so the quotas sum to exactly `n`. Rounding each layer's
// share down on its own would lose up to one site per layer — all of
// them, when `n` is smaller than the number of layers.
let mut total_kept = 0usize;
let mut cumulative_mass = 0.0f64;
let mut allotted = 0usize;
for (raw_index, mut plan) in plans.into_iter().enumerate() {
@@ -268,26 +313,37 @@ pub(crate) fn sample_index(
let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize;
let quota = cumulative_quota.saturating_sub(allotted);
allotted = allotted.max(cumulative_quota);
if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 {
continue;
}
total_kept += sample_layer(
cache,
raw_index,
&mut plan,
let (resolved, kept) = if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 {
(0, 0)
} else {
sample_layer(
cache,
raw_index,
&mut plan,
quota,
n_genomes,
fast_mode,
spec,
excluded,
&mut on_layer,
)?
};
report.layers.push(LayerReport {
partition: plan.partition,
layer: plan.layer,
eligible: plan.eligible_count,
mass: plan.mass,
quota,
n_genomes,
fast_mode,
spec,
excluded,
&mut on_layer,
)?;
resolved,
kept,
});
}
Ok(total_kept)
Ok(report)
}
/// One layer's share of [`sample_index`] — see the module docs for the
/// One layer's share of [`sample_index`], as `(families drawn and resolved,
/// families kept after masking)` — see the module docs for the
/// per-family accept/reject/mark rules and the round-batching scheme. Each
/// eligible family is accepted with probability
/// `quota / mass × pair weight × entropy kernel` (capped at 1): the pair
@@ -303,11 +359,11 @@ fn sample_layer(
spec: SamplingSpec,
excluded: &[bool],
on_layer: &mut impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
) -> OKIResult<usize> {
) -> OKIResult<(usize, usize)> {
let LayerPlan { partition, layer: layer_idx, n_minorants, mass, .. } = *plan;
let eligible = &mut plan.eligible;
if n_minorants == 0 || mass <= 0.0 {
return Ok(0);
return Ok((0, 0));
}
let p0 = quota as f64 / mass;
let layer_dir = cache
@@ -435,5 +491,5 @@ fn sample_layer(
on_layer(partition, layer_idx, raw_index, Arc::new(survivors));
}
Ok(kept)
Ok((resolved, kept))
}
@@ -14,9 +14,8 @@ use obikindex::{OKIError, OKIResult};
use obisys::progress_bar;
use crate::siblings::algorithms::{
AnnexRequest, Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy,
family_entropy_4, is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families,
sibling_annex_stats, sibling_family_size_histogram,
AnnexRequest, EntropySummary, SiblingAnnexStats, build_layer_sibling_annex, is_fast_mode,
scan_family_entropies, sibling_annex_stats, sibling_family_size_histogram,
};
use crate::siblings::extensions::SiblingBuilder;
use crate::siblings::{
@@ -66,6 +65,18 @@ pub trait SiblingExt {
/// `--entropy-bias` yet (see the project memory on this).
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>;
/// The same scan as [`shannon_entropy_csv`](Self::shannon_entropy_csv), but
/// written as a distribution instead of one row per family: a CSV
/// (`entropy,n_genomes_present,n_genomes_unambiguous,bin_start,n_families`)
/// counting the families per `entropy15` (all carriers) or `entropy4`
/// (single-member genomes; families with none are left out of its lines),
/// number of genomes present, number of those without ambiguity, and
/// 0.05-bit entropy bin. At most a few hundred thousand lines whatever the
/// size of the index — enough to answer how entropy varies with the number
/// of carriers, or how many carriers `--no-ambiguity` loses, without the
/// tens of gigabytes the per-family file takes.
fn shannon_entropy_summary(&self, path: &Path) -> OKIResult<()>;
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
/// no `per_genome`) — reads only the already-built annex
/// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits,
@@ -175,72 +186,35 @@ impl SiblingExt for IndexCache {
}
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {
// Needs only the sibling annex (`non_monomorphic_reindex_layer`
// below): this method runs its own, differently-scoped scan and
// recomputes every value it writes, so it neither reads nor
// requires the entropy/family-overlap annexes.
let n_genomes = self.meta().genomes().len();
let fast_mode = is_fast_mode(self);
let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?);
writeln!(f, "partition,layer,family_idx,entropy15,entropy4,family_size,n_genomes_present,n_genomes_unambiguous")
.map_err(OKIError::Io)?;
scan_family_entropies(self, |row| {
writeln!(
f,
"{},{},{},{:.6},{:.6},{},{},{}",
row.partition,
row.layer,
row.family_idx,
row.entropy15,
row.entropy4,
row.family_size,
row.n_present,
row.n_unambiguous
)
})?;
f.flush().map_err(OKIError::Io)
}
let pb = progress_bar("shannon_entropy", self.n_partition() as u64, "partitions");
for part in self.partitions() {
let n_layer = self.n_layer(part).unwrap_or(0);
for l in 0..n_layer {
let layer = self
.get_layer(part, l)
.expect("cache-consistent: n_layer(part) already bounds l");
// Bounded to non-monomorphic minorants only, same reasoning
// as `ensure_layer_entropy_annex`'s own docs: monomorphism
// is already known from the annex bits alone (~98% of a
// layer's minorants, measured on real data), so there is no
// reason to pay `scan_layer_families`'s expensive
// cross-partition resolution for families this method
// skips right below anyway.
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
let eligible: std::collections::HashSet<usize> = reindex.keys().copied().collect();
let mut write_err = None;
scan_layer_families(
self,
part,
l,
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, mask, genome_mask| {
if write_err.is_some() {
return;
}
debug_assert!(
mask.family_size() >= 2,
"Selection::Some(eligible) must only visit non-monomorphic minorants"
);
let Some((h15, n_present)) = family_entropy(genome_mask) else {
return;
};
let (h4, n_unambiguous) = family_entropy_4(genome_mask).unwrap_or((0.0, 0));
if let Err(e) = writeln!(
f,
"{part},{l},{family_idx},{h15:.6},{h4:.6},{},{n_present},{n_unambiguous}",
mask.family_size()
) {
write_err = Some(e);
}
},
)?;
if let Some(e) = write_err {
return Err(OKIError::Io(e));
}
}
pb.inc(1);
}
pb.finish_and_clear();
Ok(())
fn shannon_entropy_summary(&self, path: &Path) -> OKIResult<()> {
let mut summary = EntropySummary::default();
scan_family_entropies(self, |row| {
summary.add(row);
Ok(())
})?;
let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?);
summary.write(&mut f).map_err(OKIError::Io)?;
f.flush().map_err(OKIError::Io)
}
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> {
+2 -2
View File
@@ -42,8 +42,8 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use algorithms::{
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
PairwiseReducer, RawSnpDistanceOutput, SamplingSpec, SankoffBundle, SiblingAnnexStats,
SnpAlignment,
LayerReport, PairwiseReducer, RawSnpDistanceOutput, SamplePlan, SampleStats,
SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SnpAlignment,
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
pairwise_cost_matrix, sampled_scan,
};