Refactor sampling logic and introduce diagnostic outputs
This commit introduces new features for sampling, including detailed `SamplePlan` generation, Shannon entropy summaries, and site statistics reporting. It formalizes conditional distance matrix computation and clarifies the logic for family weighting and site selection.
This commit is contained in:
1 parent
2232628e58
commit
cb5fc202b6
13 files changed
+799
-122
No files matched your search
@@ -101,6 +101,51 @@ pub enum DistanceArg {
|
||||
SnpTv,
|
||||
}
|
||||
|
||||
/// The distance matrix this invocation must compute, if any: the one asked for
|
||||
/// with `--distance`, else `jaccard` when `--nj`, `--upgma` or
|
||||
/// `--shared-kmers` needs a matrix to work from, else none — a matrix is never
|
||||
/// computed (nor written) just because nothing said otherwise.
|
||||
pub fn distance_to_compute(
|
||||
distance: Option<DistanceArg>,
|
||||
nj: bool,
|
||||
upgma: bool,
|
||||
shared_kmers: bool,
|
||||
) -> Option<DistanceArg> {
|
||||
distance.or_else(|| (nj || upgma || shared_kmers).then_some(DistanceArg::Jaccard))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod distance_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn nothing_asked_nothing_computed() {
|
||||
assert!(distance_to_compute(None, false, false, false).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_and_shared_kmer_outputs_imply_jaccard() {
|
||||
for (nj, upgma, shared) in [(true, false, false), (false, true, false), (false, false, true)] {
|
||||
assert!(matches!(
|
||||
distance_to_compute(None, nj, upgma, shared),
|
||||
Some(DistanceArg::Jaccard)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_distance_wins() {
|
||||
assert!(matches!(
|
||||
distance_to_compute(Some(DistanceArg::Mash), true, true, true),
|
||||
Some(DistanceArg::Mash)
|
||||
));
|
||||
assert!(matches!(
|
||||
distance_to_compute(Some(DistanceArg::SnpJc), false, false, false),
|
||||
Some(DistanceArg::SnpJc)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
impl DistanceArg {
|
||||
/// `Some` for the whole-index metrics, `None` for `snp-*` values.
|
||||
pub fn as_classic(self) -> Option<DistanceMetric> {
|
||||
@@ -215,6 +260,15 @@ pub struct PhyloArgs {
|
||||
#[arg(long)]
|
||||
pub shannon: bool,
|
||||
|
||||
/// Write the same entropy scan as `--shannon` as a distribution
|
||||
/// (`<prefix>_shannon_summary.csv`): the number of families per entropy
|
||||
/// kind, number of genomes carrying the family, number of those without
|
||||
/// ambiguity and 0.05-bit entropy bin — a few hundred thousand lines at
|
||||
/// most, whatever the size of the index, instead of one line per family.
|
||||
/// Requires an already-built sibling annex, like `--shannon`.
|
||||
#[arg(long)]
|
||||
pub shannon_summary: bool,
|
||||
|
||||
/// Write a SNP-only pseudo-alignment (FASTA) — requires
|
||||
/// `--n-sites <N|all>` and an already-built sibling annex
|
||||
/// (`--sibling-annex` first, in this invocation or an earlier one).
|
||||
@@ -260,6 +314,24 @@ pub struct PhyloArgs {
|
||||
#[arg(long)]
|
||||
pub pair_balance: bool,
|
||||
|
||||
/// Write what the draw contains, from the very sites it kept
|
||||
/// (requires `--pseudo-alignment`, `--sankoff` or a `snp-*`
|
||||
/// `--distance`): `<prefix>_sample_genomes.csv` (per genome, the sites it
|
||||
/// has a call in and the informative sites it takes part in),
|
||||
/// `<prefix>_sample_carriers.csv` (sites and informative sites per number
|
||||
/// of carriers) and `<prefix>_sample_overlap.csv` (the sample's own
|
||||
/// genome × genome overlap matrix, to compare with `--family-overlap`).
|
||||
#[arg(long)]
|
||||
pub sample_stats: bool,
|
||||
|
||||
/// Write how the draw was spread over the layers of the index
|
||||
/// (requires `--pseudo-alignment`, `--sankoff` or a `snp-*`
|
||||
/// `--distance`): `<prefix>_sample_plan.csv`, one line per layer with the
|
||||
/// families eligible, their total weight, the layer's quota, the families
|
||||
/// drawn and the ones kept after masking.
|
||||
#[arg(long)]
|
||||
pub sample_plan: bool,
|
||||
|
||||
/// Entropy-biased sampling target (Gaussian kernel mean) for
|
||||
/// `--pseudo-alignment` — activates biasing as soon as this or
|
||||
/// `--entropy-sd` is given; the other defaults to 1.0/0.5.
|
||||
@@ -371,7 +443,10 @@ pub struct PhyloArgs {
|
||||
#[arg(long, default_value = "100")]
|
||||
pub sankoff_cost_scale: f64,
|
||||
|
||||
/// Distance to compute — either a whole-index metric (`jaccard`,
|
||||
/// Distance matrix to compute and write — none is computed unless this
|
||||
/// is given, or `--nj`/`--upgma`/`--shared-kmers` is (they need a matrix
|
||||
/// and use `jaccard` then, see [`distance_to_compute`]). Either a
|
||||
/// whole-index metric (`jaccard`,
|
||||
/// `mash`, `hamming`, `bray-curtis`, ...) or a `snp-*` correction over
|
||||
/// the central-position SNP substitution spectrum (`snp-raw`, `snp-jc`,
|
||||
/// `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv`) —
|
||||
@@ -381,8 +456,8 @@ pub struct PhyloArgs {
|
||||
/// `DevDocMD/theory/evolutionary_distances.md`, "`--distance`
|
||||
/// unification" for the full catalog and why LogDet/Tajima-Nei/F84/
|
||||
/// HKY85 aren't offered yet).
|
||||
#[arg(long, value_enum, default_value = "jaccard")]
|
||||
pub distance: DistanceArg,
|
||||
#[arg(long, value_enum)]
|
||||
pub distance: Option<DistanceArg>,
|
||||
|
||||
/// Rate-heterogeneity correction (Jin-Nei gamma shape parameter `α`)
|
||||
/// for `snp-*` `--distance` values that support it
|
||||
|
||||
@@ -3,6 +3,7 @@ mod iqtree;
|
||||
mod phyg;
|
||||
mod phylip;
|
||||
mod sankoff;
|
||||
mod stats;
|
||||
mod tnt;
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
@@ -18,9 +19,10 @@ use obikphylo::{Metrics, neighbor_joining, upgma};
|
||||
use obisys::{Reporter, Stage};
|
||||
use tracing::info;
|
||||
|
||||
use args::NSitesArg;
|
||||
use args::{DistanceArg, NSitesArg, distance_to_compute};
|
||||
use iqtree::write_iqtree;
|
||||
use phyg::write_sankoff_phyg;
|
||||
use stats::{write_sample_plan, write_sample_stats};
|
||||
use phylip::write_phylip_relaxed;
|
||||
use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
|
||||
use tnt::write_sankoff_tnt;
|
||||
@@ -194,6 +196,21 @@ pub fn run(args: PhyloArgs) {
|
||||
info!("entropy report → {path}");
|
||||
}
|
||||
|
||||
// ── Shannon entropy distribution (`--shannon-summary`) ──────────────────────
|
||||
if args.shannon_summary {
|
||||
let path = args.output.as_ref()
|
||||
.map(|p| format!("{}_shannon_summary.csv", p.display()))
|
||||
.unwrap_or_else(|| "shannon_summary.csv".into());
|
||||
info!("summarising per-family Shannon entropy");
|
||||
let t = Stage::start("shannon_summary");
|
||||
cache.shannon_entropy_summary(std::path::Path::new(&path)).unwrap_or_else(|e| {
|
||||
eprintln!("error summarising Shannon entropy: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
rep.push(t.stop());
|
||||
info!("entropy summary → {path}");
|
||||
}
|
||||
|
||||
// ── `--min-shared-family` auto-exclusion ────────────────────────────────────
|
||||
// Layered on top of `--exclude-genome`, not instead of it — a separate
|
||||
// mask (not folded into `exclude_mask` itself) since it must reach
|
||||
@@ -270,7 +287,8 @@ pub fn run(args: PhyloArgs) {
|
||||
// different sample per flag — see `obikphylo::siblings::LayerReducer`'s
|
||||
// own module docs).
|
||||
let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree;
|
||||
let snp_kind = args.distance.as_snp();
|
||||
let distance = distance_to_compute(args.distance, args.nj, args.upgma, args.shared_kmers);
|
||||
let snp_kind = distance.and_then(DistanceArg::as_snp);
|
||||
let need_alignment = args.pseudo_alignment || sankoff_requested;
|
||||
let need_pairwise = sankoff_requested || snp_kind.is_some();
|
||||
// `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional)
|
||||
@@ -284,10 +302,20 @@ pub fn run(args: PhyloArgs) {
|
||||
eprintln!("error: --sankoff requires --n-sites <N|all>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
if (args.sample_stats || args.sample_plan) && !(need_alignment || need_pairwise) {
|
||||
eprintln!(
|
||||
"error: --sample-stats/--sample-plan describe a draw: they need --pseudo-alignment, \
|
||||
--sankoff or a snp-* --distance"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let mut alignment_reducer =
|
||||
need_alignment.then(|| obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask));
|
||||
let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n));
|
||||
let mut stats_reducer =
|
||||
args.sample_stats.then(|| obikphylo::siblings::SampleStatsReducer::new(n, &snp_exclude_mask));
|
||||
let mut sample_plan = None;
|
||||
|
||||
if need_alignment || need_pairwise {
|
||||
let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new();
|
||||
@@ -297,6 +325,9 @@ pub fn run(args: PhyloArgs) {
|
||||
if let Some(r) = pairwise_reducer.as_mut() {
|
||||
reducers.push(r);
|
||||
}
|
||||
if let Some(r) = stats_reducer.as_mut() {
|
||||
reducers.push(r);
|
||||
}
|
||||
|
||||
info!(
|
||||
"sampling SNP sites{}",
|
||||
@@ -306,14 +337,24 @@ pub fn run(args: PhyloArgs) {
|
||||
}
|
||||
);
|
||||
let t = Stage::start("sampled_scan");
|
||||
obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("error sampling SNP sites: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
sample_plan = Some(
|
||||
obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("error sampling SNP sites: {e}");
|
||||
std::process::exit(1);
|
||||
}),
|
||||
);
|
||||
rep.push(t.stop());
|
||||
}
|
||||
|
||||
// ── What the draw contains and how it was spread (`--sample-stats`/`--sample-plan`) ──
|
||||
if let Some(reducer) = stats_reducer {
|
||||
write_sample_stats(&reducer.finish(), &labels, &snp_exclude_mask, &args.output);
|
||||
}
|
||||
if args.sample_plan {
|
||||
write_sample_plan(&sample_plan.expect("scanned above: need_alignment || need_pairwise"), &args.output);
|
||||
}
|
||||
|
||||
// Finished once here (not left as reducers) so `--pseudo-alignment`
|
||||
// and the Sankoff group can each read it below without fighting over
|
||||
// which one consumes it — see `SankoffBundle::finish`'s own docs.
|
||||
@@ -396,9 +437,17 @@ pub fn run(args: PhyloArgs) {
|
||||
}
|
||||
|
||||
// ── Distance computation: classic whole-index metric vs. `snp-*` ───────────
|
||||
// Nothing below (matrix, shared-kmer matrix, NJ/UPGMA trees) is wanted
|
||||
// unless a distance was asked for (`--distance`) or implied
|
||||
// (`--nj`/`--upgma`/`--shared-kmers`): stop here rather than compute and
|
||||
// write a matrix nobody requested.
|
||||
let Some(distance) = distance else {
|
||||
rep.print();
|
||||
return;
|
||||
};
|
||||
// Two genuinely different code paths behind one `--distance` value — see
|
||||
// `args::DistanceArg`'s own docs.
|
||||
let (matrix, shared_kmers) = match args.distance.as_classic() {
|
||||
let (matrix, shared_kmers) = match distance.as_classic() {
|
||||
Some(metric) => {
|
||||
info!("computing {metric:?} distances for {n} genome(s)");
|
||||
let need_shared = args.shared_kmers || args.nj || args.upgma;
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
// Diagnostic CSV outputs of a draw — `--sample-stats`, `--sample-plan` — kept
|
||||
// out of `mod.rs` (already long) the way `tnt.rs`/`phyg.rs` hold their own
|
||||
// writers. Each writer only formats what `obikphylo` computed; none of them
|
||||
// decides anything.
|
||||
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use obikphylo::siblings::{SamplePlan, SampleStats};
|
||||
use tracing::info;
|
||||
|
||||
/// `<prefix>_<suffix>`, or `<suffix>` alone without `-o` — the convention of
|
||||
/// every other file `phylo` writes.
|
||||
fn output_path(output: &Option<PathBuf>, suffix: &str) -> String {
|
||||
match output {
|
||||
Some(prefix) => format!("{}_{suffix}", prefix.display()),
|
||||
None => suffix.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create(path: &str) -> BufWriter<std::fs::File> {
|
||||
BufWriter::new(std::fs::File::create(path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}))
|
||||
}
|
||||
|
||||
/// `--sample-stats`: three CSV files describing what the draw contains.
|
||||
///
|
||||
/// - `_sample_genomes.csv`: per genome, the sampled sites it has a call in,
|
||||
/// their share of all sampled sites, and the informative sites in which it
|
||||
/// takes part in the bipartition; `excluded` genomes are listed with zeros.
|
||||
/// - `_sample_carriers.csv`: per number of carriers, the sampled sites and how
|
||||
/// many of them are informative.
|
||||
/// - `_sample_overlap.csv`: the sample's own genome × genome overlap matrix
|
||||
/// (sampled sites both genomes have a call in), same layout as
|
||||
/// `--family-overlap`, to set against the index's.
|
||||
pub fn write_sample_stats(
|
||||
stats: &SampleStats,
|
||||
labels: &[String],
|
||||
excluded: &[bool],
|
||||
output: &Option<PathBuf>,
|
||||
) {
|
||||
let n = stats.n_genomes;
|
||||
|
||||
let path = output_path(output, "sample_genomes.csv");
|
||||
let mut f = create(&path);
|
||||
writeln!(f, "genome,excluded,sites_with_call,share_of_sites,informative_sites").unwrap();
|
||||
for g in 0..n {
|
||||
let share = if stats.n_sites == 0 { 0.0 } else { stats.calls[g] as f64 / stats.n_sites as f64 };
|
||||
writeln!(
|
||||
f,
|
||||
"{},{},{},{:.6},{}",
|
||||
labels[g],
|
||||
excluded.get(g).copied().unwrap_or(false),
|
||||
stats.calls[g],
|
||||
share,
|
||||
stats.informative_calls[g]
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
info!("sample genomes → {path}");
|
||||
|
||||
let path = output_path(output, "sample_carriers.csv");
|
||||
let mut f = create(&path);
|
||||
writeln!(f, "n_carriers,sites,informative_sites").unwrap();
|
||||
for (carriers, (&sites, &informative)) in
|
||||
stats.sites_by_carriers.iter().zip(&stats.informative_by_carriers).enumerate()
|
||||
{
|
||||
if sites > 0 {
|
||||
writeln!(f, "{carriers},{sites},{informative}").unwrap();
|
||||
}
|
||||
}
|
||||
info!("sample carriers → {path}");
|
||||
|
||||
let path = output_path(output, "sample_overlap.csv");
|
||||
let mut f = create(&path);
|
||||
write!(f, "genome").unwrap();
|
||||
for label in labels.iter().take(n) {
|
||||
write!(f, ",{label}").unwrap();
|
||||
}
|
||||
writeln!(f).unwrap();
|
||||
for i in 0..n {
|
||||
write!(f, "{}", labels[i]).unwrap();
|
||||
for j in 0..n {
|
||||
write!(f, ",{}", stats.overlap(i, j)).unwrap();
|
||||
}
|
||||
writeln!(f).unwrap();
|
||||
}
|
||||
info!("sample overlap matrix → {path}");
|
||||
}
|
||||
|
||||
/// `--sample-plan`: `_sample_plan.csv`, one line per layer in the order the
|
||||
/// draw visited them — the families eligible, their total weight, the layer's
|
||||
/// quota, how many families were drawn and resolved, and how many survived
|
||||
/// masking. Only layer lines (a column sum is one `awk` away); the target and
|
||||
/// the total kept are logged.
|
||||
pub fn write_sample_plan(plan: &SamplePlan, output: &Option<PathBuf>) {
|
||||
let path = output_path(output, "sample_plan.csv");
|
||||
let mut f = create(&path);
|
||||
writeln!(f, "partition,layer,eligible,mass,quota,resolved,kept").unwrap();
|
||||
for layer in &plan.layers {
|
||||
writeln!(
|
||||
f,
|
||||
"{},{},{},{:.6},{},{},{}",
|
||||
layer.partition, layer.layer, layer.eligible, layer.mass, layer.quota, layer.resolved, layer.kept
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
info!(
|
||||
"sample plan ({} layer(s), target {}, kept {}) → {path}",
|
||||
plan.layers.len(),
|
||||
plan.target,
|
||||
plan.kept()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
//! The full, unsampled entropy scan behind `--shannon` and
|
||||
//! `--shannon-summary`: every non-monomorphic minorant of every cached layer,
|
||||
//! resolved once, with both entropies and the genome counts they were computed
|
||||
//! over. [`scan_family_entropies`] is the one traversal; the two outputs differ
|
||||
//! only in what they do with each [`FamilyEntropyRow`] — one row per family
|
||||
//! (huge on a large index), or counts per bin ([`EntropySummary`], a few
|
||||
//! hundred thousand rows at most whatever the index size).
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::io::{self, Write};
|
||||
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obisys::progress_bar;
|
||||
|
||||
use super::entropy::{family_entropy, family_entropy_4};
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::is_fast_mode;
|
||||
use super::minorant_selection::non_monomorphic_reindex_layer;
|
||||
|
||||
/// One variable family, as `--shannon` reports it.
|
||||
pub(crate) struct FamilyEntropyRow {
|
||||
pub partition: usize,
|
||||
pub layer: usize,
|
||||
pub family_idx: usize,
|
||||
pub family_size: usize,
|
||||
/// Entropy over the 15 non-empty states, every genome carrying the family.
|
||||
pub entropy15: f64,
|
||||
/// Entropy over the 4 bases, genomes carrying exactly one member only;
|
||||
/// `0.0` (and `n_unambiguous == 0`) when there is none.
|
||||
pub entropy4: f64,
|
||||
pub n_present: usize,
|
||||
pub n_unambiguous: usize,
|
||||
}
|
||||
|
||||
/// Calls `visit` once per non-monomorphic family of the index, layer by layer.
|
||||
/// Needs only the sibling annex (`non_monomorphic_reindex_layer`): it
|
||||
/// recomputes every value, so it neither reads nor requires the entropy,
|
||||
/// quartet, overlap or pair-weight annexes. Bounded to non-monomorphic
|
||||
/// minorants — monomorphism is already known from the annex bits alone (~98%
|
||||
/// of a layer's minorants, measured on real data), so there is no reason to
|
||||
/// pay `scan_layer_families`'s expensive cross-partition resolution for
|
||||
/// families that would be skipped right away. A failing `visit` stops the scan.
|
||||
pub(crate) fn scan_family_entropies(
|
||||
cache: &IndexCache,
|
||||
mut visit: impl FnMut(&FamilyEntropyRow) -> io::Result<()>,
|
||||
) -> OKIResult<()> {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(cache);
|
||||
let pb = progress_bar("shannon_entropy", cache.n_partition() as u64, "partitions");
|
||||
for partition in cache.partitions() {
|
||||
let n_layer = cache.n_layer(partition).unwrap_or(0);
|
||||
for layer_idx in 0..n_layer {
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: n_layer(partition) already bounds layer_idx");
|
||||
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
|
||||
let eligible: HashSet<usize> = reindex.keys().copied().collect();
|
||||
|
||||
let mut failure = None;
|
||||
scan_layer_families(
|
||||
cache,
|
||||
partition,
|
||||
layer_idx,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, mask, genome_mask| {
|
||||
if failure.is_some() {
|
||||
return;
|
||||
}
|
||||
debug_assert!(
|
||||
mask.family_size() >= 2,
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
let Some((entropy15, n_present)) = family_entropy(genome_mask) else {
|
||||
return;
|
||||
};
|
||||
let (entropy4, n_unambiguous) =
|
||||
family_entropy_4(genome_mask).unwrap_or((0.0, 0));
|
||||
let row = FamilyEntropyRow {
|
||||
partition,
|
||||
layer: layer_idx,
|
||||
family_idx,
|
||||
family_size: mask.family_size() as usize,
|
||||
entropy15,
|
||||
entropy4,
|
||||
n_present,
|
||||
n_unambiguous,
|
||||
};
|
||||
if let Err(e) = visit(&row) {
|
||||
failure = Some(e);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
if let Some(e) = failure {
|
||||
return Err(OKIError::Io(e));
|
||||
}
|
||||
}
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Width, in bits, of an entropy bin of [`EntropySummary`].
|
||||
pub(crate) const ENTROPY_BIN_WIDTH: f64 = 0.05;
|
||||
|
||||
/// Which entropy a summary line is about.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum Which {
|
||||
Entropy15,
|
||||
Entropy4,
|
||||
}
|
||||
|
||||
/// Number of families per (entropy, genomes present, genomes without
|
||||
/// ambiguity, entropy bin) — what `--shannon` rows reduce to once the
|
||||
/// question is a distribution (entropy against the number of carriers, how
|
||||
/// many carriers `--no-ambiguity` loses) and not a particular family.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct EntropySummary {
|
||||
counts: BTreeMap<(Which, usize, usize, usize), u64>,
|
||||
}
|
||||
|
||||
impl EntropySummary {
|
||||
fn bin(entropy: f64) -> usize {
|
||||
(entropy / ENTROPY_BIN_WIDTH).floor() as usize
|
||||
}
|
||||
|
||||
pub(crate) fn add(&mut self, row: &FamilyEntropyRow) {
|
||||
*self
|
||||
.counts
|
||||
.entry((Which::Entropy15, row.n_present, row.n_unambiguous, Self::bin(row.entropy15)))
|
||||
.or_default() += 1;
|
||||
// `entropy4` is not defined without a genome carrying exactly one member
|
||||
if row.n_unambiguous > 0 {
|
||||
*self
|
||||
.counts
|
||||
.entry((Which::Entropy4, row.n_present, row.n_unambiguous, Self::bin(row.entropy4)))
|
||||
.or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn write(&self, w: &mut impl Write) -> io::Result<()> {
|
||||
writeln!(w, "entropy,n_genomes_present,n_genomes_unambiguous,bin_start,n_families")?;
|
||||
for (&(which, present, unambiguous, bin), &count) in &self.counts {
|
||||
let name = match which {
|
||||
Which::Entropy15 => "entropy15",
|
||||
Which::Entropy4 => "entropy4",
|
||||
};
|
||||
writeln!(
|
||||
w,
|
||||
"{name},{present},{unambiguous},{:.2},{count}",
|
||||
bin as f64 * ENTROPY_BIN_WIDTH
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn row(entropy15: f64, entropy4: f64, n_present: usize, n_unambiguous: usize) -> FamilyEntropyRow {
|
||||
FamilyEntropyRow {
|
||||
partition: 0,
|
||||
layer: 0,
|
||||
family_idx: 0,
|
||||
family_size: 2,
|
||||
entropy15,
|
||||
entropy4,
|
||||
n_present,
|
||||
n_unambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
fn written(summary: &EntropySummary) -> Vec<String> {
|
||||
let mut buf = Vec::new();
|
||||
summary.write(&mut buf).unwrap();
|
||||
String::from_utf8(buf).unwrap().lines().map(str::to_owned).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn families_are_counted_per_carrier_numbers_and_entropy_bin() {
|
||||
let mut s = EntropySummary::default();
|
||||
s.add(&row(1.0, 1.0, 4, 4));
|
||||
s.add(&row(1.02, 1.02, 4, 4)); // same 0.05 bin as 1.0
|
||||
s.add(&row(0.2, 0.2, 4, 4));
|
||||
let lines = written(&s);
|
||||
assert_eq!(lines[0], "entropy,n_genomes_present,n_genomes_unambiguous,bin_start,n_families");
|
||||
assert!(lines.contains(&"entropy15,4,4,1.00,2".to_string()));
|
||||
assert!(lines.contains(&"entropy15,4,4,0.20,1".to_string()));
|
||||
assert!(lines.contains(&"entropy4,4,4,1.00,2".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy4_is_left_out_when_no_genome_has_a_single_member() {
|
||||
let mut s = EntropySummary::default();
|
||||
s.add(&row(0.0, 0.0, 1, 0));
|
||||
let lines = written(&s);
|
||||
assert!(lines.contains(&"entropy15,1,0,0.00,1".to_string()));
|
||||
assert!(!lines.iter().any(|l| l.starts_with("entropy4")));
|
||||
}
|
||||
}
|
||||
@@ -72,7 +72,7 @@ pub(crate) fn survives_masking(masked: &[Option<u8>], excluded: &[bool]) -> bool
|
||||
/// genomes. That is the pattern that can separate two pairs of genomes from
|
||||
/// each other (`AB|CD`); a singleton variant, or a family carried by fewer
|
||||
/// than four genomes, fits every topology and informs none.
|
||||
fn has_bipartition(counts: &[u32; 16]) -> bool {
|
||||
pub(crate) fn has_bipartition(counts: &[u32; 16]) -> bool {
|
||||
counts[1..].iter().filter(|&&c| c >= 2).count() >= 2
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ mod alignment;
|
||||
mod annex;
|
||||
mod cardcomp;
|
||||
mod entropy;
|
||||
mod entropy_scan;
|
||||
mod family_annexes;
|
||||
mod family_scan;
|
||||
mod masking;
|
||||
@@ -24,6 +25,7 @@ mod minorant_selection;
|
||||
mod pair_weight;
|
||||
mod pairwise;
|
||||
mod quartet;
|
||||
mod sample_stats;
|
||||
mod sankoff;
|
||||
mod scan;
|
||||
mod snp_distance;
|
||||
@@ -41,17 +43,16 @@ pub use sankoff::SankoffBundle;
|
||||
pub use scan::{LayerReducer, sampled_scan};
|
||||
pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
||||
pub use stats::SiblingAnnexStats;
|
||||
pub use subsample::{EntropyBias, SamplingSpec, SurvivingFamily};
|
||||
pub use sample_stats::{SampleStats, SampleStatsReducer};
|
||||
pub use subsample::{EntropyBias, LayerReport, SamplePlan, SamplingSpec, SurvivingFamily};
|
||||
|
||||
pub(crate) use annex::build_layer_sibling_annex;
|
||||
pub(crate) use entropy::{family_entropy, family_entropy_4};
|
||||
pub(crate) use entropy_scan::{EntropySummary, scan_family_entropies};
|
||||
pub(crate) use family_annexes::{
|
||||
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
|
||||
};
|
||||
pub(crate) use pair_weight::inverse_overlap_matrix;
|
||||
pub(crate) use quartet::restrict_to_quartet;
|
||||
pub(crate) use family_scan::{Selection, scan_layer_families};
|
||||
pub(crate) use minorant_selection::non_monomorphic_reindex_layer;
|
||||
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
|
||||
|
||||
/// Whether every layer number in `cache` fits in a `FamilyMask` field
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
//! `--sample-stats`: what a draw actually contains, accumulated as a
|
||||
//! [`LayerReducer`] over [`super::scan::sampled_scan`]'s batches, side by side
|
||||
//! with the alignment, from the very same families — per genome (how many
|
||||
//! sampled sites it has a call in, how many of them can resolve a
|
||||
//! bipartition), per number of carriers (how many sampled sites, how many
|
||||
//! informative), and per pair of genomes (how many sampled sites both have a
|
||||
//! call in: the sample's own family-overlap matrix, to set against the index's).
|
||||
//!
|
||||
//! A *call* is a masked state other than the empty one (`0`): absence is never
|
||||
//! a call, with or without `--free-loss`. A site is *informative* when its
|
||||
//! carriers show a bipartition (`masking::has_bipartition`).
|
||||
|
||||
use super::masking::has_bipartition;
|
||||
use super::scan::LayerReducer;
|
||||
use super::subsample::SurvivingFamily;
|
||||
use crate::siblings::helpers::triangle_index;
|
||||
|
||||
/// The statistics of one draw. `pub`: part of the public signature of the
|
||||
/// `--sample-stats` output.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SampleStats {
|
||||
pub n_genomes: usize,
|
||||
/// Number of sampled sites.
|
||||
pub n_sites: u64,
|
||||
/// Per genome: sampled sites it has a call in.
|
||||
pub calls: Vec<u64>,
|
||||
/// Per genome: informative sampled sites in which its state is one of the
|
||||
/// two or more carried by at least two genomes — those where it takes
|
||||
/// part in the bipartition.
|
||||
pub informative_calls: Vec<u64>,
|
||||
/// Index = number of carriers (`0..=n_genomes`): sampled sites.
|
||||
pub sites_by_carriers: Vec<u64>,
|
||||
/// Same, informative sites only.
|
||||
pub informative_by_carriers: Vec<u64>,
|
||||
/// Sampled sites both genomes of a pair have a call in, upper triangle in
|
||||
/// the family-overlap annex's own order — read it with
|
||||
/// [`overlap`](Self::overlap).
|
||||
overlap: Vec<u64>,
|
||||
}
|
||||
|
||||
impl SampleStats {
|
||||
/// Sampled sites genomes `i` and `j` both have a call in (`0` for `i == j`,
|
||||
/// as in `FamilyOverlap::get`).
|
||||
pub fn overlap(&self, i: usize, j: usize) -> u64 {
|
||||
if i == j {
|
||||
0
|
||||
} else {
|
||||
self.overlap[triangle_index(self.n_genomes, i, j)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Accumulates a [`SampleStats`] over a [`super::scan::sampled_scan`] pass.
|
||||
/// `excluded[g]` genomes are not counted anywhere, as in the alignment.
|
||||
pub struct SampleStatsReducer {
|
||||
excluded: Vec<bool>,
|
||||
stats: SampleStats,
|
||||
}
|
||||
|
||||
impl SampleStatsReducer {
|
||||
pub fn new(n_genomes: usize, excluded: &[bool]) -> Self {
|
||||
let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2;
|
||||
Self {
|
||||
excluded: (0..n_genomes).map(|g| excluded.get(g).copied().unwrap_or(false)).collect(),
|
||||
stats: SampleStats {
|
||||
n_genomes,
|
||||
n_sites: 0,
|
||||
calls: vec![0; n_genomes],
|
||||
informative_calls: vec![0; n_genomes],
|
||||
sites_by_carriers: vec![0; n_genomes + 1],
|
||||
informative_by_carriers: vec![0; n_genomes + 1],
|
||||
overlap: vec![0; n_pairs],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(self) -> SampleStats {
|
||||
self.stats
|
||||
}
|
||||
|
||||
/// One sampled site, from its per-genome masked states.
|
||||
fn add_site(&mut self, masked: &[Option<u8>]) {
|
||||
let n = self.stats.n_genomes;
|
||||
let mut counts = [0u32; 16];
|
||||
let mut carriers: Vec<usize> = Vec::new();
|
||||
for (g, &state) in masked.iter().enumerate() {
|
||||
if let Some(state) = state.filter(|&s| s != 0 && !self.excluded[g]) {
|
||||
counts[(state & 0b1111) as usize] += 1;
|
||||
carriers.push(g);
|
||||
}
|
||||
}
|
||||
let informative = has_bipartition(&counts);
|
||||
self.stats.n_sites += 1;
|
||||
self.stats.sites_by_carriers[carriers.len()] += 1;
|
||||
if informative {
|
||||
self.stats.informative_by_carriers[carriers.len()] += 1;
|
||||
}
|
||||
for (a, &i) in carriers.iter().enumerate() {
|
||||
self.stats.calls[i] += 1;
|
||||
if informative && counts[(masked[i].unwrap() & 0b1111) as usize] >= 2 {
|
||||
self.stats.informative_calls[i] += 1;
|
||||
}
|
||||
for &j in &carriers[a + 1..] {
|
||||
self.stats.overlap[triangle_index(n, i, j)] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl LayerReducer for SampleStatsReducer {
|
||||
fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
|
||||
for family in survivors {
|
||||
self.add_site(&family.masked);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn counts_calls_carriers_and_pairs() {
|
||||
let mut r = SampleStatsReducer::new(5, &[]);
|
||||
// informative: A A G G, genome 4 absent (state 0)
|
||||
r.add_site(&[Some(1), Some(1), Some(4), Some(4), Some(0)]);
|
||||
// not informative: three A and one lone G, genome 4 missing
|
||||
r.add_site(&[Some(1), Some(1), Some(1), Some(4), None]);
|
||||
let s = r.finish();
|
||||
assert_eq!(s.n_sites, 2);
|
||||
assert_eq!(s.calls, vec![2, 2, 2, 2, 0]);
|
||||
assert_eq!(s.sites_by_carriers[4], 2);
|
||||
assert_eq!(s.informative_by_carriers[4], 1);
|
||||
// genome 3 is in the G pair of the informative site only
|
||||
assert_eq!(s.informative_calls, vec![1, 1, 1, 1, 0]);
|
||||
assert_eq!(s.overlap(0, 1), 2);
|
||||
assert_eq!(s.overlap(2, 3), 2);
|
||||
assert_eq!(s.overlap(0, 4), 0);
|
||||
assert_eq!(s.overlap(1, 1), 0);
|
||||
assert_eq!(s.overlap(3, 0), s.overlap(0, 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excluded_genomes_count_nowhere() {
|
||||
let mut r = SampleStatsReducer::new(4, &[false, false, false, true]);
|
||||
r.add_site(&[Some(1), Some(1), Some(4), Some(4)]);
|
||||
let s = r.finish();
|
||||
// without genome 3 the G state has a single carrier: not informative
|
||||
assert_eq!(s.calls, vec![1, 1, 1, 0]);
|
||||
assert_eq!(s.sites_by_carriers[3], 1);
|
||||
assert_eq!(s.informative_by_carriers[3], 0);
|
||||
assert_eq!(s.overlap(2, 3), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn genome_with_a_lone_state_takes_no_part_in_the_bipartition() {
|
||||
let mut r = SampleStatsReducer::new(5, &[]);
|
||||
// A A C C G: genome 4 is the lone G
|
||||
r.add_site(&[Some(1), Some(1), Some(2), Some(2), Some(4)]);
|
||||
let s = r.finish();
|
||||
assert_eq!(s.informative_calls, vec![1, 1, 1, 1, 0]);
|
||||
assert_eq!(s.calls[4], 1);
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
|
||||
use super::stats::sibling_family_size_histogram;
|
||||
use super::subsample::{SamplingSpec, SurvivingFamily, sample_index};
|
||||
use super::subsample::{SamplePlan, SamplingSpec, SurvivingFamily, sample_index};
|
||||
|
||||
/// One accumulator fed every layer's [`SurvivingFamily`] batch during a
|
||||
/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of
|
||||
@@ -47,22 +47,22 @@ fn resolve_target(cache: &IndexCache, n: Option<usize>) -> OKIResult<usize> {
|
||||
/// own docs and [`SamplingSpec`] for what each controls), feeding every
|
||||
/// surviving family of every visited layer to every reducer in
|
||||
/// `reducers`, in the same order, before moving to the next layer. A
|
||||
/// no-op (not an error) if the resolved target is `0`.
|
||||
/// no-op (not an error) if the resolved target is `0`. Returns how the draw
|
||||
/// was spread over the layers ([`SamplePlan`]).
|
||||
pub fn sampled_scan(
|
||||
cache: &IndexCache,
|
||||
n: Option<usize>,
|
||||
spec: SamplingSpec,
|
||||
excluded: &[bool],
|
||||
reducers: &mut [&mut dyn LayerReducer],
|
||||
) -> OKIResult<()> {
|
||||
) -> OKIResult<SamplePlan> {
|
||||
let target = resolve_target(cache, n)?;
|
||||
if target == 0 {
|
||||
return Ok(());
|
||||
return Ok(SamplePlan::default());
|
||||
}
|
||||
sample_index(cache, target, spec, excluded, 0, |partition, layer, _raw_index, survivors| {
|
||||
for reducer in reducers.iter_mut() {
|
||||
reducer.reduce(partition, layer, &survivors);
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
@@ -115,6 +115,41 @@ pub struct SamplingSpec {
|
||||
pub entropy_bias: Option<EntropyBias>,
|
||||
}
|
||||
|
||||
/// What one layer contributed to a draw — one row of `--sample-plan`.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LayerReport {
|
||||
pub partition: usize,
|
||||
pub layer: usize,
|
||||
/// Families eligible when the draw starts (non-monomorphic, and
|
||||
/// quartet-informative with `--quartet`).
|
||||
pub eligible: u64,
|
||||
/// Their total sampling weight: their number, or with `--pair-balance` the
|
||||
/// sum of their pair weights.
|
||||
pub mass: f64,
|
||||
/// The layer's share of the target.
|
||||
pub quota: usize,
|
||||
/// Families drawn and resolved.
|
||||
pub resolved: usize,
|
||||
/// Of those, the ones that survived masking (and `--quartet`'s exact check).
|
||||
pub kept: usize,
|
||||
}
|
||||
|
||||
/// How a draw was spread over the layers of the index.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct SamplePlan {
|
||||
/// The number of sites asked for.
|
||||
pub target: usize,
|
||||
/// One report per layer, in the order the draw visited them.
|
||||
pub layers: Vec<LayerReport>,
|
||||
}
|
||||
|
||||
impl SamplePlan {
|
||||
/// Sites actually kept, all layers together.
|
||||
pub fn kept(&self) -> usize {
|
||||
self.layers.iter().map(|layer| layer.kept).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// [`iter_full_scalar`] of `kind` (entropy, pair weight), rotated to start at
|
||||
/// `start` and wrap once back to it — the same `family_idx` order
|
||||
/// [`sample_layer`]'s own circular index sequence walks, so the streams can be
|
||||
@@ -143,6 +178,8 @@ struct LayerPlan {
|
||||
/// One bit per minorant (`family_idx` order): still eligible to be drawn.
|
||||
eligible: TempBitVecBuilder,
|
||||
n_minorants: usize,
|
||||
/// How many of those bits are set when the draw starts.
|
||||
eligible_count: u64,
|
||||
/// Total sampling weight of the eligible families: their number, or, with
|
||||
/// `--pair-balance`, the sum of their pair weights. A layer's quota is its
|
||||
/// share of this mass over every layer, so a layer rich in families that
|
||||
@@ -191,10 +228,10 @@ struct LayerPlan {
|
||||
/// the seam a future resume mechanism would hook into, rather than being
|
||||
/// ripped out and re-added later.
|
||||
///
|
||||
/// Returns the actual number of *newly processed* sites kept (not
|
||||
/// counting whatever `skip_layers` layers already contributed on an
|
||||
/// earlier, interrupted call) — may be less than `n` (see the module
|
||||
/// docs).
|
||||
/// Returns how the draw was spread over the layers ([`SamplePlan`]): one
|
||||
/// report per layer, with its quota and what it actually kept (nothing for the
|
||||
/// `skip_layers` layers an earlier, interrupted call already covered). The
|
||||
/// sum of what was kept may be less than `n` (see the module docs).
|
||||
pub(crate) fn sample_index(
|
||||
cache: &IndexCache,
|
||||
n: usize,
|
||||
@@ -202,7 +239,7 @@ pub(crate) fn sample_index(
|
||||
excluded: &[bool],
|
||||
skip_layers: usize,
|
||||
mut on_layer: impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
|
||||
) -> OKIResult<usize> {
|
||||
) -> OKIResult<SamplePlan> {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(cache);
|
||||
|
||||
@@ -237,6 +274,7 @@ pub(crate) fn sample_index(
|
||||
restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?;
|
||||
}
|
||||
let n_minorants = eligible.view().len();
|
||||
let eligible_count = eligible.view().count_ones();
|
||||
let mass = if spec.pair_balance {
|
||||
iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)?
|
||||
.enumerate()
|
||||
@@ -244,15 +282,23 @@ pub(crate) fn sample_index(
|
||||
.map(|(_, weight)| f64::from(weight))
|
||||
.sum()
|
||||
} else {
|
||||
eligible.view().count_ones() as f64
|
||||
eligible_count as f64
|
||||
};
|
||||
plans.push(LayerPlan { partition, layer: layer_idx, eligible, n_minorants, mass });
|
||||
plans.push(LayerPlan {
|
||||
partition,
|
||||
layer: layer_idx,
|
||||
eligible,
|
||||
n_minorants,
|
||||
eligible_count,
|
||||
mass,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum();
|
||||
let mut report = SamplePlan { target: n, layers: Vec::with_capacity(plans.len()) };
|
||||
if total_mass <= 0.0 {
|
||||
return Ok(0);
|
||||
return Ok(report);
|
||||
}
|
||||
|
||||
// Quotas by cumulative rounding: layer `i` is allotted
|
||||
@@ -260,7 +306,6 @@ pub(crate) fn sample_index(
|
||||
// were allotted`, so the quotas sum to exactly `n`. Rounding each layer's
|
||||
// share down on its own would lose up to one site per layer — all of
|
||||
// them, when `n` is smaller than the number of layers.
|
||||
let mut total_kept = 0usize;
|
||||
let mut cumulative_mass = 0.0f64;
|
||||
let mut allotted = 0usize;
|
||||
for (raw_index, mut plan) in plans.into_iter().enumerate() {
|
||||
@@ -268,26 +313,37 @@ pub(crate) fn sample_index(
|
||||
let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize;
|
||||
let quota = cumulative_quota.saturating_sub(allotted);
|
||||
allotted = allotted.max(cumulative_quota);
|
||||
if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 {
|
||||
continue;
|
||||
}
|
||||
total_kept += sample_layer(
|
||||
cache,
|
||||
raw_index,
|
||||
&mut plan,
|
||||
let (resolved, kept) = if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 {
|
||||
(0, 0)
|
||||
} else {
|
||||
sample_layer(
|
||||
cache,
|
||||
raw_index,
|
||||
&mut plan,
|
||||
quota,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
spec,
|
||||
excluded,
|
||||
&mut on_layer,
|
||||
)?
|
||||
};
|
||||
report.layers.push(LayerReport {
|
||||
partition: plan.partition,
|
||||
layer: plan.layer,
|
||||
eligible: plan.eligible_count,
|
||||
mass: plan.mass,
|
||||
quota,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
spec,
|
||||
excluded,
|
||||
&mut on_layer,
|
||||
)?;
|
||||
resolved,
|
||||
kept,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(total_kept)
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// One layer's share of [`sample_index`] — see the module docs for the
|
||||
/// One layer's share of [`sample_index`], as `(families drawn and resolved,
|
||||
/// families kept after masking)` — see the module docs for the
|
||||
/// per-family accept/reject/mark rules and the round-batching scheme. Each
|
||||
/// eligible family is accepted with probability
|
||||
/// `quota / mass × pair weight × entropy kernel` (capped at 1): the pair
|
||||
@@ -303,11 +359,11 @@ fn sample_layer(
|
||||
spec: SamplingSpec,
|
||||
excluded: &[bool],
|
||||
on_layer: &mut impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
|
||||
) -> OKIResult<usize> {
|
||||
) -> OKIResult<(usize, usize)> {
|
||||
let LayerPlan { partition, layer: layer_idx, n_minorants, mass, .. } = *plan;
|
||||
let eligible = &mut plan.eligible;
|
||||
if n_minorants == 0 || mass <= 0.0 {
|
||||
return Ok(0);
|
||||
return Ok((0, 0));
|
||||
}
|
||||
let p0 = quota as f64 / mass;
|
||||
let layer_dir = cache
|
||||
@@ -435,5 +491,5 @@ fn sample_layer(
|
||||
on_layer(partition, layer_idx, raw_index, Arc::new(survivors));
|
||||
}
|
||||
|
||||
Ok(kept)
|
||||
Ok((resolved, kept))
|
||||
}
|
||||
@@ -14,9 +14,8 @@ use obikindex::{OKIError, OKIResult};
|
||||
use obisys::progress_bar;
|
||||
|
||||
use crate::siblings::algorithms::{
|
||||
AnnexRequest, Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy,
|
||||
family_entropy_4, is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families,
|
||||
sibling_annex_stats, sibling_family_size_histogram,
|
||||
AnnexRequest, EntropySummary, SiblingAnnexStats, build_layer_sibling_annex, is_fast_mode,
|
||||
scan_family_entropies, sibling_annex_stats, sibling_family_size_histogram,
|
||||
};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
use crate::siblings::{
|
||||
@@ -66,6 +65,18 @@ pub trait SiblingExt {
|
||||
/// `--entropy-bias` yet (see the project memory on this).
|
||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>;
|
||||
|
||||
/// The same scan as [`shannon_entropy_csv`](Self::shannon_entropy_csv), but
|
||||
/// written as a distribution instead of one row per family: a CSV
|
||||
/// (`entropy,n_genomes_present,n_genomes_unambiguous,bin_start,n_families`)
|
||||
/// counting the families per `entropy15` (all carriers) or `entropy4`
|
||||
/// (single-member genomes; families with none are left out of its lines),
|
||||
/// number of genomes present, number of those without ambiguity, and
|
||||
/// 0.05-bit entropy bin. At most a few hundred thousand lines whatever the
|
||||
/// size of the index — enough to answer how entropy varies with the number
|
||||
/// of carriers, or how many carriers `--no-ambiguity` loses, without the
|
||||
/// tens of gigabytes the per-family file takes.
|
||||
fn shannon_entropy_summary(&self, path: &Path) -> OKIResult<()>;
|
||||
|
||||
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
|
||||
/// no `per_genome`) — reads only the already-built annex
|
||||
/// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits,
|
||||
@@ -175,72 +186,35 @@ impl SiblingExt for IndexCache {
|
||||
}
|
||||
|
||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {
|
||||
// Needs only the sibling annex (`non_monomorphic_reindex_layer`
|
||||
// below): this method runs its own, differently-scoped scan and
|
||||
// recomputes every value it writes, so it neither reads nor
|
||||
// requires the entropy/family-overlap annexes.
|
||||
let n_genomes = self.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(self);
|
||||
|
||||
let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?);
|
||||
writeln!(f, "partition,layer,family_idx,entropy15,entropy4,family_size,n_genomes_present,n_genomes_unambiguous")
|
||||
.map_err(OKIError::Io)?;
|
||||
scan_family_entropies(self, |row| {
|
||||
writeln!(
|
||||
f,
|
||||
"{},{},{},{:.6},{:.6},{},{},{}",
|
||||
row.partition,
|
||||
row.layer,
|
||||
row.family_idx,
|
||||
row.entropy15,
|
||||
row.entropy4,
|
||||
row.family_size,
|
||||
row.n_present,
|
||||
row.n_unambiguous
|
||||
)
|
||||
})?;
|
||||
f.flush().map_err(OKIError::Io)
|
||||
}
|
||||
|
||||
let pb = progress_bar("shannon_entropy", self.n_partition() as u64, "partitions");
|
||||
for part in self.partitions() {
|
||||
let n_layer = self.n_layer(part).unwrap_or(0);
|
||||
for l in 0..n_layer {
|
||||
let layer = self
|
||||
.get_layer(part, l)
|
||||
.expect("cache-consistent: n_layer(part) already bounds l");
|
||||
// Bounded to non-monomorphic minorants only, same reasoning
|
||||
// as `ensure_layer_entropy_annex`'s own docs: monomorphism
|
||||
// is already known from the annex bits alone (~98% of a
|
||||
// layer's minorants, measured on real data), so there is no
|
||||
// reason to pay `scan_layer_families`'s expensive
|
||||
// cross-partition resolution for families this method
|
||||
// skips right below anyway.
|
||||
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
|
||||
let eligible: std::collections::HashSet<usize> = reindex.keys().copied().collect();
|
||||
|
||||
let mut write_err = None;
|
||||
scan_layer_families(
|
||||
self,
|
||||
part,
|
||||
l,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, mask, genome_mask| {
|
||||
if write_err.is_some() {
|
||||
return;
|
||||
}
|
||||
debug_assert!(
|
||||
mask.family_size() >= 2,
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
let Some((h15, n_present)) = family_entropy(genome_mask) else {
|
||||
return;
|
||||
};
|
||||
let (h4, n_unambiguous) = family_entropy_4(genome_mask).unwrap_or((0.0, 0));
|
||||
if let Err(e) = writeln!(
|
||||
f,
|
||||
"{part},{l},{family_idx},{h15:.6},{h4:.6},{},{n_present},{n_unambiguous}",
|
||||
mask.family_size()
|
||||
) {
|
||||
write_err = Some(e);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
if let Some(e) = write_err {
|
||||
return Err(OKIError::Io(e));
|
||||
}
|
||||
}
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(())
|
||||
fn shannon_entropy_summary(&self, path: &Path) -> OKIResult<()> {
|
||||
let mut summary = EntropySummary::default();
|
||||
scan_family_entropies(self, |row| {
|
||||
summary.add(row);
|
||||
Ok(())
|
||||
})?;
|
||||
let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?);
|
||||
summary.write(&mut f).map_err(OKIError::Io)?;
|
||||
f.flush().map_err(OKIError::Io)
|
||||
}
|
||||
|
||||
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> {
|
||||
|
||||
@@ -42,8 +42,8 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
|
||||
pub use algorithms::{
|
||||
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
|
||||
PairwiseReducer, RawSnpDistanceOutput, SamplingSpec, SankoffBundle, SiblingAnnexStats,
|
||||
SnpAlignment,
|
||||
LayerReport, PairwiseReducer, RawSnpDistanceOutput, SamplePlan, SampleStats,
|
||||
SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SnpAlignment,
|
||||
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
|
||||
pairwise_cost_matrix, sampled_scan,
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user