Implement advanced sampling and family annex logic
Introduces complex sampling mechanisms including quartet filtering, pair balance weighting, and entropy bias. Enforces the `--no-ambiguity` convention for family counting and updates annex storage for scalar data. This includes new logic for calculating family weights based on overlap, quota-based layer determination, and filtering families based on quartet informativeness. Updates also include argument parsing for `--n-sites` and improved handling of family overlap files.
This commit is contained in:
1 parent
0d12494f1a
commit
f29dcbcfb8
25 files changed
+1471
-486
No files matched your search
Generated
+1
-1
@@ -1532,7 +1532,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "obikmer"
|
||||
version = "1.3.5"
|
||||
version = "1.3.6"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"csv",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "obikmer"
|
||||
version = "1.3.5"
|
||||
version = "1.3.6"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -32,6 +32,36 @@ fn parse_gamma_shape(s: &str) -> Result<GammaShapeArg, String> {
|
||||
.map_err(|_| format!("invalid --gamma-shape value {s:?}: expected a positive number or \"auto\"/\"estimate\""))
|
||||
}
|
||||
|
||||
/// `--n-sites` CLI value: a number of sites to draw, or `all` for every site
|
||||
/// — the draw then keeps every variable family that passes the other filters,
|
||||
/// with no sampling involved. Its own tiny type (like [`GammaShapeArg`]) so
|
||||
/// the two cases stay distinct up to the call that needs the target, instead
|
||||
/// of an in-band sentinel (a negative number, `0`, ...) that a typo could hit.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NSitesArg {
|
||||
All,
|
||||
N(usize),
|
||||
}
|
||||
|
||||
impl NSitesArg {
|
||||
/// The target to hand to the sampler: `None` = exhaustive.
|
||||
pub fn target(self) -> Option<usize> {
|
||||
match self {
|
||||
NSitesArg::All => None,
|
||||
NSitesArg::N(n) => Some(n),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_n_sites(s: &str) -> Result<NSitesArg, String> {
|
||||
if s.eq_ignore_ascii_case("all") {
|
||||
return Ok(NSitesArg::All);
|
||||
}
|
||||
s.parse::<usize>()
|
||||
.map(NSitesArg::N)
|
||||
.map_err(|_| format!("invalid --n-sites value {s:?}: expected a non-negative integer or \"all\""))
|
||||
}
|
||||
|
||||
/// `--distance` value — either one of `obikphylo::DistanceMetric`'s
|
||||
/// whole-index metrics (routed to `IndexCache::distance`) or one of
|
||||
/// `obikphylo::SnpDistanceKind`'s `snp-*` corrections (routed to
|
||||
@@ -108,7 +138,7 @@ impl DistanceArg {
|
||||
/// (classic metrics + `snp-*` corrections/NJ/UPGMA), annex construction
|
||||
/// (`--sibling-annex`), annex diagnostics (`--sibling-stats`,
|
||||
/// `--sibling-hist`), entropy reporting (`--shannon`), SNP pseudo-alignment
|
||||
/// sampling (`--pseudo-alignment`, `--subsample`, `--free-loss`,
|
||||
/// sampling (`--pseudo-alignment`, `--n-sites`, `--free-loss`,
|
||||
/// `--no-ambiguity`, `--entropy`/`--entropy-sd`), Sankoff cost-matrix
|
||||
/// calibration (`--sankoff`, `--sankoff-ratio-ceiling`) and its TNT/PhyG/
|
||||
/// IQ-TREE exports (`--tnt`, `--phyg`, `--iqtree`/`--iqtree-min-freq`,
|
||||
@@ -180,26 +210,25 @@ pub struct PhyloArgs {
|
||||
/// Write a per-family Shannon entropy report (CSV) — requires an
|
||||
/// already-built sibling annex (`--sibling-annex` first, in this
|
||||
/// invocation or an earlier one). Always a full, unsampled scan of
|
||||
/// every family (`--subsample`/`--entropy`/`--entropy-sd` below only
|
||||
/// every family (`--n-sites`/`--entropy`/`--entropy-sd` below only
|
||||
/// apply to `--pseudo-alignment`, not this).
|
||||
#[arg(long)]
|
||||
pub shannon: bool,
|
||||
|
||||
/// Write a SNP-only pseudo-alignment (FASTA) — requires
|
||||
/// `--subsample <N>` and an already-built sibling annex
|
||||
/// `--n-sites <N|all>` and an already-built sibling annex
|
||||
/// (`--sibling-annex` first, in this invocation or an earlier one).
|
||||
#[arg(long)]
|
||||
pub pseudo_alignment: bool,
|
||||
|
||||
/// Target number of variable sites to sample index-wide for
|
||||
/// `--pseudo-alignment`/`--sankoff` (mandatory for both) and for a
|
||||
/// `snp-*` `--distance` value (optional there: omitted means exhaustive
|
||||
/// — every non-monomorphic minorant of the whole index, not an
|
||||
/// approximation) — a target, not a guarantee when given (proportional
|
||||
/// per-layer sampling; see `obikphylo::siblings::sampled_scan`'s own
|
||||
/// docs).
|
||||
#[arg(long)]
|
||||
pub subsample: Option<usize>,
|
||||
/// Number of variable sites to draw index-wide, or `all` for every site
|
||||
/// that passes the other filters (no sampling then). Mandatory for
|
||||
/// `--pseudo-alignment`/`--sankoff`; optional for a `snp-*`
|
||||
/// `--distance` value, where omitting it also means every site. A
|
||||
/// number is a target, not a guarantee (proportional per-layer
|
||||
/// sampling; see `obikphylo::siblings::sampled_scan`'s own docs).
|
||||
#[arg(long = "n-sites", value_name = "N|all", value_parser = parse_n_sites)]
|
||||
pub n_sites: Option<NSitesArg>,
|
||||
|
||||
/// In `--pseudo-alignment`, treat a genome carrying none of a family's
|
||||
/// observed members (`∅`) as missing data (`?`) rather than a real
|
||||
@@ -213,6 +242,24 @@ pub struct PhyloArgs {
|
||||
#[arg(long)]
|
||||
pub no_ambiguity: bool,
|
||||
|
||||
/// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, keep only
|
||||
/// the families that can separate two pairs of genomes from each other
|
||||
/// (a bipartition): at least two distinct called states, each carried
|
||||
/// by at least two genomes. A singleton variant, or a family carried by
|
||||
/// fewer than four genomes, fits every topology and tells a tree
|
||||
/// nothing. Which genomes count as carriers follows `--no-ambiguity`.
|
||||
#[arg(long)]
|
||||
pub quartet: bool,
|
||||
|
||||
/// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, favour the
|
||||
/// families that cover the pairs of genomes sharing the fewest families
|
||||
/// across the index: each family is weighted by the mean, over the pairs
|
||||
/// of genomes carrying it, of 1 / (number of families the pair shares).
|
||||
/// Evens out how well each pair of genomes is represented in the sample.
|
||||
/// Which genomes count as carriers follows `--no-ambiguity`.
|
||||
#[arg(long)]
|
||||
pub pair_balance: bool,
|
||||
|
||||
/// Entropy-biased sampling target (Gaussian kernel mean) for
|
||||
/// `--pseudo-alignment` — activates biasing as soon as this or
|
||||
/// `--entropy-sd` is given; the other defaults to 1.0/0.5.
|
||||
@@ -227,7 +274,7 @@ pub struct PhyloArgs {
|
||||
/// Persist the `snp-*` `--distance` sample (the pairwise substitution
|
||||
/// tally) in `DIR` across separate `obikmer phylo` invocations: a later
|
||||
/// run against the same index with the exact same
|
||||
/// `--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/
|
||||
/// `--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/
|
||||
/// `--min-shared-family`/`--entropy`/`--entropy-sd` restores it instead
|
||||
/// of resampling. `DIR` is created if it doesn't exist. If it already
|
||||
/// holds a sample built under *different* selection parameters, this
|
||||
@@ -247,7 +294,7 @@ pub struct PhyloArgs {
|
||||
|
||||
/// Calibrate a 16-state Sankoff cost matrix (and its matching
|
||||
/// pseudo-alignment) from an already-built sibling annex — requires
|
||||
/// `--subsample <N>`, and shares `--free-loss`/`--no-ambiguity`/
|
||||
/// `--n-sites <N|all>`, and shares `--free-loss`/`--no-ambiguity`/`--quartet`/
|
||||
/// `--entropy`/`--entropy-sd` with `--pseudo-alignment` (one draw, same
|
||||
/// selection feeds both the alignment and every calibration tally).
|
||||
#[arg(long)]
|
||||
|
||||
@@ -11,13 +11,14 @@ use std::sync::Arc;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::KmerIndex;
|
||||
use obikphylo::siblings::{
|
||||
EntropyBias, GammaShape, SiblingExt, cardinality_transition_probs,
|
||||
EntropyBias, GammaShape, SamplingSpec, SiblingExt, cardinality_transition_probs,
|
||||
composition_transition_probs, pairwise_cost_matrix,
|
||||
};
|
||||
use obikphylo::{Metrics, neighbor_joining, upgma};
|
||||
use obisys::{Reporter, Stage};
|
||||
use tracing::info;
|
||||
|
||||
use args::NSitesArg;
|
||||
use iqtree::write_iqtree;
|
||||
use phyg::write_sankoff_phyg;
|
||||
use phylip::write_phylip_relaxed;
|
||||
@@ -154,7 +155,7 @@ pub fn run(args: PhyloArgs) {
|
||||
// ── Family Overlap matrix (`--family-overlap`) ──────────────────────────────
|
||||
if args.family_overlap {
|
||||
let t = Stage::start("family_overlap");
|
||||
let overlap = cache.family_overlap().unwrap_or_else(|e| {
|
||||
let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| {
|
||||
eprintln!("error computing family overlap: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
@@ -201,7 +202,7 @@ pub fn run(args: PhyloArgs) {
|
||||
// `args::PhyloArgs::min_shared_family`'s own docs on why).
|
||||
let snp_exclude_mask: Vec<bool> = match args.min_shared_family {
|
||||
Some(threshold) => {
|
||||
let overlap = cache.family_overlap().unwrap_or_else(|e| {
|
||||
let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| {
|
||||
eprintln!("error computing family overlap: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
@@ -235,6 +236,13 @@ pub fn run(args: PhyloArgs) {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let sampling = SamplingSpec {
|
||||
free_loss: args.free_loss,
|
||||
no_ambiguity: args.no_ambiguity,
|
||||
quartet: args.quartet,
|
||||
pair_balance: args.pair_balance,
|
||||
entropy_bias,
|
||||
};
|
||||
|
||||
// ── `--session` ──────────────────────────────────────────────────────────
|
||||
// Temporarily unavailable: the per-driver checkpoint/resume plumbing
|
||||
@@ -253,7 +261,7 @@ pub fn run(args: PhyloArgs) {
|
||||
|
||||
// ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ──
|
||||
// All three consume exactly the same selection parameters
|
||||
// (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/
|
||||
// (`--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/
|
||||
// `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from
|
||||
// the *same* random sample when combined in one invocation — so
|
||||
// whichever of them are requested here share exactly one
|
||||
@@ -265,13 +273,15 @@ pub fn run(args: PhyloArgs) {
|
||||
let snp_kind = args.distance.as_snp();
|
||||
let need_alignment = args.pseudo_alignment || sankoff_requested;
|
||||
let need_pairwise = sankoff_requested || snp_kind.is_some();
|
||||
// `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional)
|
||||
let target = args.n_sites.and_then(NSitesArg::target);
|
||||
|
||||
if args.pseudo_alignment && args.subsample.is_none() {
|
||||
eprintln!("error: --pseudo-alignment requires --subsample <N>");
|
||||
if args.pseudo_alignment && args.n_sites.is_none() {
|
||||
eprintln!("error: --pseudo-alignment requires --n-sites <N|all>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
if sankoff_requested && args.subsample.is_none() {
|
||||
eprintln!("error: --sankoff requires --subsample <N>");
|
||||
if sankoff_requested && args.n_sites.is_none() {
|
||||
eprintln!("error: --sankoff requires --n-sites <N|all>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
@@ -290,13 +300,13 @@ pub fn run(args: PhyloArgs) {
|
||||
|
||||
info!(
|
||||
"sampling SNP sites{}",
|
||||
match args.subsample {
|
||||
match target {
|
||||
Some(n) => format!(" (target {n} site(s))"),
|
||||
None => " (exhaustive)".into(),
|
||||
}
|
||||
);
|
||||
let t = Stage::start("sampled_scan");
|
||||
obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers)
|
||||
obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers)
|
||||
.unwrap_or_else(|e| {
|
||||
eprintln!("error sampling SNP sites: {e}");
|
||||
std::process::exit(1);
|
||||
|
||||
@@ -8,19 +8,6 @@
|
||||
//! it is actually observed) is the consistent informativeness proxy for
|
||||
//! this project, not a 4-symbol reduction.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::minorant_selection::{is_non_monomorphic_minorant, non_monomorphic_reindex_layer};
|
||||
use crate::siblings::{
|
||||
ANNEX_FILE_NAME, ENTROPY_ANNEX_FILE_NAME, EntropyAnnex, EntropyAnnexBuilder,
|
||||
FamilyOverlapAccumulator, SiblingAnnex,
|
||||
};
|
||||
|
||||
/// Shannon entropy (bits) of one family's per-genome states, over the 15
|
||||
/// non-empty subsets of `{A,C,G,T}` actually observed among the genomes
|
||||
/// carrying it (`genome_mask[g] != 0`) — genomes where the family is absent
|
||||
@@ -103,6 +90,26 @@ mod tests {
|
||||
assert_eq!(n, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_15_over_single_member_genomes_equals_entropy_4() {
|
||||
// `--no-ambiguity` leaves only the single-base states, over which the
|
||||
// 15-state entropy and the 4-base one are the same quantity
|
||||
let masks = [0b0001u8, 0b0001, 0b0100, 0b1000, 0b0100, 0b0100, 0b0010];
|
||||
let (h15, n15) = family_entropy(&masks).unwrap();
|
||||
let (h4, n4) = family_entropy_4(&masks).unwrap();
|
||||
assert!((h15 - h4).abs() < 1e-12);
|
||||
assert_eq!(n15, n4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_under_selects_the_counting_convention() {
|
||||
let masks = [0b0001u8, 0b0001, 0b0011]; // the last genome carries two members
|
||||
assert_eq!(family_entropy_under(&masks, true).unwrap().1, 2);
|
||||
assert_eq!(family_entropy_under(&masks, false).unwrap().1, 3);
|
||||
assert_eq!(family_entropy_under(&masks, true).unwrap().0, 0.0);
|
||||
assert!(family_entropy_under(&masks, false).unwrap().0 > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_4_is_none_without_unambiguous_genome() {
|
||||
assert!(family_entropy_4(&[0, 0b0011, 0b0110]).is_none());
|
||||
@@ -110,104 +117,21 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds one layer's entropy annex if it doesn't already exist, and/or
|
||||
/// feeds `overlap_acc` (`Some` iff the index-wide family-overlap annex is
|
||||
/// itself missing — see
|
||||
/// [`crate::siblings::FamilyOverlap`]'s own module docs) — a single shared
|
||||
/// `scan_layer_families` pass covers both, since both need the exact same
|
||||
/// per-family `genome_mask` resolution and there is no reason to pay for it
|
||||
/// twice just because only one of the two happens to be missing. A no-op
|
||||
/// (no scan at all) when entropy already exists for this layer *and*
|
||||
/// `overlap_acc` is `None`. Bounded to non-monomorphic minorants only
|
||||
/// ([`non_monomorphic_reindex_layer`], a cheap annex-only pass — no
|
||||
/// cross-partition resolution) whenever a scan does happen: monomorphism is
|
||||
/// already known from the annex bits alone, so there is no reason to pay
|
||||
/// `scan_layer_families`'s expensive per-genome resolution for the ~98% of
|
||||
/// minorants that are monomorphic only to discard the result — true for
|
||||
/// family-overlap's own "variable families only" scope just as much as for
|
||||
/// entropy's.
|
||||
pub(crate) fn ensure_layer_entropy_annex(
|
||||
cache: &IndexCache,
|
||||
partition: usize,
|
||||
layer_idx: usize,
|
||||
n_genomes: usize,
|
||||
fast_mode: bool,
|
||||
mut overlap_acc: Option<&mut FamilyOverlapAccumulator>,
|
||||
) -> OKIResult<()> {
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
|
||||
let path = layer.dir().join(ENTROPY_ANNEX_FILE_NAME);
|
||||
let need_entropy = !path.exists();
|
||||
if !need_entropy && overlap_acc.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
|
||||
let eligible: HashSet<usize> = reindex.keys().copied().collect();
|
||||
let mut builder = need_entropy
|
||||
.then(|| EntropyAnnexBuilder::new(reindex.len(), &path))
|
||||
.transpose()
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
scan_layer_families(
|
||||
cache,
|
||||
partition,
|
||||
layer_idx,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, mask, genome_mask| {
|
||||
debug_assert!(
|
||||
mask.family_size() >= 2,
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
if let Some(builder) = builder.as_mut() {
|
||||
if let Some((h15, _)) = family_entropy(genome_mask) {
|
||||
builder.set(reindex[&family_idx], h15 as f32);
|
||||
}
|
||||
}
|
||||
if let Some(acc) = overlap_acc.as_deref_mut() {
|
||||
acc.add_family(genome_mask);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
if let Some(builder) = builder {
|
||||
builder.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Streams one entropy value per minorant of the layer, **in `family_idx`
|
||||
/// order** — monomorphic minorants included, valued `0.0` by convention
|
||||
/// (trivially uninformative, never computed or stored in the compact
|
||||
/// entropy annex). Exists so a caller can `.zip()` this directly against
|
||||
/// any other `family_idx`-ordered stream (e.g.
|
||||
/// `SiblingLayerExt::iter_minorants`) instead of separately consulting
|
||||
/// [`non_monomorphic_reindex_layer`]'s map at each site.
|
||||
/// Shannon entropy (bits) of one family under the counting convention
|
||||
/// `--no-ambiguity` selects:
|
||||
/// - `false`: [`family_entropy`] — every genome carrying the family counts,
|
||||
/// a genome with several members being a state of its own;
|
||||
/// - `true`: [`family_entropy_4`] — only the genomes carrying exactly one
|
||||
/// member count, the others being missing data, exactly as the alignment
|
||||
/// itself will see them. Over those genomes the 15-state entropy can only
|
||||
/// ever take the four single-base states, so the two definitions coincide.
|
||||
///
|
||||
/// Requires this layer's entropy annex to already exist
|
||||
/// ([`ensure_layer_entropy_annex`] first) — reads it positionally
|
||||
/// (`compact` advances only on non-monomorphic minorants, mirroring
|
||||
/// exactly how the annex was written), never recomputes anything.
|
||||
pub(crate) fn iter_full_entropy(layer_dir: &Path) -> OKIResult<impl Iterator<Item = f32> + use<>> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
|
||||
let entropy_annex =
|
||||
EntropyAnnex::open(&layer_dir.join(ENTROPY_ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
|
||||
|
||||
let mut compact = 0usize;
|
||||
Ok((0..annex.len()).filter_map(move |slot| {
|
||||
let mask = annex.get(slot)?;
|
||||
if !mask.is_minorant() {
|
||||
return None; // not a minorant at all — no entry in family_idx numbering either
|
||||
}
|
||||
if is_non_monomorphic_minorant(mask) {
|
||||
let value = entropy_annex.get(compact).unwrap_or(0.0);
|
||||
compact += 1;
|
||||
Some(value)
|
||||
} else {
|
||||
Some(0.0) // monomorphic — trivially zero by convention, never stored
|
||||
}
|
||||
}))
|
||||
/// Second value: the number of genomes the entropy was computed over.
|
||||
pub(crate) fn family_entropy_under(genome_mask: &[u8], no_ambiguity: bool) -> Option<(f64, usize)> {
|
||||
if no_ambiguity {
|
||||
family_entropy_4(genome_mask)
|
||||
} else {
|
||||
family_entropy(genome_mask)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
//! Per-layer family annexes built from a `scan_layer_families` sweep: the
|
||||
//! entropy annex and the pair-weight annex (both a
|
||||
//! [`crate::siblings::FamilyScalarAnnex`]), the quartet annex (a
|
||||
//! [`obicompactvec::PersistentBitVec`], see `crate::siblings::quartet_annex`)
|
||||
//! and the index-level family-overlap accumulators. The entropy, quartet and
|
||||
//! overlap builders all need the identical per-family `genome_mask`
|
||||
//! resolution, so one sweep feeds every one that has something missing; the
|
||||
//! pair weights need the *finished* overlap matrix, hence a second sweep
|
||||
//! (`ensure_layer_pair_weight_annexes`).
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use obicompactvec::PersistentBitVecBuilder;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::entropy::family_entropy_under;
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::masking::is_quartet_informative;
|
||||
use super::minorant_selection::non_monomorphic_reindex_layer;
|
||||
use super::pair_weight::pair_balance_weight;
|
||||
use crate::siblings::quartet_annex::quartet_annex_file_name;
|
||||
use crate::siblings::{FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind};
|
||||
|
||||
/// Which annexes a caller needs, each listed once per `--no-ambiguity`
|
||||
/// counting convention (`false` = genomes carrying several members count as a
|
||||
/// state of their own, `true` = they are left out) — an empty list asks for
|
||||
/// none. A pair weight is computed from the family-overlap annex of its own
|
||||
/// convention, so asking for one asks for the other.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct AnnexRequest {
|
||||
pub entropy: Vec<bool>,
|
||||
pub quartet: Vec<bool>,
|
||||
pub overlap: Vec<bool>,
|
||||
pub pair_weight: Vec<bool>,
|
||||
}
|
||||
|
||||
impl AnnexRequest {
|
||||
/// Everything under both conventions — what `--sibling-annex` precomputes.
|
||||
pub(crate) fn everything() -> Self {
|
||||
let both = vec![false, true];
|
||||
Self { entropy: both.clone(), quartet: both.clone(), overlap: both.clone(), pair_weight: both }
|
||||
}
|
||||
|
||||
/// The overlap conventions to ensure: the requested ones plus every one a
|
||||
/// requested pair weight is computed from, each once.
|
||||
pub(crate) fn overlap_conventions(&self) -> Vec<bool> {
|
||||
dedup(self.overlap.iter().chain(&self.pair_weight).copied())
|
||||
}
|
||||
|
||||
/// The pair-weight conventions requested, each once.
|
||||
pub(crate) fn pair_weight_conventions(&self) -> Vec<bool> {
|
||||
dedup(self.pair_weight.iter().copied())
|
||||
}
|
||||
}
|
||||
|
||||
fn dedup(variants: impl IntoIterator<Item = bool>) -> Vec<bool> {
|
||||
let mut v: Vec<bool> = variants.into_iter().collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
}
|
||||
|
||||
/// Builds the entropy and quartet annexes of one layer that `request` asks
|
||||
/// for and that don't exist yet, and feeds `overlap_accs` (one accumulator
|
||||
/// per family-overlap annex still missing, see
|
||||
/// [`crate::siblings::FamilyOverlap`]'s own module docs), in one
|
||||
/// `scan_layer_families` pass. A no-op (no scan at all) when everything
|
||||
/// requested already exists for this layer *and* `overlap_accs` is empty.
|
||||
/// Bounded to non-monomorphic minorants only
|
||||
/// ([`non_monomorphic_reindex_layer`], a cheap annex-only pass — no
|
||||
/// cross-partition resolution) whenever a scan does happen: monomorphism is
|
||||
/// already known from the annex bits alone, so there is no reason to pay
|
||||
/// `scan_layer_families`'s expensive per-genome resolution for the ~98% of
|
||||
/// minorants that are monomorphic only to discard the result.
|
||||
pub(crate) fn ensure_layer_family_annexes(
|
||||
cache: &IndexCache,
|
||||
partition: usize,
|
||||
layer_idx: usize,
|
||||
n_genomes: usize,
|
||||
fast_mode: bool,
|
||||
request: &AnnexRequest,
|
||||
overlap_accs: &mut [(bool, FamilyOverlapAccumulator)],
|
||||
) -> OKIResult<()> {
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
|
||||
let missing = |name: &dyn Fn(bool) -> &'static str, variants: &[bool]| -> Vec<(bool, PathBuf)> {
|
||||
dedup(variants.iter().copied())
|
||||
.into_iter()
|
||||
.map(|no_ambiguity| (no_ambiguity, layer.dir().join(name(no_ambiguity))))
|
||||
.filter(|(_, path)| !path.exists())
|
||||
.collect()
|
||||
};
|
||||
let missing_entropy = missing(&|nb| ScalarKind::Entropy.file_name(nb), &request.entropy);
|
||||
let missing_quartet = missing(&quartet_annex_file_name, &request.quartet);
|
||||
if missing_entropy.is_empty() && missing_quartet.is_empty() && overlap_accs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
|
||||
let eligible: HashSet<usize> = reindex.keys().copied().collect();
|
||||
let mut entropy_builders = missing_entropy
|
||||
.iter()
|
||||
.map(|(no_ambiguity, path)| {
|
||||
FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *no_ambiguity)
|
||||
.map(|b| (*no_ambiguity, b))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()
|
||||
.map_err(OKIError::Io)?;
|
||||
let mut quartet_builders = missing_quartet
|
||||
.iter()
|
||||
.map(|(no_ambiguity, path)| {
|
||||
PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*no_ambiguity, b))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
scan_layer_families(
|
||||
cache,
|
||||
partition,
|
||||
layer_idx,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, mask, genome_mask| {
|
||||
debug_assert!(
|
||||
mask.family_size() >= 2,
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
let compact = reindex[&family_idx];
|
||||
for (no_ambiguity, builder) in entropy_builders.iter_mut() {
|
||||
if let Some((h, _)) = family_entropy_under(genome_mask, *no_ambiguity) {
|
||||
builder.set(compact, h as f32);
|
||||
}
|
||||
}
|
||||
for (no_ambiguity, builder) in quartet_builders.iter_mut() {
|
||||
builder.set(compact, is_quartet_informative(genome_mask, *no_ambiguity));
|
||||
}
|
||||
for (_, acc) in overlap_accs.iter_mut() {
|
||||
acc.add_family(genome_mask);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
for (_, builder) in entropy_builders {
|
||||
builder.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
for (_, builder) in quartet_builders {
|
||||
builder.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Builds the pair-weight annexes of one layer that don't exist yet, one per
|
||||
/// `(no_ambiguity, inverse overlap matrix)` in `matrices` (see
|
||||
/// [`super::pair_weight::inverse_overlap_matrix`]), in one scan. A no-op (no
|
||||
/// scan) when they all exist. Needs the family-overlap annex of each
|
||||
/// convention to be complete: run after the sweep that builds it.
|
||||
pub(crate) fn ensure_layer_pair_weight_annexes(
|
||||
cache: &IndexCache,
|
||||
partition: usize,
|
||||
layer_idx: usize,
|
||||
n_genomes: usize,
|
||||
fast_mode: bool,
|
||||
matrices: &[(bool, Vec<f32>)],
|
||||
) -> OKIResult<()> {
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
|
||||
let pending: Vec<&(bool, Vec<f32>)> = matrices
|
||||
.iter()
|
||||
.filter(|(no_ambiguity, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)).exists())
|
||||
.collect();
|
||||
if pending.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
|
||||
let eligible: HashSet<usize> = reindex.keys().copied().collect();
|
||||
let mut builders = pending
|
||||
.iter()
|
||||
.map(|(no_ambiguity, inverse)| {
|
||||
let path = layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity));
|
||||
FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *no_ambiguity)
|
||||
.map(|b| (*no_ambiguity, inverse, b))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
scan_layer_families(
|
||||
cache,
|
||||
partition,
|
||||
layer_idx,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, _mask, genome_mask| {
|
||||
let compact = reindex[&family_idx];
|
||||
for (no_ambiguity, inverse, builder) in builders.iter_mut() {
|
||||
builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse));
|
||||
}
|
||||
},
|
||||
)?;
|
||||
for (_, _, builder) in builders {
|
||||
builder.close().map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -17,6 +17,8 @@
|
||||
//! waste sample quota on sites that die anyway once the excluded genome's
|
||||
//! data is dropped.
|
||||
|
||||
use crate::siblings::helpers::is_carrier;
|
||||
|
||||
/// This genome's masked character state, or `None` for "?" (missing). `m`
|
||||
/// is one `genome_mask` entry (bit `b` set iff this genome carries the
|
||||
/// family member whose own canonical central base is `b`; `0` = no member
|
||||
@@ -64,6 +66,97 @@ pub(crate) fn survives_masking(masked: &[Option<u8>], excluded: &[bool]) -> bool
|
||||
seen.count_ones() >= 2
|
||||
}
|
||||
|
||||
/// Whether `counts` — the number of genomes per *called* state, indexed by
|
||||
/// the `0..=15` state value, index `0` (absence) never counted — shows a
|
||||
/// bipartition: at least two distinct states each carried by at least two
|
||||
/// genomes. That is the pattern that can separate two pairs of genomes from
|
||||
/// each other (`AB|CD`); a singleton variant, or a family carried by fewer
|
||||
/// than four genomes, fits every topology and informs none.
|
||||
fn has_bipartition(counts: &[u32; 16]) -> bool {
|
||||
counts[1..].iter().filter(|&&c| c >= 2).count() >= 2
|
||||
}
|
||||
|
||||
/// Quartet-informativeness of a family from its raw per-genome masks alone,
|
||||
/// under the `--no-ambiguity` counting convention: absent genomes (`0`)
|
||||
/// never count, and with `no_ambiguity` a genome carrying several members
|
||||
/// doesn't either — otherwise its combination of bases is a state of its
|
||||
/// own. Ignores `--exclude-genome` and `--free-loss` on purpose: absence is
|
||||
/// never a called state here, and removing genomes can only destroy a
|
||||
/// bipartition, never create one — so `false` is final, and `true` is only
|
||||
/// a necessary condition, re-checked exactly by [`survives_quartet`] once
|
||||
/// the family is resolved. This is what the persisted quartet annex stores.
|
||||
pub(crate) fn is_quartet_informative(genome_mask: &[u8], no_ambiguity: bool) -> bool {
|
||||
let mut counts = [0u32; 16];
|
||||
for &m in genome_mask {
|
||||
if !is_carrier(m, no_ambiguity) {
|
||||
continue;
|
||||
}
|
||||
counts[(m & 0b1111) as usize] += 1;
|
||||
}
|
||||
has_bipartition(&counts)
|
||||
}
|
||||
|
||||
/// Exact quartet check on an already-[`masked_state`]d family, over the
|
||||
/// *included* genomes (`excluded` as in [`survives_masking`]). Only called
|
||||
/// states count: a genome whose masked state is `0` (absence kept as a real
|
||||
/// state, i.e. no `--free-loss`) is not a carrier of anything.
|
||||
pub(crate) fn survives_quartet(masked: &[Option<u8>], excluded: &[bool]) -> bool {
|
||||
let mut counts = [0u32; 16];
|
||||
for (g, &state) in masked.iter().enumerate() {
|
||||
if excluded.get(g).copied().unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
if let Some(state) = state {
|
||||
counts[(state & 0b1111) as usize] += 1;
|
||||
}
|
||||
}
|
||||
has_bipartition(&counts)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod quartet_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn two_pairs_of_distinct_states_is_informative() {
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], false));
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn singleton_variant_is_not_informative() {
|
||||
// three genomes in one state plus one lone variant: fits any topology
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], false));
|
||||
// fewer than four carriers can never be informative
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], false));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_genomes_are_not_a_state() {
|
||||
// two carriers of one base, two absences: absence must not form the 2nd pair
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_drops_multi_member_genomes_from_the_count() {
|
||||
// the two-member genomes form a pair of their own state without the flag
|
||||
let masks = [0b0001u8, 0b0001, 0b0011, 0b0011];
|
||||
assert!(is_quartet_informative(&masks, false));
|
||||
assert!(!is_quartet_informative(&masks, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_check_honours_exclusions_and_missing() {
|
||||
let masked = [Some(1u8), Some(1), Some(4), Some(4), None, Some(0)];
|
||||
assert!(survives_quartet(&masked, &[]));
|
||||
// excluding one genome of the second pair breaks the bipartition
|
||||
assert!(!survives_quartet(&masked, &[false, false, true]));
|
||||
// an absence kept as state 0 (no --free-loss) is not a called state
|
||||
assert!(!survives_quartet(&[Some(1), Some(1), Some(0), Some(0)], &[]));
|
||||
}
|
||||
}
|
||||
|
||||
/// IUPAC ambiguity code for a [`masked_state`] value (`0..=15`): single bit
|
||||
/// -> the plain base; 2 or 3 bits -> the matching IUPAC ambiguity code
|
||||
/// (preserves partial information instead of collapsing to `N`, the same
|
||||
|
||||
@@ -11,7 +11,7 @@ use std::path::Path;
|
||||
use obicompactvec::TempBitVecBuilder;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use crate::siblings::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex};
|
||||
use crate::siblings::{ANNEX_FILE_NAME, FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex};
|
||||
|
||||
pub(crate) fn is_non_monomorphic_minorant(mask: FamilyMask) -> bool {
|
||||
mask.is_minorant() && mask.siblings() >= 1
|
||||
@@ -82,3 +82,49 @@ pub(crate) fn eligible_bitset_layer(layer_dir: &Path) -> OKIResult<TempBitVecBui
|
||||
}
|
||||
Ok(eligible)
|
||||
}
|
||||
|
||||
/// Streams one stored `f32` per minorant of the layer, **in `family_idx`
|
||||
/// order** — monomorphic minorants included, valued `0.0` by convention
|
||||
/// (trivially uninformative, never computed or stored in the compact
|
||||
/// annex). Exists so a caller can `.zip()` this directly against any other
|
||||
/// `family_idx`-ordered stream (e.g. `SiblingLayerExt::iter_minorants`)
|
||||
/// instead of separately consulting [`non_monomorphic_reindex_layer`]'s map
|
||||
/// at each site.
|
||||
///
|
||||
/// Reads the `kind` annex (entropy, pair weight) built under the
|
||||
/// `no_ambiguity` counting convention; it must already exist
|
||||
/// (`ensure_layer_family_annexes` / `ensure_layer_pair_weight_annexes`
|
||||
/// first) — read positionally (`compact` advances only on non-monomorphic
|
||||
/// minorants, mirroring exactly how the annex was written), never
|
||||
/// recomputed. A file whose header records the other convention is rejected
|
||||
/// rather than silently read as this one.
|
||||
pub(crate) fn iter_full_scalar(
|
||||
layer_dir: &Path,
|
||||
kind: ScalarKind,
|
||||
no_ambiguity: bool,
|
||||
) -> OKIResult<impl Iterator<Item = f32> + use<>> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
|
||||
let scalars =
|
||||
FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(no_ambiguity)), kind).map_err(OKIError::Io)?;
|
||||
if scalars.no_ambiguity() != no_ambiguity {
|
||||
return Err(OKIError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
format!("{kind:?} annex was built under the other --no-ambiguity convention"),
|
||||
)));
|
||||
}
|
||||
|
||||
let mut compact = 0usize;
|
||||
Ok((0..annex.len()).filter_map(move |slot| {
|
||||
let mask = annex.get(slot)?;
|
||||
if !mask.is_minorant() {
|
||||
return None; // not a minorant at all — no entry in family_idx numbering either
|
||||
}
|
||||
if is_non_monomorphic_minorant(mask) {
|
||||
let value = scalars.get(compact).unwrap_or(0.0);
|
||||
compact += 1;
|
||||
Some(value)
|
||||
} else {
|
||||
Some(0.0) // monomorphic — trivially zero by convention, never stored
|
||||
}
|
||||
}))
|
||||
}
|
||||
@@ -17,10 +17,13 @@ mod alignment;
|
||||
mod annex;
|
||||
mod cardcomp;
|
||||
mod entropy;
|
||||
mod family_annexes;
|
||||
mod family_scan;
|
||||
mod masking;
|
||||
mod minorant_selection;
|
||||
mod pair_weight;
|
||||
mod pairwise;
|
||||
mod quartet;
|
||||
mod sankoff;
|
||||
mod scan;
|
||||
mod snp_distance;
|
||||
@@ -38,10 +41,15 @@ pub use sankoff::SankoffBundle;
|
||||
pub use scan::{LayerReducer, sampled_scan};
|
||||
pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
|
||||
pub use stats::SiblingAnnexStats;
|
||||
pub use subsample::{EntropyBias, SurvivingFamily};
|
||||
pub use subsample::{EntropyBias, SamplingSpec, SurvivingFamily};
|
||||
|
||||
pub(crate) use annex::build_layer_sibling_annex;
|
||||
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4};
|
||||
pub(crate) use entropy::{family_entropy, family_entropy_4};
|
||||
pub(crate) use family_annexes::{
|
||||
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
|
||||
};
|
||||
pub(crate) use pair_weight::inverse_overlap_matrix;
|
||||
pub(crate) use quartet::restrict_to_quartet;
|
||||
pub(crate) use family_scan::{Selection, scan_layer_families};
|
||||
pub(crate) use minorant_selection::non_monomorphic_reindex_layer;
|
||||
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
//! `--pair-balance`: a per-family sampling weight that favours the families
|
||||
//! covering the pairs of genomes the index shares the fewest families between.
|
||||
//!
|
||||
//! The weight of a family is the **mean**, over the pairs of genomes carrying
|
||||
//! it, of `1 / O(pair)`, where `O` is the index-wide number of variable
|
||||
//! families the pair shares (the family-overlap annex, under the same
|
||||
//! `--no-ambiguity` counting convention). It is the mean, not the sum: a sum
|
||||
//! grows with the square of the number of carriers, so it would chase the
|
||||
//! families carried by most genomes — in a skimming dataset typically
|
||||
//! high-copy regions, sampled deeply for reasons unrelated to phylogeny —
|
||||
//! instead of the ones covering rare pairs.
|
||||
//!
|
||||
//! Values are only meaningful relative to each other; the sampler rescales
|
||||
//! them per layer (`SamplingSpec::pair_balance`).
|
||||
|
||||
use crate::siblings::FamilyOverlap;
|
||||
use crate::siblings::helpers::is_carrier;
|
||||
|
||||
/// `1 / O` for every pair of genomes, as a dense symmetric `n × n` matrix
|
||||
/// (diagonal and pairs sharing nothing hold `0.0`) — what
|
||||
/// [`pair_balance_weight`] reads, built once per run from the persisted
|
||||
/// family-overlap annex.
|
||||
pub(crate) fn inverse_overlap_matrix(overlap: &FamilyOverlap) -> Vec<f32> {
|
||||
let n = overlap.n_genomes();
|
||||
let mut inv = vec![0.0f32; n * n];
|
||||
for i in 0..n {
|
||||
for j in (i + 1)..n {
|
||||
let shared = overlap.get(i, j);
|
||||
if shared > 0 {
|
||||
let r = 1.0 / shared as f32;
|
||||
inv[i * n + j] = r;
|
||||
inv[j * n + i] = r;
|
||||
}
|
||||
}
|
||||
}
|
||||
inv
|
||||
}
|
||||
|
||||
/// The pair-balance weight of one family from its raw per-genome masks:
|
||||
/// mean of `inv_overlap` over the pairs of carriers (carriers per
|
||||
/// [`is_carrier`], under `no_ambiguity`). `0.0` for a family with fewer than
|
||||
/// two carriers — it covers no pair, so it can balance none.
|
||||
pub(crate) fn pair_balance_weight(
|
||||
genome_mask: &[u8],
|
||||
no_ambiguity: bool,
|
||||
inv_overlap: &[f32],
|
||||
) -> f32 {
|
||||
let n = genome_mask.len();
|
||||
let carriers: Vec<usize> =
|
||||
(0..n).filter(|&g| is_carrier(genome_mask[g], no_ambiguity)).collect();
|
||||
let k = carriers.len();
|
||||
if k < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
let mut sum = 0.0f64;
|
||||
for (a, &i) in carriers.iter().enumerate() {
|
||||
for &j in &carriers[a + 1..] {
|
||||
sum += f64::from(inv_overlap[i * n + j]);
|
||||
}
|
||||
}
|
||||
(sum / (k * (k - 1) / 2) as f64) as f32
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// 4 genomes; pair (0,1) shares 2 families, (0,2) shares 4, the others 1.
|
||||
fn matrix() -> Vec<f32> {
|
||||
let n = 4;
|
||||
let mut m = vec![0.0f32; n * n];
|
||||
let mut set = |i: usize, j: usize, shared: f32| {
|
||||
m[i * n + j] = 1.0 / shared;
|
||||
m[j * n + i] = 1.0 / shared;
|
||||
};
|
||||
set(0, 1, 2.0);
|
||||
set(0, 2, 4.0);
|
||||
for (i, j) in [(0, 3), (1, 2), (1, 3), (2, 3)] {
|
||||
set(i, j, 1.0);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_pair_weight_is_the_inverse_overlap() {
|
||||
let w = pair_balance_weight(&[1, 1, 0, 0], false, &matrix());
|
||||
assert!((w - 0.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weight_is_the_mean_not_the_sum_over_pairs() {
|
||||
// carriers 0, 1, 2: pairs (0,1)=1/2, (0,2)=1/4, (1,2)=1 -> mean 7/12
|
||||
let w = pair_balance_weight(&[1, 1, 1, 0], false, &matrix());
|
||||
assert!((w - 7.0 / 12.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fewer_than_two_carriers_weigh_nothing() {
|
||||
assert_eq!(pair_balance_weight(&[1, 0, 0, 0], false, &matrix()), 0.0);
|
||||
assert_eq!(pair_balance_weight(&[0, 0, 0, 0], false, &matrix()), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_leaves_multi_member_genomes_out_of_the_pairs() {
|
||||
let masks = [1u8, 1, 0b0011, 0];
|
||||
// without the flag genome 2 carries the family: pairs (0,1), (0,2), (1,2)
|
||||
let with = pair_balance_weight(&masks, false, &matrix());
|
||||
// with it, only the pair (0,1) remains
|
||||
let without = pair_balance_weight(&masks, true, &matrix());
|
||||
assert!((with - 7.0 / 12.0).abs() < 1e-6);
|
||||
assert!((without - 0.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rare_pair_outweighs_a_common_one() {
|
||||
// genomes (0,2) share many families, (1,3) few
|
||||
let common = pair_balance_weight(&[1, 0, 1, 0], false, &matrix());
|
||||
let rare = pair_balance_weight(&[0, 1, 0, 1], false, &matrix());
|
||||
assert!(rare > common);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
//! `--quartet`: restricting a layer's eligible families to the
|
||||
//! quartet-informative ones, from the persisted quartet annex.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use obicompactvec::{PersistentBitVec, TempBitVecBuilder};
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::minorant_selection::is_non_monomorphic_minorant;
|
||||
use crate::siblings::quartet_annex::quartet_annex_file_name;
|
||||
use crate::siblings::{ANNEX_FILE_NAME, SiblingAnnex};
|
||||
|
||||
/// Clears, in `eligible` (indexed by `family_idx`, monomorphic minorants
|
||||
/// included), every family the quartet annex flags as not quartet-informative
|
||||
/// under the `no_ambiguity` counting convention. The annex is indexed by the
|
||||
/// compact non-monomorphic ordinal, so the sibling annex is walked in the same
|
||||
/// order to translate one numbering into the other — annex-only, no
|
||||
/// cross-partition resolution. The annex must already exist
|
||||
/// ([`super::family_annexes::ensure_layer_family_annexes`] first).
|
||||
pub(crate) fn restrict_to_quartet(
|
||||
eligible: &mut TempBitVecBuilder,
|
||||
layer_dir: &Path,
|
||||
no_ambiguity: bool,
|
||||
) -> OKIResult<()> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
|
||||
let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(no_ambiguity)))
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
let mut family_idx = 0usize;
|
||||
let mut compact = 0usize;
|
||||
for slot in 0..annex.len() {
|
||||
let Some(mask) = annex.get(slot) else { continue };
|
||||
if !mask.is_minorant() {
|
||||
continue;
|
||||
}
|
||||
if is_non_monomorphic_minorant(mask) {
|
||||
if !flags.get(compact) {
|
||||
eligible.set(family_idx, false);
|
||||
}
|
||||
compact += 1;
|
||||
}
|
||||
family_idx += 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -14,7 +14,7 @@ use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
|
||||
use super::stats::sibling_family_size_histogram;
|
||||
use super::subsample::{EntropyBias, SurvivingFamily, sample_index};
|
||||
use super::subsample::{SamplingSpec, SurvivingFamily, sample_index};
|
||||
|
||||
/// One accumulator fed every layer's [`SurvivingFamily`] batch during a
|
||||
/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of
|
||||
@@ -42,26 +42,24 @@ fn resolve_target(cache: &IndexCache, n: Option<usize>) -> OKIResult<usize> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one shared, possibly-subsampled/entropy-biased selection over
|
||||
/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see
|
||||
/// [`sample_index`]'s own docs for what each controls), feeding every
|
||||
/// Run one shared, possibly-subsampled/entropy-biased/quartet-restricted
|
||||
/// selection over `cache` (`n`/`spec`/`excluded` — see [`sample_index`]'s
|
||||
/// own docs and [`SamplingSpec`] for what each controls), feeding every
|
||||
/// surviving family of every visited layer to every reducer in
|
||||
/// `reducers`, in the same order, before moving to the next layer. A
|
||||
/// no-op (not an error) if the resolved target is `0`.
|
||||
pub fn sampled_scan(
|
||||
cache: &IndexCache,
|
||||
n: Option<usize>,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
spec: SamplingSpec,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
reducers: &mut [&mut dyn LayerReducer],
|
||||
) -> OKIResult<()> {
|
||||
let target = resolve_target(cache, n)?;
|
||||
if target == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| {
|
||||
sample_index(cache, target, spec, excluded, 0, |partition, layer, _raw_index, survivors| {
|
||||
for reducer in reducers.iter_mut() {
|
||||
reducer.reduce(partition, layer, &survivors);
|
||||
}
|
||||
|
||||
@@ -51,12 +51,11 @@ use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::OKIResult;
|
||||
use rand::RngExt;
|
||||
|
||||
use super::entropy::iter_full_entropy;
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::is_fast_mode;
|
||||
use super::masking::{masked_state, survives_masking};
|
||||
use super::minorant_selection::eligible_bitset_layer;
|
||||
use crate::siblings::FamilyMask;
|
||||
use super::masking::{masked_state, survives_masking, survives_quartet};
|
||||
use super::minorant_selection::{eligible_bitset_layer, iter_full_scalar};
|
||||
use super::{AnnexRequest, is_fast_mode, restrict_to_quartet};
|
||||
use crate::siblings::{FamilyMask, ScalarKind};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
|
||||
/// Resolution rounds a single layer's sampling may take before giving up on
|
||||
@@ -96,21 +95,61 @@ pub struct EntropyBias {
|
||||
pub sigma: f64,
|
||||
}
|
||||
|
||||
/// [`iter_full_entropy`], rotated to start at `start` and wrap once back to
|
||||
/// it — the same `family_idx` order [`sample_layer`]'s own circular index
|
||||
/// sequence walks, so the two can be `.zip()`-ed directly. Two more streamed
|
||||
/// passes over the layer's (small) annex/entropy files, not a materialised
|
||||
/// `Vec`/`HashMap` — `iter_full_entropy` itself never allocates more than
|
||||
/// its own iterator state.
|
||||
fn circular_entropy_values(
|
||||
/// How a draw is conditioned: the character masking (`--free-loss`,
|
||||
/// `--no-ambiguity`), the quartet restriction (`--quartet`: only families
|
||||
/// that can separate two pairs of genomes) and the optional entropy bias.
|
||||
/// `--no-ambiguity` also selects the counting convention of the persisted
|
||||
/// entropy and quartet annexes the draw reads, so the draw and the alignment
|
||||
/// it feeds always describe the same genomes. `pub`, not `pub(crate)`: part
|
||||
/// of the public signature of [`super::scan::sampled_scan`].
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct SamplingSpec {
|
||||
pub free_loss: bool,
|
||||
pub no_ambiguity: bool,
|
||||
pub quartet: bool,
|
||||
/// `--pair-balance`: weight each family by the mean, over the pairs of
|
||||
/// genomes carrying it, of 1 / (number of families the pair shares), so
|
||||
/// the draw favours the families covering the rarest pairs (see
|
||||
/// `algorithms::pair_weight`).
|
||||
pub pair_balance: bool,
|
||||
pub entropy_bias: Option<EntropyBias>,
|
||||
}
|
||||
|
||||
/// [`iter_full_scalar`] of `kind` (entropy, pair weight), rotated to start at
|
||||
/// `start` and wrap once back to it — the same `family_idx` order
|
||||
/// [`sample_layer`]'s own circular index sequence walks, so the streams can be
|
||||
/// `.zip()`-ed directly. Two more streamed passes over the layer's (small)
|
||||
/// annex files, not a materialised `Vec`/`HashMap` — `iter_full_scalar`
|
||||
/// itself never allocates more than its own iterator state. `no_ambiguity`
|
||||
/// selects the annex built under the same counting convention as the masking
|
||||
/// the survivors go through afterwards.
|
||||
fn circular_scalar_values(
|
||||
layer_dir: &Path,
|
||||
kind: ScalarKind,
|
||||
start: usize,
|
||||
no_ambiguity: bool,
|
||||
) -> OKIResult<impl Iterator<Item = f32> + use<>> {
|
||||
let head = iter_full_entropy(layer_dir)?.skip(start);
|
||||
let tail = iter_full_entropy(layer_dir)?.take(start);
|
||||
let head = iter_full_scalar(layer_dir, kind, no_ambiguity)?.skip(start);
|
||||
let tail = iter_full_scalar(layer_dir, kind, no_ambiguity)?.take(start);
|
||||
Ok(head.chain(tail))
|
||||
}
|
||||
|
||||
/// One layer's place in a draw, planned before any layer is sampled: which of
|
||||
/// its families may be drawn, and how much of the index-wide quota it
|
||||
/// deserves.
|
||||
struct LayerPlan {
|
||||
partition: usize,
|
||||
layer: usize,
|
||||
/// One bit per minorant (`family_idx` order): still eligible to be drawn.
|
||||
eligible: TempBitVecBuilder,
|
||||
n_minorants: usize,
|
||||
/// Total sampling weight of the eligible families: their number, or, with
|
||||
/// `--pair-balance`, the sum of their pair weights. A layer's quota is its
|
||||
/// share of this mass over every layer, so a layer rich in families that
|
||||
/// cover rare pairs gets more of the draw.
|
||||
mass: f64,
|
||||
}
|
||||
|
||||
/// Samples up to `n` non-monomorphic families index-wide, proportionally
|
||||
/// per layer, calling `on_layer(partition, layer, survivors)` once per
|
||||
/// cached layer that contributed anything, with every family from that
|
||||
@@ -159,10 +198,8 @@ fn circular_entropy_values(
|
||||
pub(crate) fn sample_index(
|
||||
cache: &IndexCache,
|
||||
n: usize,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
spec: SamplingSpec,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
skip_layers: usize,
|
||||
mut on_layer: impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
|
||||
) -> OKIResult<usize> {
|
||||
@@ -170,64 +207,79 @@ pub(crate) fn sample_index(
|
||||
let fast_mode = is_fast_mode(cache);
|
||||
|
||||
// Entropy-biased mode reads each candidate's entropy from the entropy
|
||||
// annex (`EntropyWeights::open`, per layer, below) — build any missing
|
||||
// one now, once, rather than mid-walk.
|
||||
if entropy_bias.is_some() {
|
||||
cache.ensure_entropy_and_overlap_annexes()?;
|
||||
// annex, `--quartet` restricts the candidates through the quartet annex,
|
||||
// and `--pair-balance` weights them by the pair-weight annex (all per
|
||||
// layer, below, each under this run's `--no-ambiguity` counting
|
||||
// convention) — build any missing one now, once, rather than mid-walk.
|
||||
let request = AnnexRequest {
|
||||
entropy: spec.entropy_bias.map(|_| vec![spec.no_ambiguity]).unwrap_or_default(),
|
||||
quartet: if spec.quartet { vec![spec.no_ambiguity] } else { Vec::new() },
|
||||
overlap: Vec::new(),
|
||||
pair_weight: if spec.pair_balance { vec![spec.no_ambiguity] } else { Vec::new() },
|
||||
};
|
||||
if !request.entropy.is_empty() || !request.quartet.is_empty() || !request.pair_weight.is_empty() {
|
||||
cache.ensure_family_annexes(&request)?;
|
||||
}
|
||||
|
||||
// Every cached layer's eligibility bitset, built up front — annex-only,
|
||||
// no cross-partition resolution, so cheap even summed over the whole
|
||||
// index — needed before any layer's quota can be computed (each
|
||||
// layer's share depends on every other layer's own count).
|
||||
let mut layers: Vec<(usize, usize, TempBitVecBuilder, usize)> = Vec::new();
|
||||
for part in cache.partitions() {
|
||||
let n_layer = cache.n_layer(part).unwrap_or(0);
|
||||
for l in 0..n_layer {
|
||||
// Every cached layer's plan, built up front — annex-only, no
|
||||
// cross-partition resolution, so cheap even summed over the whole index
|
||||
// — needed before any layer's quota can be computed (each layer's share
|
||||
// depends on every other layer's own mass).
|
||||
let mut plans: Vec<LayerPlan> = Vec::new();
|
||||
for partition in cache.partitions() {
|
||||
let n_layer = cache.n_layer(partition).unwrap_or(0);
|
||||
for layer_idx in 0..n_layer {
|
||||
let layer = cache
|
||||
.get_layer(part, l)
|
||||
.expect("cache-consistent: n_layer(part) already bounds l");
|
||||
let eligible = eligible_bitset_layer(layer.dir())?;
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: n_layer(partition) already bounds layer_idx");
|
||||
let mut eligible = eligible_bitset_layer(layer.dir())?;
|
||||
if spec.quartet {
|
||||
restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?;
|
||||
}
|
||||
let n_minorants = eligible.view().len();
|
||||
layers.push((part, l, eligible, n_minorants));
|
||||
let mass = if spec.pair_balance {
|
||||
iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)?
|
||||
.enumerate()
|
||||
.filter(|&(family_idx, _)| eligible.view().get(family_idx))
|
||||
.map(|(_, weight)| f64::from(weight))
|
||||
.sum()
|
||||
} else {
|
||||
eligible.view().count_ones() as f64
|
||||
};
|
||||
plans.push(LayerPlan { partition, layer: layer_idx, eligible, n_minorants, mass });
|
||||
}
|
||||
}
|
||||
|
||||
let total_eligible: u64 = layers
|
||||
.iter()
|
||||
.map(|(_, _, e, _)| e.view().count_ones())
|
||||
.sum();
|
||||
if total_eligible == 0 {
|
||||
let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum();
|
||||
if total_mass <= 0.0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Quotas by cumulative rounding: layer `i` is allotted
|
||||
// `floor(n × cumulative mass up to i / total) − what the previous layers
|
||||
// were allotted`, so the quotas sum to exactly `n`. Rounding each layer's
|
||||
// share down on its own would lose up to one site per layer — all of
|
||||
// them, when `n` is smaller than the number of layers.
|
||||
let mut total_kept = 0usize;
|
||||
for (raw_index, (part, l, mut eligible, n_minorants)) in layers.into_iter().enumerate() {
|
||||
if raw_index < skip_layers {
|
||||
continue;
|
||||
}
|
||||
let count_layer = eligible.view().count_ones();
|
||||
if count_layer == 0 {
|
||||
continue;
|
||||
}
|
||||
let quota = ((n as u128 * count_layer as u128) / total_eligible as u128) as usize;
|
||||
if quota == 0 {
|
||||
let mut cumulative_mass = 0.0f64;
|
||||
let mut allotted = 0usize;
|
||||
for (raw_index, mut plan) in plans.into_iter().enumerate() {
|
||||
cumulative_mass += plan.mass;
|
||||
let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize;
|
||||
let quota = cumulative_quota.saturating_sub(allotted);
|
||||
allotted = allotted.max(cumulative_quota);
|
||||
if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 {
|
||||
continue;
|
||||
}
|
||||
total_kept += sample_layer(
|
||||
cache,
|
||||
part,
|
||||
l,
|
||||
raw_index,
|
||||
&mut eligible,
|
||||
n_minorants,
|
||||
&mut plan,
|
||||
quota,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
free_loss,
|
||||
no_ambiguity,
|
||||
spec,
|
||||
excluded,
|
||||
entropy_bias,
|
||||
&mut on_layer,
|
||||
)?;
|
||||
}
|
||||
@@ -236,32 +288,28 @@ pub(crate) fn sample_index(
|
||||
}
|
||||
|
||||
/// One layer's share of [`sample_index`] — see the module docs for the
|
||||
/// per-family accept/reject/mark rules and the round-batching scheme.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// per-family accept/reject/mark rules and the round-batching scheme. Each
|
||||
/// eligible family is accepted with probability
|
||||
/// `quota / mass × pair weight × entropy kernel` (capped at 1): the pair
|
||||
/// weight is `1` and `mass` the number of eligible families without
|
||||
/// `--pair-balance`, and the kernel `1` without an entropy bias.
|
||||
fn sample_layer(
|
||||
cache: &IndexCache,
|
||||
partition: usize,
|
||||
layer_idx: usize,
|
||||
raw_index: usize,
|
||||
eligible: &mut TempBitVecBuilder,
|
||||
n_minorants: usize,
|
||||
plan: &mut LayerPlan,
|
||||
quota: usize,
|
||||
n_genomes: usize,
|
||||
fast_mode: bool,
|
||||
free_loss: bool,
|
||||
no_ambiguity: bool,
|
||||
spec: SamplingSpec,
|
||||
excluded: &[bool],
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
on_layer: &mut impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
|
||||
) -> OKIResult<usize> {
|
||||
if n_minorants == 0 {
|
||||
let LayerPlan { partition, layer: layer_idx, n_minorants, mass, .. } = *plan;
|
||||
let eligible = &mut plan.eligible;
|
||||
if n_minorants == 0 || mass <= 0.0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let count_layer = eligible.view().count_ones() as usize;
|
||||
if count_layer == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let p0 = (quota as f64 / count_layer as f64).min(1.0);
|
||||
let p0 = quota as f64 / mass;
|
||||
let layer_dir = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent")
|
||||
@@ -301,34 +349,50 @@ fn sample_layer(
|
||||
|
||||
// This round's own circular walk, resuming exactly where the
|
||||
// previous round left off (`start + visited`, wrapped) — zipped
|
||||
// against that same rotation of the entropy stream when biased, so
|
||||
// each candidate's own kernel weight is read in lockstep, never via
|
||||
// a separate lookup structure. Unbiased draws skip the entropy
|
||||
// stream entirely (a flat `1.0` kernel), never touching the entropy
|
||||
// annex.
|
||||
// against that same rotation of the entropy and pair-weight streams
|
||||
// when they are used, so each candidate's own kernel and weight are
|
||||
// read in lockstep, never via a separate lookup structure. A draw
|
||||
// without the corresponding option skips its stream entirely (flat
|
||||
// `1.0`), never touching the annex.
|
||||
let round_start = (start + visited) % n_minorants;
|
||||
let entropies: Box<dyn Iterator<Item = f32>> = match entropy_bias {
|
||||
Some(_) => Box::new(circular_entropy_values(&layer_dir, round_start)?),
|
||||
let entropies: Box<dyn Iterator<Item = f32>> = match spec.entropy_bias {
|
||||
Some(_) => Box::new(circular_scalar_values(
|
||||
&layer_dir,
|
||||
ScalarKind::Entropy,
|
||||
round_start,
|
||||
spec.no_ambiguity,
|
||||
)?),
|
||||
None => Box::new(std::iter::repeat(0.0f32)), // never read: kernel forced to 1.0 below
|
||||
};
|
||||
let weights: Box<dyn Iterator<Item = f32>> = if spec.pair_balance {
|
||||
Box::new(circular_scalar_values(
|
||||
&layer_dir,
|
||||
ScalarKind::PairWeight,
|
||||
round_start,
|
||||
spec.no_ambiguity,
|
||||
)?)
|
||||
} else {
|
||||
Box::new(std::iter::repeat(1.0f32))
|
||||
};
|
||||
let candidates = (round_start..n_minorants)
|
||||
.chain(0..round_start)
|
||||
.zip(entropies);
|
||||
.zip(entropies)
|
||||
.zip(weights);
|
||||
|
||||
let mut accepted: HashSet<usize> = HashSet::new();
|
||||
for (family_idx, entropy) in candidates {
|
||||
for ((family_idx, entropy), weight) in candidates {
|
||||
visited += 1;
|
||||
if !eligible.view().get(family_idx) {
|
||||
continue;
|
||||
}
|
||||
let kernel = match entropy_bias {
|
||||
let kernel = match spec.entropy_bias {
|
||||
Some(bias) => {
|
||||
let d = (entropy as f64 - bias.mu) / bias.sigma;
|
||||
(-0.5 * d * d).exp()
|
||||
}
|
||||
None => 1.0,
|
||||
};
|
||||
if rng.random::<f64>() < (p0 * kernel).min(1.0) {
|
||||
if rng.random::<f64>() < (p0 * f64::from(weight) * kernel).min(1.0) {
|
||||
eligible.set(family_idx, false);
|
||||
accepted.insert(family_idx);
|
||||
if accepted.len() >= target_this_round {
|
||||
@@ -353,9 +417,13 @@ fn sample_layer(
|
||||
mask.family_size() >= 2,
|
||||
"eligible bitset must only accept non-monomorphic minorants"
|
||||
);
|
||||
let masked: Vec<Option<u8>> =
|
||||
genome_mask.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect();
|
||||
if survives_masking(&masked, excluded) {
|
||||
let masked: Vec<Option<u8>> = genome_mask
|
||||
.iter()
|
||||
.map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity))
|
||||
.collect();
|
||||
if survives_masking(&masked, excluded)
|
||||
&& (!spec.quartet || survives_quartet(&masked, excluded))
|
||||
{
|
||||
survivors.push(SurvivingFamily { family_idx, mask, masked });
|
||||
kept += 1;
|
||||
}
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
//! Persisted, per-layer Shannon entropy (`entropy15`, see
|
||||
//! `algorithms::entropy`), one value per **non-monomorphic minorant
|
||||
//! family** — compact: indexed by the dedicated ordinal
|
||||
//! `algorithms::non_monomorphic_reindex_layer` computes (skips monomorphic
|
||||
//! minorants entirely, ~98% of minorants on real data), never by
|
||||
//! `scan_layer_families`'s own `family_idx` (which counts every minorant,
|
||||
//! monomorphic included).
|
||||
//!
|
||||
//! Exists so repeated reads of the same layer's entropy values (e.g.
|
||||
//! entropy-biased family selection, once reconnected) are a plain
|
||||
//! positional mmap lookup instead of re-paying `scan_layer_families`'s
|
||||
//! cross-partition resolution — building this file costs exactly that same
|
||||
//! full scan the first time, it does not make any single computation of
|
||||
//! entropy itself faster.
|
||||
//!
|
||||
//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat
|
||||
//! per-entry data, read-only after build), just with `f32` payloads and a
|
||||
//! different (compact) numbering — see the module docs above.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PENT";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// Sentinel: not yet computed. Entropy is always `>= 0.0`, so any negative
|
||||
/// value is unambiguous — chosen over `NaN` to avoid `NaN`-comparison
|
||||
/// footguns at every read site.
|
||||
const SENTINEL: f32 = -1.0;
|
||||
|
||||
pub(crate) const ENTROPY_ANNEX_FILE_NAME: &str = "entropy.pent";
|
||||
|
||||
pub struct EntropyAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
}
|
||||
|
||||
impl EntropyAnnex {
|
||||
pub fn open(path: &Path) -> io::Result<Self> {
|
||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||
if mmap.len() < HEADER_SIZE {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"PENT file too short",
|
||||
));
|
||||
}
|
||||
if mmap[0..4] != MAGIC {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PENT magic"));
|
||||
}
|
||||
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
|
||||
if mmap.len() < HEADER_SIZE + n * 4 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"PENT file truncated",
|
||||
));
|
||||
}
|
||||
Ok(Self { mmap, n })
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
/// `None` for the sentinel (not yet computed — shouldn't happen against
|
||||
/// a fully-built annex).
|
||||
pub fn get(&self, idx: usize) -> Option<f32> {
|
||||
let off = HEADER_SIZE + idx * 4;
|
||||
let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap());
|
||||
(v >= 0.0).then_some(v)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct EntropyAnnexBuilder {
|
||||
mmap: MmapMut,
|
||||
}
|
||||
|
||||
impl EntropyAnnexBuilder {
|
||||
/// Create a new annex of `n` entries (a layer's non-monomorphic
|
||||
/// minorant count) at `path`, pre-initialised to the sentinel.
|
||||
pub(crate) fn new(n: usize, path: &Path) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n * 4;
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&MAGIC);
|
||||
mmap[4..8].copy_from_slice(&[0u8; 4]);
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
for i in 0..n {
|
||||
let off = HEADER_SIZE + i * 4;
|
||||
mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes());
|
||||
}
|
||||
Ok(Self { mmap })
|
||||
}
|
||||
|
||||
pub(crate) fn set(&mut self, idx: usize, value: f32) {
|
||||
debug_assert!(value >= 0.0, "entropy must be non-negative, got {value}");
|
||||
let off = HEADER_SIZE + idx * 4;
|
||||
self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
pub(crate) fn close(self) -> io::Result<()> {
|
||||
self.mmap.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn sentinel_is_negative_and_unset_entries_read_as_none() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let builder = EntropyAnnexBuilder::new(4, &path).unwrap();
|
||||
builder.close().unwrap();
|
||||
let annex = EntropyAnnex::open(&path).unwrap();
|
||||
for i in 0..4 {
|
||||
assert_eq!(annex.get(i), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_including_zero_entropy() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = EntropyAnnexBuilder::new(4, &path).unwrap();
|
||||
let values = [0.0f32, 1.5, 3.9, 0.0001];
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
builder.set(i, v);
|
||||
}
|
||||
builder.close().unwrap();
|
||||
let annex = EntropyAnnex::open(&path).unwrap();
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
assert_eq!(annex.get(i), Some(v), "entry {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_sentinel_and_real_values() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = EntropyAnnexBuilder::new(3, &path).unwrap();
|
||||
builder.set(1, 2.0);
|
||||
builder.close().unwrap();
|
||||
let annex = EntropyAnnex::open(&path).unwrap();
|
||||
assert_eq!(annex.get(0), None);
|
||||
assert_eq!(annex.get(1), Some(2.0));
|
||||
assert_eq!(annex.get(2), None);
|
||||
assert_eq!(annex.len(), 3);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
//! [`SiblingBuilder`] — construction triggered *internally*, as a side
|
||||
//! effect of a [`crate::siblings::extensions::SiblingExt`] method (e.g.
|
||||
//! `shannon_entropy_csv`/`family_overlap` ensuring their annexes exist
|
||||
//! before reading them), never called directly from outside this crate.
|
||||
//! `family_overlap` ensuring its annex exists before reading it, or the
|
||||
//! sampler ensuring the entropy/quartet/pair-weight annexes it draws from),
|
||||
//! never called directly from outside this crate.
|
||||
//! Unlike `SiblingExt` (explicit, CLI-facing actions), nothing here is
|
||||
//! meant to be its own CLI flag — same `pub`/`pub(crate)` split
|
||||
//! `obikindexer` uses for its own `IndexBuilder`/`PrivateBuilder`.
|
||||
@@ -10,44 +11,75 @@ use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
use obisys::progress_bar;
|
||||
|
||||
use crate::siblings::algorithms::{ensure_layer_entropy_annex, is_fast_mode};
|
||||
use crate::siblings::{FAMILY_OVERLAP_FILE_NAME, FamilyOverlapAccumulator};
|
||||
use crate::siblings::algorithms::{
|
||||
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
|
||||
inverse_overlap_matrix, is_fast_mode,
|
||||
};
|
||||
use crate::siblings::{FamilyOverlap, FamilyOverlapAccumulator, family_overlap_file_name};
|
||||
|
||||
pub(crate) trait SiblingBuilder {
|
||||
/// Build every cached layer's entropy annex, and the whole index's
|
||||
/// family-overlap annex, that don't already exist — one call covers
|
||||
/// both, whichever of the two a caller actually needs
|
||||
/// (`shannon_entropy_csv`/entropy-biased sampling on one side,
|
||||
/// `family_overlap`/`--min-shared-family` on the other): a missing
|
||||
/// annex always triggers a scan that feeds both, per
|
||||
/// `ensure_layer_entropy_annex`'s own docs. A no-op, cheap
|
||||
/// existence-check pass, once both already exist everywhere.
|
||||
fn ensure_entropy_and_overlap_annexes(&self) -> OKIResult<()>;
|
||||
/// Build, for every cached layer, the annexes `request` asks for that
|
||||
/// don't already exist (entropy, quartet and pair weight, each per
|
||||
/// `--no-ambiguity` counting convention), and the index-level
|
||||
/// family-overlap annexes it asks for or that a pair weight is computed
|
||||
/// from. One call covers all of them, whichever a caller actually needs:
|
||||
/// a missing entropy, quartet or overlap annex triggers one scan that
|
||||
/// feeds every missing one, per `ensure_layer_family_annexes`'s own docs,
|
||||
/// then a second scan builds the missing pair weights, which need the
|
||||
/// finished overlap matrix. A no-op, cheap existence-check pass, once
|
||||
/// everything requested already exists.
|
||||
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>;
|
||||
}
|
||||
|
||||
impl SiblingBuilder for IndexCache {
|
||||
fn ensure_entropy_and_overlap_annexes(&self) -> OKIResult<()> {
|
||||
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
|
||||
let n_genomes = self.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(self);
|
||||
let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity));
|
||||
|
||||
let overlap_path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
|
||||
let mut overlap_acc =
|
||||
(!overlap_path.exists()).then(|| FamilyOverlapAccumulator::new(n_genomes));
|
||||
|
||||
let pb = progress_bar("entropy_overlap_annex", self.n_partition() as u64, "partitions");
|
||||
// Sweep 1: entropy, quartet, and every family-overlap annex still missing.
|
||||
let mut overlap_accs: Vec<(bool, FamilyOverlapAccumulator)> = request
|
||||
.overlap_conventions()
|
||||
.into_iter()
|
||||
.filter(|&no_ambiguity| !overlap_path(no_ambiguity).exists())
|
||||
.map(|no_ambiguity| (no_ambiguity, FamilyOverlapAccumulator::new(n_genomes, no_ambiguity)))
|
||||
.collect();
|
||||
let pb = progress_bar("family_annexes", self.n_partition() as u64, "partitions");
|
||||
for part in self.partitions() {
|
||||
let n_layer = self.n_layer(part).unwrap_or(0);
|
||||
for l in 0..n_layer {
|
||||
ensure_layer_entropy_annex(self, part, l, n_genomes, fast_mode, overlap_acc.as_mut())?;
|
||||
ensure_layer_family_annexes(self, part, l, n_genomes, fast_mode, request, &mut overlap_accs)?;
|
||||
}
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
for (no_ambiguity, acc) in overlap_accs {
|
||||
acc.write(&overlap_path(no_ambiguity)).map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
// Sweep 2: pair weights, from the now-complete overlap annexes.
|
||||
if request.pair_weight.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let matrices = request
|
||||
.pair_weight_conventions()
|
||||
.into_iter()
|
||||
.map(|no_ambiguity| {
|
||||
FamilyOverlap::open(&overlap_path(no_ambiguity))
|
||||
.map(|overlap| (no_ambiguity, inverse_overlap_matrix(&overlap)))
|
||||
.map_err(OKIError::Io)
|
||||
})
|
||||
.collect::<OKIResult<Vec<_>>>()?;
|
||||
let pb = progress_bar("pair_weight_annex", self.n_partition() as u64, "partitions");
|
||||
for part in self.partitions() {
|
||||
let n_layer = self.n_layer(part).unwrap_or(0);
|
||||
for l in 0..n_layer {
|
||||
ensure_layer_pair_weight_annexes(self, part, l, n_genomes, fast_mode, &matrices)?;
|
||||
}
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
if let Some(acc) = overlap_acc {
|
||||
acc.write(&overlap_path).map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -14,12 +14,15 @@ use obikindex::{OKIError, OKIResult};
|
||||
use obisys::progress_bar;
|
||||
|
||||
use crate::siblings::algorithms::{
|
||||
Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4,
|
||||
is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, sibling_annex_stats,
|
||||
sibling_family_size_histogram,
|
||||
AnnexRequest, Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy,
|
||||
family_entropy_4, is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families,
|
||||
sibling_annex_stats, sibling_family_size_histogram,
|
||||
};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap};
|
||||
use crate::siblings::{
|
||||
FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, QUARTET_ANNEX_FILE_NAMES, ScalarKind,
|
||||
family_overlap_file_name,
|
||||
};
|
||||
|
||||
pub trait SiblingExt {
|
||||
/// Build the sibling-count/minorant annex for every layer of every
|
||||
@@ -81,8 +84,10 @@ pub trait SiblingExt {
|
||||
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
|
||||
|
||||
/// The Family Overlap annex — number of shared *variable* families per
|
||||
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs).
|
||||
/// Built (alongside the entropy annex) if missing, same as every other
|
||||
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs),
|
||||
/// under the `--no-ambiguity` counting convention `no_ambiguity` selects
|
||||
/// (genomes carrying several members of a family do not count as
|
||||
/// carrying it when `true`). Built if missing, same as every other
|
||||
/// annex-backed method here. Feeds `--family-overlap`'s own matrix
|
||||
/// output directly, and `--min-shared-family`'s auto-exclusion
|
||||
/// (`FamilyOverlap::mean_row(g) < threshold` — same statistic
|
||||
@@ -90,7 +95,7 @@ pub trait SiblingExt {
|
||||
/// triggers adding `g` to the excluded-genome set consumed by
|
||||
/// `algorithms::scan::sampled_scan`'s callers; this needs no
|
||||
/// computation of its own beyond that check).
|
||||
fn family_overlap(&self) -> OKIResult<FamilyOverlap>;
|
||||
fn family_overlap(&self, no_ambiguity: bool) -> OKIResult<FamilyOverlap>;
|
||||
}
|
||||
|
||||
impl SiblingExt for IndexCache {
|
||||
@@ -115,16 +120,24 @@ impl SiblingExt for IndexCache {
|
||||
.expect("cache-consistent: n_layer(part) already bounds l");
|
||||
part_slots += build_layer_sibling_annex(self, &layer, l, n_parts, fast_mode)?;
|
||||
|
||||
// Invalidate this layer's entropy annex, if any — it was
|
||||
// built against the sibling annex's *previous*
|
||||
// `family_idx` numbering (`non_monomorphic_reindex_layer`
|
||||
// reads the sibling annex), now stale.
|
||||
// `ensure_entropy_and_overlap_annexes` below only builds
|
||||
// what's *missing*, so the old file must be removed here
|
||||
// for it to be rebuilt rather than silently left stale.
|
||||
let entropy_path = layer.dir().join(ENTROPY_ANNEX_FILE_NAME);
|
||||
if entropy_path.exists() {
|
||||
std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?;
|
||||
// Invalidate this layer's entropy and quartet annexes, if
|
||||
// any — they were built against the sibling annex's
|
||||
// *previous* `family_idx` numbering
|
||||
// (`non_monomorphic_reindex_layer` reads the sibling
|
||||
// annex), now stale. `ensure_family_annexes` below only
|
||||
// builds what's *missing*, so the old files must be removed
|
||||
// here for them to be rebuilt rather than silently left
|
||||
// stale.
|
||||
for name in ScalarKind::Entropy
|
||||
.file_names()
|
||||
.into_iter()
|
||||
.chain(ScalarKind::PairWeight.file_names())
|
||||
.chain(QUARTET_ANNEX_FILE_NAMES)
|
||||
{
|
||||
let entropy_path = layer.dir().join(name);
|
||||
if entropy_path.exists() {
|
||||
std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
total_slots += part_slots;
|
||||
@@ -136,13 +149,16 @@ impl SiblingExt for IndexCache {
|
||||
pb.finish_and_clear();
|
||||
tracing::info!("sibling annex built — {total_slots} kmers across {n_parts} partitions");
|
||||
|
||||
// The family-overlap annex is index-level, not per-layer (see its
|
||||
// own module docs) — any layer's sibling annex changing makes the
|
||||
// whole thing stale, so it's invalidated once here rather than
|
||||
// per-layer above.
|
||||
let overlap_path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
|
||||
if overlap_path.exists() {
|
||||
std::fs::remove_file(&overlap_path).map_err(OKIError::Io)?;
|
||||
// The family-overlap annexes are index-level, not per-layer (see
|
||||
// their own module docs) — any layer's sibling annex changing makes
|
||||
// them all stale (and, with them, every pair weight computed from
|
||||
// them, removed per layer above), so they are invalidated once here
|
||||
// rather than per-layer above.
|
||||
for name in FAMILY_OVERLAP_FILE_NAMES {
|
||||
let overlap_path = self.index().dir().join(name);
|
||||
if overlap_path.exists() {
|
||||
std::fs::remove_file(&overlap_path).map_err(OKIError::Io)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Precompute the entropy and family-overlap annexes too, now that
|
||||
@@ -152,19 +168,17 @@ impl SiblingExt for IndexCache {
|
||||
// rather than leaving them to be built lazily on first
|
||||
// `--shannon`/`--entropy-bias`/`--min-shared-family`/
|
||||
// `--family-overlap` use.
|
||||
tracing::info!("building entropy and family-overlap annexes");
|
||||
self.ensure_entropy_and_overlap_annexes()?;
|
||||
tracing::info!("building entropy and quartet (both --no-ambiguity conventions) and family-overlap annexes");
|
||||
self.ensure_family_annexes(&AnnexRequest::everything())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {
|
||||
// Entropy/family-overlap annexes aren't this method's own output,
|
||||
// but ensuring they exist is a cheap prerequisite check, not a
|
||||
// reason to reuse their scan here — see below, this method
|
||||
// deliberately runs its own, differently-scoped one.
|
||||
self.ensure_entropy_and_overlap_annexes()?;
|
||||
|
||||
// Needs only the sibling annex (`non_monomorphic_reindex_layer`
|
||||
// below): this method runs its own, differently-scoped scan and
|
||||
// recomputes every value it writes, so it neither reads nor
|
||||
// requires the entropy/family-overlap annexes.
|
||||
let n_genomes = self.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(self);
|
||||
|
||||
@@ -237,9 +251,9 @@ impl SiblingExt for IndexCache {
|
||||
sibling_annex_stats(self)
|
||||
}
|
||||
|
||||
fn family_overlap(&self) -> OKIResult<FamilyOverlap> {
|
||||
self.ensure_entropy_and_overlap_annexes()?;
|
||||
let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
|
||||
fn family_overlap(&self, no_ambiguity: bool) -> OKIResult<FamilyOverlap> {
|
||||
self.ensure_family_annexes(&AnnexRequest { overlap: vec![no_ambiguity], ..Default::default() })?;
|
||||
let path = self.index().dir().join(family_overlap_file_name(no_ambiguity));
|
||||
FamilyOverlap::open(&path).map_err(OKIError::Io)
|
||||
}
|
||||
}
|
||||
@@ -24,20 +24,43 @@ use std::path::Path;
|
||||
|
||||
use memmap2::Mmap;
|
||||
|
||||
use crate::siblings::helpers::triangle_index;
|
||||
use crate::siblings::helpers::{is_carrier, triangle_index};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"POVL";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair,
|
||||
// upper triangle) follows.
|
||||
// Header: magic(4) + flags(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair,
|
||||
// upper triangle) follows. `flags` bit 0 = `no_ambiguity` (see
|
||||
// `FamilyOverlapAccumulator::new`); older files have `flags == 0`, i.e.
|
||||
// `no_ambiguity == false`, which is what they were computed under.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
const FLAG_NO_AMBIGUITY: u32 = 1;
|
||||
|
||||
pub(crate) const FAMILY_OVERLAP_FILE_NAME: &str = "family_overlap.povl";
|
||||
|
||||
/// Same annex, built with `no_ambiguity == true`.
|
||||
pub(crate) const FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME: &str = "family_overlap.noamb.povl";
|
||||
|
||||
/// Every family-overlap file name, whatever its counting convention — for
|
||||
/// invalidating them all together when the sibling annex they derive from is
|
||||
/// rebuilt.
|
||||
pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 2] =
|
||||
[FAMILY_OVERLAP_FILE_NAME, FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME];
|
||||
|
||||
/// File name of the family-overlap annex built under the given
|
||||
/// `--no-ambiguity` counting convention. The two never share a file.
|
||||
pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str {
|
||||
if no_ambiguity {
|
||||
FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME
|
||||
} else {
|
||||
FAMILY_OVERLAP_FILE_NAME
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only, mmap-backed view of an already-built Family Overlap annex.
|
||||
pub struct FamilyOverlap {
|
||||
mmap: Mmap,
|
||||
n_genomes: usize,
|
||||
no_ambiguity: bool,
|
||||
}
|
||||
|
||||
impl FamilyOverlap {
|
||||
@@ -54,13 +77,22 @@ impl FamilyOverlap {
|
||||
if mmap.len() < HEADER_SIZE + n_pairs * 8 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "POVL file truncated"));
|
||||
}
|
||||
Ok(Self { mmap, n_genomes })
|
||||
let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap());
|
||||
Ok(Self { mmap, n_genomes, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 })
|
||||
}
|
||||
|
||||
pub fn n_genomes(&self) -> usize {
|
||||
self.n_genomes
|
||||
}
|
||||
|
||||
/// The counting convention these counts were computed under: `true`
|
||||
/// when a genome carrying several members of a family was *not* counted
|
||||
/// as carrying it (`--no-ambiguity`), `false` when any non-empty
|
||||
/// state counts as carrying it.
|
||||
pub fn no_ambiguity(&self) -> bool {
|
||||
self.no_ambiguity
|
||||
}
|
||||
|
||||
/// Number of variable families shared by genomes `i` and `j` — `0` for
|
||||
/// `i == j` (not stored; a genome trivially "shares" every one of its
|
||||
/// own families with itself, not a meaningful statistic here).
|
||||
@@ -95,24 +127,30 @@ impl FamilyOverlap {
|
||||
/// `EntropyAnnexBuilder` do for their own, much larger, per-k-mer data.
|
||||
pub(crate) struct FamilyOverlapAccumulator {
|
||||
n_genomes: usize,
|
||||
no_ambiguity: bool,
|
||||
counts: Vec<u64>,
|
||||
}
|
||||
|
||||
impl FamilyOverlapAccumulator {
|
||||
pub(crate) fn new(n_genomes: usize) -> Self {
|
||||
/// `no_ambiguity` is the counting convention: `false` counts every
|
||||
/// genome with a non-empty state as carrying the family, `true` only the
|
||||
/// genomes carrying exactly one member of it — the others are missing
|
||||
/// data in the alignment (`--no-ambiguity`), so they share nothing there.
|
||||
pub(crate) fn new(n_genomes: usize, no_ambiguity: bool) -> Self {
|
||||
let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2;
|
||||
Self { n_genomes, counts: vec![0u64; n_pairs] }
|
||||
Self { n_genomes, no_ambiguity, counts: vec![0u64; n_pairs] }
|
||||
}
|
||||
|
||||
/// One variable family's contribution: every pair of genomes both
|
||||
/// carrying it (`genome_mask[g] != 0`) gets its count incremented once.
|
||||
/// carrying it (per the counting convention) gets its count incremented
|
||||
/// once.
|
||||
pub(crate) fn add_family(&mut self, genome_mask: &[u8]) {
|
||||
for i in 0..self.n_genomes {
|
||||
if genome_mask[i] == 0 {
|
||||
if !is_carrier(genome_mask[i], self.no_ambiguity) {
|
||||
continue;
|
||||
}
|
||||
for j in (i + 1)..self.n_genomes {
|
||||
if genome_mask[j] != 0 {
|
||||
if is_carrier(genome_mask[j], self.no_ambiguity) {
|
||||
self.counts[triangle_index(self.n_genomes, i, j)] += 1;
|
||||
}
|
||||
}
|
||||
@@ -122,7 +160,8 @@ impl FamilyOverlapAccumulator {
|
||||
pub(crate) fn write(&self, path: &Path) -> io::Result<()> {
|
||||
let mut buf = Vec::with_capacity(HEADER_SIZE + self.counts.len() * 8);
|
||||
buf.extend_from_slice(&MAGIC);
|
||||
buf.extend_from_slice(&[0u8; 4]);
|
||||
let flags = if self.no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 };
|
||||
buf.extend_from_slice(&flags.to_le_bytes());
|
||||
buf.extend_from_slice(&(self.n_genomes as u64).to_le_bytes());
|
||||
for &c in &self.counts {
|
||||
buf.extend_from_slice(&c.to_le_bytes());
|
||||
@@ -140,7 +179,7 @@ mod tests {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("test.povl");
|
||||
|
||||
let mut acc = FamilyOverlapAccumulator::new(3);
|
||||
let mut acc = FamilyOverlapAccumulator::new(3, false);
|
||||
acc.add_family(&[1, 1, 0]); // genomes 0,1 share this family
|
||||
acc.add_family(&[1, 1, 1]); // all three share this one
|
||||
acc.add_family(&[0, 1, 1]); // genomes 1,2 share this family
|
||||
@@ -160,8 +199,27 @@ mod tests {
|
||||
fn single_genome_mean_row_is_zero() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("test.povl");
|
||||
FamilyOverlapAccumulator::new(1).write(&path).unwrap();
|
||||
FamilyOverlapAccumulator::new(1, false).write(&path).unwrap();
|
||||
let overlap = FamilyOverlap::open(&path).unwrap();
|
||||
assert_eq!(overlap.mean_row(0), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_does_not_count_multi_member_genomes_as_carriers() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0100]];
|
||||
for no_ambiguity in [false, true] {
|
||||
let path = dir.path().join(family_overlap_file_name(no_ambiguity));
|
||||
let mut acc = FamilyOverlapAccumulator::new(3, no_ambiguity);
|
||||
for m in masks {
|
||||
acc.add_family(m);
|
||||
}
|
||||
acc.write(&path).unwrap();
|
||||
let overlap = FamilyOverlap::open(&path).unwrap();
|
||||
assert_eq!(overlap.no_ambiguity(), no_ambiguity);
|
||||
// genome 2 carries two members in the first family: only counted without the flag
|
||||
assert_eq!(overlap.get(0, 2), if no_ambiguity { 1 } else { 2 });
|
||||
assert_eq!(overlap.get(0, 1), 1 + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
//! Persisted, per-layer `f32` value per **non-monomorphic minorant family**:
|
||||
//! the Shannon entropy of the family ([`ScalarKind::Entropy`], see
|
||||
//! `algorithms::entropy`) or its pair-balance weight
|
||||
//! ([`ScalarKind::PairWeight`], see `algorithms::pair_weight`). Compact:
|
||||
//! indexed by the dedicated ordinal `algorithms::non_monomorphic_reindex_layer`
|
||||
//! computes (skips monomorphic minorants entirely, ~98% of minorants on real
|
||||
//! data), never by `scan_layer_families`'s own `family_idx` (which counts
|
||||
//! every minorant, monomorphic included).
|
||||
//!
|
||||
//! Exists so repeated reads of the same layer's values (entropy-biased or
|
||||
//! pair-balanced family selection) are a plain positional mmap lookup
|
||||
//! instead of re-paying `scan_layer_families`'s cross-partition resolution —
|
||||
//! building a file costs exactly that same full scan the first time, it does
|
||||
//! not make any single computation of its value faster.
|
||||
//!
|
||||
//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat
|
||||
//! per-entry data, read-only after build), just with `f32` payloads and a
|
||||
//! different (compact) numbering. The two kinds share the format and differ
|
||||
//! by magic and file name only, so one kind can never be read as the other.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
// Header: magic(4) + flags(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
|
||||
// `flags` bit 0 = `no_ambiguity` (the counting convention the values were
|
||||
// computed under, see `FamilyScalarAnnexBuilder::new`); every file written
|
||||
// before the flag existed has `flags == 0`, i.e. `no_ambiguity == false`,
|
||||
// which is what it was computed under.
|
||||
const FLAG_NO_AMBIGUITY: u32 = 1;
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// Sentinel: not yet computed. Every stored value is `>= 0.0`, so any
|
||||
/// negative value is unambiguous — chosen over `NaN` to avoid
|
||||
/// `NaN`-comparison footguns at every read site.
|
||||
const SENTINEL: f32 = -1.0;
|
||||
|
||||
/// What the stored `f32` means.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScalarKind {
|
||||
/// Shannon entropy of the family (bits).
|
||||
Entropy,
|
||||
/// Mean, over the pairs of genomes carrying the family, of 1 / (number of
|
||||
/// families the pair shares) — see `algorithms::pair_weight`.
|
||||
PairWeight,
|
||||
}
|
||||
|
||||
impl ScalarKind {
|
||||
fn magic(self) -> [u8; 4] {
|
||||
match self {
|
||||
ScalarKind::Entropy => *b"PENT",
|
||||
ScalarKind::PairWeight => *b"PPWT",
|
||||
}
|
||||
}
|
||||
|
||||
/// File name of this kind's annex built under the given `--no-ambiguity`
|
||||
/// counting convention. The two conventions never share a file:
|
||||
/// switching between them builds the missing annex without discarding the
|
||||
/// other.
|
||||
pub(crate) fn file_name(self, no_ambiguity: bool) -> &'static str {
|
||||
match (self, no_ambiguity) {
|
||||
(ScalarKind::Entropy, false) => "entropy.pent",
|
||||
(ScalarKind::Entropy, true) => "entropy.noamb.pent",
|
||||
(ScalarKind::PairWeight, false) => "pairweight.ppwt",
|
||||
(ScalarKind::PairWeight, true) => "pairweight.noamb.ppwt",
|
||||
}
|
||||
}
|
||||
|
||||
/// Every file name of this kind, whatever its counting convention — for
|
||||
/// invalidating them all together (e.g. when the sibling annex they were
|
||||
/// numbered against is rebuilt).
|
||||
pub(crate) fn file_names(self) -> [&'static str; 2] {
|
||||
[self.file_name(false), self.file_name(true)]
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FamilyScalarAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
no_ambiguity: bool,
|
||||
}
|
||||
|
||||
impl FamilyScalarAnnex {
|
||||
pub(crate) fn open(path: &Path, kind: ScalarKind) -> io::Result<Self> {
|
||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||
if mmap.len() < HEADER_SIZE {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file too short"));
|
||||
}
|
||||
if mmap[0..4] != kind.magic() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("bad magic: not a {kind:?} annex"),
|
||||
));
|
||||
}
|
||||
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
|
||||
if mmap.len() < HEADER_SIZE + n * 4 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file truncated"));
|
||||
}
|
||||
let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap());
|
||||
Ok(Self { mmap, n, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 })
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
/// The counting convention these values were computed under: `true`
|
||||
/// when genomes carrying several members of a family were left out of
|
||||
/// the count (`--no-ambiguity`), `false` when they were counted as a
|
||||
/// state of their own.
|
||||
pub fn no_ambiguity(&self) -> bool {
|
||||
self.no_ambiguity
|
||||
}
|
||||
|
||||
/// `None` for the sentinel (not yet computed — shouldn't happen against
|
||||
/// a fully-built annex).
|
||||
pub fn get(&self, idx: usize) -> Option<f32> {
|
||||
let off = HEADER_SIZE + idx * 4;
|
||||
let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap());
|
||||
(v >= 0.0).then_some(v)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FamilyScalarAnnexBuilder {
|
||||
mmap: MmapMut,
|
||||
}
|
||||
|
||||
impl FamilyScalarAnnexBuilder {
|
||||
/// Create a new annex of `n` entries (a layer's non-monomorphic
|
||||
/// minorant count) at `path`, pre-initialised to the sentinel.
|
||||
/// `no_ambiguity` records, in the header, the counting convention the
|
||||
/// values will be computed under (see [`FamilyScalarAnnex::no_ambiguity`]).
|
||||
pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, no_ambiguity: bool) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n * 4;
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&kind.magic());
|
||||
let flags = if no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 };
|
||||
mmap[4..8].copy_from_slice(&flags.to_le_bytes());
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
for i in 0..n {
|
||||
let off = HEADER_SIZE + i * 4;
|
||||
mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes());
|
||||
}
|
||||
Ok(Self { mmap })
|
||||
}
|
||||
|
||||
pub(crate) fn set(&mut self, idx: usize, value: f32) {
|
||||
debug_assert!(value >= 0.0, "stored values must be non-negative, got {value}");
|
||||
let off = HEADER_SIZE + idx * 4;
|
||||
self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
pub(crate) fn close(self) -> io::Result<()> {
|
||||
self.mmap.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
const E: ScalarKind = ScalarKind::Entropy;
|
||||
|
||||
#[test]
|
||||
fn sentinel_is_negative_and_unset_entries_read_as_none() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap().close().unwrap();
|
||||
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
|
||||
for i in 0..4 {
|
||||
assert_eq!(annex.get(i), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_including_zero_value() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap();
|
||||
let values = [0.0f32, 1.5, 3.9, 0.0001];
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
builder.set(i, v);
|
||||
}
|
||||
builder.close().unwrap();
|
||||
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
assert_eq!(annex.get(i), Some(v), "entry {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_sentinel_and_real_values() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, false).unwrap();
|
||||
builder.set(1, 2.0);
|
||||
builder.close().unwrap();
|
||||
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
|
||||
assert_eq!(annex.get(0), None);
|
||||
assert_eq!(annex.get(1), Some(2.0));
|
||||
assert_eq!(annex.get(2), None);
|
||||
assert_eq!(annex.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counting_convention_roundtrips_through_the_header() {
|
||||
let dir = tempdir().unwrap();
|
||||
for kind in [ScalarKind::Entropy, ScalarKind::PairWeight] {
|
||||
for no_ambiguity in [false, true] {
|
||||
let path = dir.path().join(kind.file_name(no_ambiguity));
|
||||
FamilyScalarAnnexBuilder::new(2, &path, kind, no_ambiguity).unwrap().close().unwrap();
|
||||
assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().no_ambiguity(), no_ambiguity);
|
||||
}
|
||||
assert_ne!(kind.file_name(false), kind.file_name(true));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_kind_cannot_be_read_as_the_other() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("w.ppwt");
|
||||
FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, false).unwrap().close().unwrap();
|
||||
assert!(FamilyScalarAnnex::open(&path, ScalarKind::Entropy).is_err());
|
||||
assert!(FamilyScalarAnnex::open(&path, ScalarKind::PairWeight).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_with_zero_flags_reads_as_ambiguity_counted() {
|
||||
// files written before the flag existed carried four zero bytes there
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("old.pent");
|
||||
FamilyScalarAnnexBuilder::new(1, &path, E, false).unwrap().close().unwrap();
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
assert_eq!(&bytes[4..8], &[0u8; 4]);
|
||||
assert!(!FamilyScalarAnnex::open(&path, E).unwrap().no_ambiguity());
|
||||
}
|
||||
}
|
||||
@@ -34,3 +34,15 @@ pub(super) fn triangle_index(n: usize, i: usize, j: usize) -> usize {
|
||||
debug_assert!(j < n && i != j);
|
||||
(i * (2 * n - i - 1)) / 2 + (j - i - 1)
|
||||
}
|
||||
|
||||
/// Whether a genome with this per-family `mask` (bit `b` set iff it carries
|
||||
/// the member whose central base is `b`) counts as carrying the family, under
|
||||
/// the `--no-ambiguity` counting convention: any non-empty state does, except
|
||||
/// — with `no_ambiguity` — a genome carrying several members, which is
|
||||
/// missing data in the alignment and so shares nothing there. One definition
|
||||
/// for every annex that counts carriers (family overlap, pair weight,
|
||||
/// quartet), so they can never disagree.
|
||||
#[inline]
|
||||
pub(crate) fn is_carrier(mask: u8, no_ambiguity: bool) -> bool {
|
||||
mask != 0 && !(no_ambiguity && mask.count_ones() > 1)
|
||||
}
|
||||
@@ -20,17 +20,21 @@
|
||||
//! reverse.
|
||||
|
||||
pub mod algorithms;
|
||||
mod entropy_annex;
|
||||
pub mod extensions;
|
||||
mod family_overlap;
|
||||
mod family_scalar_annex;
|
||||
mod helpers;
|
||||
pub mod iter;
|
||||
mod quartet_annex;
|
||||
mod siblingannex;
|
||||
|
||||
pub(crate) use entropy_annex::ENTROPY_ANNEX_FILE_NAME;
|
||||
pub use entropy_annex::{EntropyAnnex, EntropyAnnexBuilder};
|
||||
pub(crate) use family_overlap::{FAMILY_OVERLAP_FILE_NAME, FamilyOverlapAccumulator};
|
||||
pub(crate) use family_overlap::{
|
||||
FAMILY_OVERLAP_FILE_NAMES, FamilyOverlapAccumulator, family_overlap_file_name,
|
||||
};
|
||||
pub(crate) use family_scalar_annex::ScalarKind;
|
||||
pub use family_scalar_annex::{FamilyScalarAnnex, FamilyScalarAnnexBuilder};
|
||||
pub use family_overlap::FamilyOverlap;
|
||||
pub(crate) use quartet_annex::QUARTET_ANNEX_FILE_NAMES;
|
||||
pub use iter::{
|
||||
MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt,
|
||||
};
|
||||
@@ -38,7 +42,8 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
|
||||
pub use algorithms::{
|
||||
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
|
||||
PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment,
|
||||
PairwiseReducer, RawSnpDistanceOutput, SamplingSpec, SankoffBundle, SiblingAnnexStats,
|
||||
SnpAlignment,
|
||||
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
|
||||
pairwise_cost_matrix, sampled_scan,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Persisted, per-layer quartet-informativeness flag: one bit per
|
||||
//! non-monomorphic minorant family, stored as a plain
|
||||
//! [`obicompactvec::PersistentBitVec`] (no format of its own) and numbered
|
||||
//! by the same compact ordinal as [`crate::siblings::EntropyAnnex`]
|
||||
//! (`algorithms::non_monomorphic_reindex_layer`), never by
|
||||
//! `scan_layer_families`'s own `family_idx`.
|
||||
//!
|
||||
//! A family is quartet-informative when its carrier genomes show at least
|
||||
//! two distinct called states, each carried by at least two genomes: the
|
||||
//! pattern that can separate two pairs of genomes from each other
|
||||
//! (`AB|CD`). Anything weaker — a singleton variant, or a family carried by
|
||||
//! fewer than four genomes — fits every topology and tells a tree nothing.
|
||||
//! See `algorithms::masking::is_quartet_informative` for the exact rule.
|
||||
//!
|
||||
//! Which genomes count as carriers depends on `--no-ambiguity`, so each
|
||||
//! convention has its own file, and **the file name is the parameter**: a
|
||||
//! `PersistentBitVec` header has no room for a flag, and changing it would
|
||||
//! change a shared structure for the sake of one annex.
|
||||
//!
|
||||
//! The flag is computed over every genome of the index. A later
|
||||
//! `--exclude-genome` can only remove carriers, never create a bipartition,
|
||||
//! so a family flagged `false` here is also uninformative once genomes are
|
||||
//! excluded: the annex is a safe pre-filter, and the exact check
|
||||
//! (`algorithms::masking::survives_quartet`) still runs on the resolved
|
||||
//! family.
|
||||
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAME: &str = "quartet.pbiv";
|
||||
|
||||
/// Same annex, built with `no_ambiguity == true`.
|
||||
pub(crate) const QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME: &str = "quartet.noamb.pbiv";
|
||||
|
||||
/// Every quartet-annex file name, whatever its counting convention — for
|
||||
/// invalidating them all together when the sibling annex they were numbered
|
||||
/// against is rebuilt.
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 2] =
|
||||
[QUARTET_ANNEX_FILE_NAME, QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME];
|
||||
|
||||
/// File name of the quartet annex built under the given `--no-ambiguity`
|
||||
/// convention.
|
||||
pub(crate) fn quartet_annex_file_name(no_ambiguity: bool) -> &'static str {
|
||||
if no_ambiguity {
|
||||
QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME
|
||||
} else {
|
||||
QUARTET_ANNEX_FILE_NAME
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_two_conventions_never_share_a_file() {
|
||||
assert_ne!(quartet_annex_file_name(false), quartet_annex_file_name(true));
|
||||
for no_ambiguity in [false, true] {
|
||||
assert!(QUARTET_ANNEX_FILE_NAMES.contains(&quartet_annex_file_name(no_ambiguity)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user