Implement advanced sampling and family annex logic
Release / create-release (push) Successful in 2m35s
Release / build-linux-x86_64 (push) Successful in 8m19s
Release / build-macos-arm64 (push) Successful in 2m3s
ci.yml / build (pull_request) Successful in 4m8s

Introduces complex sampling mechanisms including quartet filtering, pair balance weighting, and entropy bias. Enforces the `--no-ambiguity` convention for family counting and updates annex storage for scalar data.

This includes new logic for calculating family weights based on overlap, quota-based layer determination, and filtering families based on quartet informativeness. Updates also include argument parsing for `--n-sites` and improved handling of family overlap files.
This commit is contained in:
Eric Coissac committed 2026-10-13 08:57:18 +02:00
1 parent 0d12494f1a
commit f29dcbcfb8
25 files changed
+1471 -486

No files matched your search

+1 -1
View File
@@ -1532,7 +1532,7 @@ dependencies = [
[[package]]
name = "obikmer"
version = "1.3.5"
version = "1.3.6"
dependencies = [
"clap",
"csv",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "obikmer"
version = "1.3.5"
version = "1.3.6"
edition = "2024"
[[bin]]
+61 -14
View File
@@ -32,6 +32,36 @@ fn parse_gamma_shape(s: &str) -> Result<GammaShapeArg, String> {
.map_err(|_| format!("invalid --gamma-shape value {s:?}: expected a positive number or \"auto\"/\"estimate\""))
}
/// `--n-sites` CLI value: a number of sites to draw, or `all` for every site
/// — the draw then keeps every variable family that passes the other filters,
/// with no sampling involved. Its own tiny type (like [`GammaShapeArg`]) so
/// the two cases stay distinct up to the call that needs the target, instead
/// of an in-band sentinel (a negative number, `0`, ...) that a typo could hit.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NSitesArg {
All,
N(usize),
}
impl NSitesArg {
/// The target to hand to the sampler: `None` = exhaustive.
pub fn target(self) -> Option<usize> {
match self {
NSitesArg::All => None,
NSitesArg::N(n) => Some(n),
}
}
}
fn parse_n_sites(s: &str) -> Result<NSitesArg, String> {
if s.eq_ignore_ascii_case("all") {
return Ok(NSitesArg::All);
}
s.parse::<usize>()
.map(NSitesArg::N)
.map_err(|_| format!("invalid --n-sites value {s:?}: expected a non-negative integer or \"all\""))
}
/// `--distance` value — either one of `obikphylo::DistanceMetric`'s
/// whole-index metrics (routed to `IndexCache::distance`) or one of
/// `obikphylo::SnpDistanceKind`'s `snp-*` corrections (routed to
@@ -108,7 +138,7 @@ impl DistanceArg {
/// (classic metrics + `snp-*` corrections/NJ/UPGMA), annex construction
/// (`--sibling-annex`), annex diagnostics (`--sibling-stats`,
/// `--sibling-hist`), entropy reporting (`--shannon`), SNP pseudo-alignment
/// sampling (`--pseudo-alignment`, `--subsample`, `--free-loss`,
/// sampling (`--pseudo-alignment`, `--n-sites`, `--free-loss`,
/// `--no-ambiguity`, `--entropy`/`--entropy-sd`), Sankoff cost-matrix
/// calibration (`--sankoff`, `--sankoff-ratio-ceiling`) and its TNT/PhyG/
/// IQ-TREE exports (`--tnt`, `--phyg`, `--iqtree`/`--iqtree-min-freq`,
@@ -180,26 +210,25 @@ pub struct PhyloArgs {
/// Write a per-family Shannon entropy report (CSV) — requires an
/// already-built sibling annex (`--sibling-annex` first, in this
/// invocation or an earlier one). Always a full, unsampled scan of
/// every family (`--subsample`/`--entropy`/`--entropy-sd` below only
/// every family (`--n-sites`/`--entropy`/`--entropy-sd` below only
/// apply to `--pseudo-alignment`, not this).
#[arg(long)]
pub shannon: bool,
/// Write a SNP-only pseudo-alignment (FASTA) — requires
/// `--subsample <N>` and an already-built sibling annex
/// `--n-sites <N|all>` and an already-built sibling annex
/// (`--sibling-annex` first, in this invocation or an earlier one).
#[arg(long)]
pub pseudo_alignment: bool,
/// Target number of variable sites to sample index-wide for
/// `--pseudo-alignment`/`--sankoff` (mandatory for both) and for a
/// `snp-*` `--distance` value (optional there: omitted means exhaustive
/// — every non-monomorphic minorant of the whole index, not an
/// approximation) — a target, not a guarantee when given (proportional
/// per-layer sampling; see `obikphylo::siblings::sampled_scan`'s own
/// docs).
#[arg(long)]
pub subsample: Option<usize>,
/// Number of variable sites to draw index-wide, or `all` for every site
/// that passes the other filters (no sampling then). Mandatory for
/// `--pseudo-alignment`/`--sankoff`; optional for a `snp-*`
/// `--distance` value, where omitting it also means every site. A
/// number is a target, not a guarantee (proportional per-layer
/// sampling; see `obikphylo::siblings::sampled_scan`'s own docs).
#[arg(long = "n-sites", value_name = "N|all", value_parser = parse_n_sites)]
pub n_sites: Option<NSitesArg>,
/// In `--pseudo-alignment`, treat a genome carrying none of a family's
/// observed members (`∅`) as missing data (`?`) rather than a real
@@ -213,6 +242,24 @@ pub struct PhyloArgs {
#[arg(long)]
pub no_ambiguity: bool,
/// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, keep only
/// the families that can separate two pairs of genomes from each other
/// (a bipartition): at least two distinct called states, each carried
/// by at least two genomes. A singleton variant, or a family carried by
/// fewer than four genomes, fits every topology and tells a tree
/// nothing. Which genomes count as carriers follows `--no-ambiguity`.
#[arg(long)]
pub quartet: bool,
/// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, favour the
/// families that cover the pairs of genomes sharing the fewest families
/// across the index: each family is weighted by the mean, over the pairs
/// of genomes carrying it, of 1 / (number of families the pair shares).
/// Evens out how well each pair of genomes is represented in the sample.
/// Which genomes count as carriers follows `--no-ambiguity`.
#[arg(long)]
pub pair_balance: bool,
/// Entropy-biased sampling target (Gaussian kernel mean) for
/// `--pseudo-alignment` — activates biasing as soon as this or
/// `--entropy-sd` is given; the other defaults to 1.0/0.5.
@@ -227,7 +274,7 @@ pub struct PhyloArgs {
/// Persist the `snp-*` `--distance` sample (the pairwise substitution
/// tally) in `DIR` across separate `obikmer phylo` invocations: a later
/// run against the same index with the exact same
/// `--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/
/// `--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/
/// `--min-shared-family`/`--entropy`/`--entropy-sd` restores it instead
/// of resampling. `DIR` is created if it doesn't exist. If it already
/// holds a sample built under *different* selection parameters, this
@@ -247,7 +294,7 @@ pub struct PhyloArgs {
/// Calibrate a 16-state Sankoff cost matrix (and its matching
/// pseudo-alignment) from an already-built sibling annex — requires
/// `--subsample <N>`, and shares `--free-loss`/`--no-ambiguity`/
/// `--n-sites <N|all>`, and shares `--free-loss`/`--no-ambiguity`/`--quartet`/
/// `--entropy`/`--entropy-sd` with `--pseudo-alignment` (one draw, same
/// selection feeds both the alignment and every calibration tally).
#[arg(long)]
+20 -10
View File
@@ -11,13 +11,14 @@ use std::sync::Arc;
use obikidxcache::index_cache::IndexCache;
use obikindex::KmerIndex;
use obikphylo::siblings::{
EntropyBias, GammaShape, SiblingExt, cardinality_transition_probs,
EntropyBias, GammaShape, SamplingSpec, SiblingExt, cardinality_transition_probs,
composition_transition_probs, pairwise_cost_matrix,
};
use obikphylo::{Metrics, neighbor_joining, upgma};
use obisys::{Reporter, Stage};
use tracing::info;
use args::NSitesArg;
use iqtree::write_iqtree;
use phyg::write_sankoff_phyg;
use phylip::write_phylip_relaxed;
@@ -154,7 +155,7 @@ pub fn run(args: PhyloArgs) {
// ── Family Overlap matrix (`--family-overlap`) ──────────────────────────────
if args.family_overlap {
let t = Stage::start("family_overlap");
let overlap = cache.family_overlap().unwrap_or_else(|e| {
let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| {
eprintln!("error computing family overlap: {e}");
std::process::exit(1);
});
@@ -201,7 +202,7 @@ pub fn run(args: PhyloArgs) {
// `args::PhyloArgs::min_shared_family`'s own docs on why).
let snp_exclude_mask: Vec<bool> = match args.min_shared_family {
Some(threshold) => {
let overlap = cache.family_overlap().unwrap_or_else(|e| {
let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| {
eprintln!("error computing family overlap: {e}");
std::process::exit(1);
});
@@ -235,6 +236,13 @@ pub fn run(args: PhyloArgs) {
} else {
None
};
let sampling = SamplingSpec {
free_loss: args.free_loss,
no_ambiguity: args.no_ambiguity,
quartet: args.quartet,
pair_balance: args.pair_balance,
entropy_bias,
};
// ── `--session` ──────────────────────────────────────────────────────────
// Temporarily unavailable: the per-driver checkpoint/resume plumbing
@@ -253,7 +261,7 @@ pub fn run(args: PhyloArgs) {
// ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ──
// All three consume exactly the same selection parameters
// (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/
// (`--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/
// `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from
// the *same* random sample when combined in one invocation — so
// whichever of them are requested here share exactly one
@@ -265,13 +273,15 @@ pub fn run(args: PhyloArgs) {
let snp_kind = args.distance.as_snp();
let need_alignment = args.pseudo_alignment || sankoff_requested;
let need_pairwise = sankoff_requested || snp_kind.is_some();
// `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional)
let target = args.n_sites.and_then(NSitesArg::target);
if args.pseudo_alignment && args.subsample.is_none() {
eprintln!("error: --pseudo-alignment requires --subsample <N>");
if args.pseudo_alignment && args.n_sites.is_none() {
eprintln!("error: --pseudo-alignment requires --n-sites <N|all>");
std::process::exit(1);
}
if sankoff_requested && args.subsample.is_none() {
eprintln!("error: --sankoff requires --subsample <N>");
if sankoff_requested && args.n_sites.is_none() {
eprintln!("error: --sankoff requires --n-sites <N|all>");
std::process::exit(1);
}
@@ -290,13 +300,13 @@ pub fn run(args: PhyloArgs) {
info!(
"sampling SNP sites{}",
match args.subsample {
match target {
Some(n) => format!(" (target {n} site(s))"),
None => " (exhaustive)".into(),
}
);
let t = Stage::start("sampled_scan");
obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers)
obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers)
.unwrap_or_else(|e| {
eprintln!("error sampling SNP sites: {e}");
std::process::exit(1);
+36 -112
View File
@@ -8,19 +8,6 @@
//! it is actually observed) is the consistent informativeness proxy for
//! this project, not a 4-symbol reduction.
use std::collections::HashSet;
use std::path::Path;
use obikidxcache::index_cache::IndexCache;
use obikindex::{OKIError, OKIResult};
use super::family_scan::{Selection, scan_layer_families};
use super::minorant_selection::{is_non_monomorphic_minorant, non_monomorphic_reindex_layer};
use crate::siblings::{
ANNEX_FILE_NAME, ENTROPY_ANNEX_FILE_NAME, EntropyAnnex, EntropyAnnexBuilder,
FamilyOverlapAccumulator, SiblingAnnex,
};
/// Shannon entropy (bits) of one family's per-genome states, over the 15
/// non-empty subsets of `{A,C,G,T}` actually observed among the genomes
/// carrying it (`genome_mask[g] != 0`) — genomes where the family is absent
@@ -103,6 +90,26 @@ mod tests {
assert_eq!(n, 2);
}
#[test]
fn entropy_15_over_single_member_genomes_equals_entropy_4() {
// `--no-ambiguity` leaves only the single-base states, over which the
// 15-state entropy and the 4-base one are the same quantity
let masks = [0b0001u8, 0b0001, 0b0100, 0b1000, 0b0100, 0b0100, 0b0010];
let (h15, n15) = family_entropy(&masks).unwrap();
let (h4, n4) = family_entropy_4(&masks).unwrap();
assert!((h15 - h4).abs() < 1e-12);
assert_eq!(n15, n4);
}
#[test]
fn entropy_under_selects_the_counting_convention() {
let masks = [0b0001u8, 0b0001, 0b0011]; // the last genome carries two members
assert_eq!(family_entropy_under(&masks, true).unwrap().1, 2);
assert_eq!(family_entropy_under(&masks, false).unwrap().1, 3);
assert_eq!(family_entropy_under(&masks, true).unwrap().0, 0.0);
assert!(family_entropy_under(&masks, false).unwrap().0 > 0.0);
}
#[test]
fn entropy_4_is_none_without_unambiguous_genome() {
assert!(family_entropy_4(&[0, 0b0011, 0b0110]).is_none());
@@ -110,104 +117,21 @@ mod tests {
}
}
/// Builds one layer's entropy annex if it doesn't already exist, and/or
/// feeds `overlap_acc` (`Some` iff the index-wide family-overlap annex is
/// itself missing — see
/// [`crate::siblings::FamilyOverlap`]'s own module docs) — a single shared
/// `scan_layer_families` pass covers both, since both need the exact same
/// per-family `genome_mask` resolution and there is no reason to pay for it
/// twice just because only one of the two happens to be missing. A no-op
/// (no scan at all) when entropy already exists for this layer *and*
/// `overlap_acc` is `None`. Bounded to non-monomorphic minorants only
/// ([`non_monomorphic_reindex_layer`], a cheap annex-only pass — no
/// cross-partition resolution) whenever a scan does happen: monomorphism is
/// already known from the annex bits alone, so there is no reason to pay
/// `scan_layer_families`'s expensive per-genome resolution for the ~98% of
/// minorants that are monomorphic only to discard the result — true for
/// family-overlap's own "variable families only" scope just as much as for
/// entropy's.
pub(crate) fn ensure_layer_entropy_annex(
cache: &IndexCache,
partition: usize,
layer_idx: usize,
n_genomes: usize,
fast_mode: bool,
mut overlap_acc: Option<&mut FamilyOverlapAccumulator>,
) -> OKIResult<()> {
let layer = cache
.get_layer(partition, layer_idx)
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
let path = layer.dir().join(ENTROPY_ANNEX_FILE_NAME);
let need_entropy = !path.exists();
if !need_entropy && overlap_acc.is_none() {
return Ok(());
}
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
let eligible: HashSet<usize> = reindex.keys().copied().collect();
let mut builder = need_entropy
.then(|| EntropyAnnexBuilder::new(reindex.len(), &path))
.transpose()
.map_err(OKIError::Io)?;
scan_layer_families(
cache,
partition,
layer_idx,
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, mask, genome_mask| {
debug_assert!(
mask.family_size() >= 2,
"Selection::Some(eligible) must only visit non-monomorphic minorants"
);
if let Some(builder) = builder.as_mut() {
if let Some((h15, _)) = family_entropy(genome_mask) {
builder.set(reindex[&family_idx], h15 as f32);
}
}
if let Some(acc) = overlap_acc.as_deref_mut() {
acc.add_family(genome_mask);
}
},
)?;
if let Some(builder) = builder {
builder.close().map_err(OKIError::Io)?;
}
Ok(())
}
/// Streams one entropy value per minorant of the layer, **in `family_idx`
/// order** — monomorphic minorants included, valued `0.0` by convention
/// (trivially uninformative, never computed or stored in the compact
/// entropy annex). Exists so a caller can `.zip()` this directly against
/// any other `family_idx`-ordered stream (e.g.
/// `SiblingLayerExt::iter_minorants`) instead of separately consulting
/// [`non_monomorphic_reindex_layer`]'s map at each site.
/// Shannon entropy (bits) of one family under the counting convention
/// `--no-ambiguity` selects:
/// - `false`: [`family_entropy`] — every genome carrying the family counts,
/// a genome with several members being a state of its own;
/// - `true`: [`family_entropy_4`] — only the genomes carrying exactly one
/// member count, the others being missing data, exactly as the alignment
/// itself will see them. Over those genomes the 15-state entropy can only
/// ever take the four single-base states, so the two definitions coincide.
///
/// Requires this layer's entropy annex to already exist
/// ([`ensure_layer_entropy_annex`] first) — reads it positionally
/// (`compact` advances only on non-monomorphic minorants, mirroring
/// exactly how the annex was written), never recomputes anything.
pub(crate) fn iter_full_entropy(layer_dir: &Path) -> OKIResult<impl Iterator<Item = f32> + use<>> {
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
let entropy_annex =
EntropyAnnex::open(&layer_dir.join(ENTROPY_ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
let mut compact = 0usize;
Ok((0..annex.len()).filter_map(move |slot| {
let mask = annex.get(slot)?;
if !mask.is_minorant() {
return None; // not a minorant at all — no entry in family_idx numbering either
}
if is_non_monomorphic_minorant(mask) {
let value = entropy_annex.get(compact).unwrap_or(0.0);
compact += 1;
Some(value)
} else {
Some(0.0) // monomorphic — trivially zero by convention, never stored
}
}))
/// Second value: the number of genomes the entropy was computed over.
pub(crate) fn family_entropy_under(genome_mask: &[u8], no_ambiguity: bool) -> Option<(f64, usize)> {
if no_ambiguity {
family_entropy_4(genome_mask)
} else {
family_entropy(genome_mask)
}
}
@@ -0,0 +1,212 @@
//! Per-layer family annexes built from a `scan_layer_families` sweep: the
//! entropy annex and the pair-weight annex (both a
//! [`crate::siblings::FamilyScalarAnnex`]), the quartet annex (a
//! [`obicompactvec::PersistentBitVec`], see `crate::siblings::quartet_annex`)
//! and the index-level family-overlap accumulators. The entropy, quartet and
//! overlap builders all need the identical per-family `genome_mask`
//! resolution, so one sweep feeds every one that has something missing; the
//! pair weights need the *finished* overlap matrix, hence a second sweep
//! (`ensure_layer_pair_weight_annexes`).
use std::collections::HashSet;
use std::io;
use std::path::PathBuf;
use obicompactvec::PersistentBitVecBuilder;
use obikidxcache::index_cache::IndexCache;
use obikindex::{OKIError, OKIResult};
use super::entropy::family_entropy_under;
use super::family_scan::{Selection, scan_layer_families};
use super::masking::is_quartet_informative;
use super::minorant_selection::non_monomorphic_reindex_layer;
use super::pair_weight::pair_balance_weight;
use crate::siblings::quartet_annex::quartet_annex_file_name;
use crate::siblings::{FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind};
/// Which annexes a caller needs, each listed once per `--no-ambiguity`
/// counting convention (`false` = genomes carrying several members count as a
/// state of their own, `true` = they are left out) — an empty list asks for
/// none. A pair weight is computed from the family-overlap annex of its own
/// convention, so asking for one asks for the other.
#[derive(Clone, Debug, Default)]
pub(crate) struct AnnexRequest {
pub entropy: Vec<bool>,
pub quartet: Vec<bool>,
pub overlap: Vec<bool>,
pub pair_weight: Vec<bool>,
}
impl AnnexRequest {
/// Everything under both conventions — what `--sibling-annex` precomputes.
pub(crate) fn everything() -> Self {
let both = vec![false, true];
Self { entropy: both.clone(), quartet: both.clone(), overlap: both.clone(), pair_weight: both }
}
/// The overlap conventions to ensure: the requested ones plus every one a
/// requested pair weight is computed from, each once.
pub(crate) fn overlap_conventions(&self) -> Vec<bool> {
dedup(self.overlap.iter().chain(&self.pair_weight).copied())
}
/// The pair-weight conventions requested, each once.
pub(crate) fn pair_weight_conventions(&self) -> Vec<bool> {
dedup(self.pair_weight.iter().copied())
}
}
fn dedup(variants: impl IntoIterator<Item = bool>) -> Vec<bool> {
let mut v: Vec<bool> = variants.into_iter().collect();
v.sort_unstable();
v.dedup();
v
}
/// Builds the entropy and quartet annexes of one layer that `request` asks
/// for and that don't exist yet, and feeds `overlap_accs` (one accumulator
/// per family-overlap annex still missing, see
/// [`crate::siblings::FamilyOverlap`]'s own module docs), in one
/// `scan_layer_families` pass. A no-op (no scan at all) when everything
/// requested already exists for this layer *and* `overlap_accs` is empty.
/// Bounded to non-monomorphic minorants only
/// ([`non_monomorphic_reindex_layer`], a cheap annex-only pass — no
/// cross-partition resolution) whenever a scan does happen: monomorphism is
/// already known from the annex bits alone, so there is no reason to pay
/// `scan_layer_families`'s expensive per-genome resolution for the ~98% of
/// minorants that are monomorphic only to discard the result.
pub(crate) fn ensure_layer_family_annexes(
cache: &IndexCache,
partition: usize,
layer_idx: usize,
n_genomes: usize,
fast_mode: bool,
request: &AnnexRequest,
overlap_accs: &mut [(bool, FamilyOverlapAccumulator)],
) -> OKIResult<()> {
let layer = cache
.get_layer(partition, layer_idx)
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
let missing = |name: &dyn Fn(bool) -> &'static str, variants: &[bool]| -> Vec<(bool, PathBuf)> {
dedup(variants.iter().copied())
.into_iter()
.map(|no_ambiguity| (no_ambiguity, layer.dir().join(name(no_ambiguity))))
.filter(|(_, path)| !path.exists())
.collect()
};
let missing_entropy = missing(&|nb| ScalarKind::Entropy.file_name(nb), &request.entropy);
let missing_quartet = missing(&quartet_annex_file_name, &request.quartet);
if missing_entropy.is_empty() && missing_quartet.is_empty() && overlap_accs.is_empty() {
return Ok(());
}
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
let eligible: HashSet<usize> = reindex.keys().copied().collect();
let mut entropy_builders = missing_entropy
.iter()
.map(|(no_ambiguity, path)| {
FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *no_ambiguity)
.map(|b| (*no_ambiguity, b))
})
.collect::<io::Result<Vec<_>>>()
.map_err(OKIError::Io)?;
let mut quartet_builders = missing_quartet
.iter()
.map(|(no_ambiguity, path)| {
PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*no_ambiguity, b))
})
.collect::<io::Result<Vec<_>>>()
.map_err(OKIError::Io)?;
scan_layer_families(
cache,
partition,
layer_idx,
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, mask, genome_mask| {
debug_assert!(
mask.family_size() >= 2,
"Selection::Some(eligible) must only visit non-monomorphic minorants"
);
let compact = reindex[&family_idx];
for (no_ambiguity, builder) in entropy_builders.iter_mut() {
if let Some((h, _)) = family_entropy_under(genome_mask, *no_ambiguity) {
builder.set(compact, h as f32);
}
}
for (no_ambiguity, builder) in quartet_builders.iter_mut() {
builder.set(compact, is_quartet_informative(genome_mask, *no_ambiguity));
}
for (_, acc) in overlap_accs.iter_mut() {
acc.add_family(genome_mask);
}
},
)?;
for (_, builder) in entropy_builders {
builder.close().map_err(OKIError::Io)?;
}
for (_, builder) in quartet_builders {
builder.close().map_err(OKIError::Io)?;
}
Ok(())
}
/// Builds the pair-weight annexes of one layer that don't exist yet, one per
/// `(no_ambiguity, inverse overlap matrix)` in `matrices` (see
/// [`super::pair_weight::inverse_overlap_matrix`]), in one scan. A no-op (no
/// scan) when they all exist. Needs the family-overlap annex of each
/// convention to be complete: run after the sweep that builds it.
pub(crate) fn ensure_layer_pair_weight_annexes(
cache: &IndexCache,
partition: usize,
layer_idx: usize,
n_genomes: usize,
fast_mode: bool,
matrices: &[(bool, Vec<f32>)],
) -> OKIResult<()> {
let layer = cache
.get_layer(partition, layer_idx)
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
let pending: Vec<&(bool, Vec<f32>)> = matrices
.iter()
.filter(|(no_ambiguity, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)).exists())
.collect();
if pending.is_empty() {
return Ok(());
}
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
let eligible: HashSet<usize> = reindex.keys().copied().collect();
let mut builders = pending
.iter()
.map(|(no_ambiguity, inverse)| {
let path = layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity));
FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *no_ambiguity)
.map(|b| (*no_ambiguity, inverse, b))
})
.collect::<io::Result<Vec<_>>>()
.map_err(OKIError::Io)?;
scan_layer_families(
cache,
partition,
layer_idx,
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, _mask, genome_mask| {
let compact = reindex[&family_idx];
for (no_ambiguity, inverse, builder) in builders.iter_mut() {
builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse));
}
},
)?;
for (_, _, builder) in builders {
builder.close().map_err(OKIError::Io)?;
}
Ok(())
}
@@ -17,6 +17,8 @@
//! waste sample quota on sites that die anyway once the excluded genome's
//! data is dropped.
use crate::siblings::helpers::is_carrier;
/// This genome's masked character state, or `None` for "?" (missing). `m`
/// is one `genome_mask` entry (bit `b` set iff this genome carries the
/// family member whose own canonical central base is `b`; `0` = no member
@@ -64,6 +66,97 @@ pub(crate) fn survives_masking(masked: &[Option<u8>], excluded: &[bool]) -> bool
seen.count_ones() >= 2
}
/// Whether `counts` — the number of genomes per *called* state, indexed by
/// the `0..=15` state value, index `0` (absence) never counted — shows a
/// bipartition: at least two distinct states each carried by at least two
/// genomes. That is the pattern that can separate two pairs of genomes from
/// each other (`AB|CD`); a singleton variant, or a family carried by fewer
/// than four genomes, fits every topology and informs none.
fn has_bipartition(counts: &[u32; 16]) -> bool {
counts[1..].iter().filter(|&&c| c >= 2).count() >= 2
}
/// Quartet-informativeness of a family from its raw per-genome masks alone,
/// under the `--no-ambiguity` counting convention: absent genomes (`0`)
/// never count, and with `no_ambiguity` a genome carrying several members
/// doesn't either — otherwise its combination of bases is a state of its
/// own. Ignores `--exclude-genome` and `--free-loss` on purpose: absence is
/// never a called state here, and removing genomes can only destroy a
/// bipartition, never create one — so `false` is final, and `true` is only
/// a necessary condition, re-checked exactly by [`survives_quartet`] once
/// the family is resolved. This is what the persisted quartet annex stores.
pub(crate) fn is_quartet_informative(genome_mask: &[u8], no_ambiguity: bool) -> bool {
let mut counts = [0u32; 16];
for &m in genome_mask {
if !is_carrier(m, no_ambiguity) {
continue;
}
counts[(m & 0b1111) as usize] += 1;
}
has_bipartition(&counts)
}
/// Exact quartet check on an already-[`masked_state`]d family, over the
/// *included* genomes (`excluded` as in [`survives_masking`]). Only called
/// states count: a genome whose masked state is `0` (absence kept as a real
/// state, i.e. no `--free-loss`) is not a carrier of anything.
pub(crate) fn survives_quartet(masked: &[Option<u8>], excluded: &[bool]) -> bool {
let mut counts = [0u32; 16];
for (g, &state) in masked.iter().enumerate() {
if excluded.get(g).copied().unwrap_or(false) {
continue;
}
if let Some(state) = state {
counts[(state & 0b1111) as usize] += 1;
}
}
has_bipartition(&counts)
}
#[cfg(test)]
mod quartet_tests {
use super::*;
#[test]
fn two_pairs_of_distinct_states_is_informative() {
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], false));
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], true));
}
#[test]
fn singleton_variant_is_not_informative() {
// three genomes in one state plus one lone variant: fits any topology
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], false));
// fewer than four carriers can never be informative
assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], false));
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], false));
}
#[test]
fn absent_genomes_are_not_a_state() {
// two carriers of one base, two absences: absence must not form the 2nd pair
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], false));
}
#[test]
fn no_ambiguity_drops_multi_member_genomes_from_the_count() {
// the two-member genomes form a pair of their own state without the flag
let masks = [0b0001u8, 0b0001, 0b0011, 0b0011];
assert!(is_quartet_informative(&masks, false));
assert!(!is_quartet_informative(&masks, true));
}
#[test]
fn exact_check_honours_exclusions_and_missing() {
let masked = [Some(1u8), Some(1), Some(4), Some(4), None, Some(0)];
assert!(survives_quartet(&masked, &[]));
// excluding one genome of the second pair breaks the bipartition
assert!(!survives_quartet(&masked, &[false, false, true]));
// an absence kept as state 0 (no --free-loss) is not a called state
assert!(!survives_quartet(&[Some(1), Some(1), Some(0), Some(0)], &[]));
}
}
/// IUPAC ambiguity code for a [`masked_state`] value (`0..=15`): single bit
/// -> the plain base; 2 or 3 bits -> the matching IUPAC ambiguity code
/// (preserves partial information instead of collapsing to `N`, the same
@@ -11,7 +11,7 @@ use std::path::Path;
use obicompactvec::TempBitVecBuilder;
use obikindex::{OKIError, OKIResult};
use crate::siblings::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex};
use crate::siblings::{ANNEX_FILE_NAME, FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex};
pub(crate) fn is_non_monomorphic_minorant(mask: FamilyMask) -> bool {
mask.is_minorant() && mask.siblings() >= 1
@@ -82,3 +82,49 @@ pub(crate) fn eligible_bitset_layer(layer_dir: &Path) -> OKIResult<TempBitVecBui
}
Ok(eligible)
}
/// Streams one stored `f32` per minorant of the layer, **in `family_idx`
/// order** — monomorphic minorants included, valued `0.0` by convention
/// (trivially uninformative, never computed or stored in the compact
/// annex). Exists so a caller can `.zip()` this directly against any other
/// `family_idx`-ordered stream (e.g. `SiblingLayerExt::iter_minorants`)
/// instead of separately consulting [`non_monomorphic_reindex_layer`]'s map
/// at each site.
///
/// Reads the `kind` annex (entropy, pair weight) built under the
/// `no_ambiguity` counting convention; it must already exist
/// (`ensure_layer_family_annexes` / `ensure_layer_pair_weight_annexes`
/// first) — read positionally (`compact` advances only on non-monomorphic
/// minorants, mirroring exactly how the annex was written), never
/// recomputed. A file whose header records the other convention is rejected
/// rather than silently read as this one.
pub(crate) fn iter_full_scalar(
layer_dir: &Path,
kind: ScalarKind,
no_ambiguity: bool,
) -> OKIResult<impl Iterator<Item = f32> + use<>> {
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
let scalars =
FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(no_ambiguity)), kind).map_err(OKIError::Io)?;
if scalars.no_ambiguity() != no_ambiguity {
return Err(OKIError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("{kind:?} annex was built under the other --no-ambiguity convention"),
)));
}
let mut compact = 0usize;
Ok((0..annex.len()).filter_map(move |slot| {
let mask = annex.get(slot)?;
if !mask.is_minorant() {
return None; // not a minorant at all — no entry in family_idx numbering either
}
if is_non_monomorphic_minorant(mask) {
let value = scalars.get(compact).unwrap_or(0.0);
compact += 1;
Some(value)
} else {
Some(0.0) // monomorphic — trivially zero by convention, never stored
}
}))
}
+10 -2
View File
@@ -17,10 +17,13 @@ mod alignment;
mod annex;
mod cardcomp;
mod entropy;
mod family_annexes;
mod family_scan;
mod masking;
mod minorant_selection;
mod pair_weight;
mod pairwise;
mod quartet;
mod sankoff;
mod scan;
mod snp_distance;
@@ -38,10 +41,15 @@ pub use sankoff::SankoffBundle;
pub use scan::{LayerReducer, sampled_scan};
pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix};
pub use stats::SiblingAnnexStats;
pub use subsample::{EntropyBias, SurvivingFamily};
pub use subsample::{EntropyBias, SamplingSpec, SurvivingFamily};
pub(crate) use annex::build_layer_sibling_annex;
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4};
pub(crate) use entropy::{family_entropy, family_entropy_4};
pub(crate) use family_annexes::{
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
};
pub(crate) use pair_weight::inverse_overlap_matrix;
pub(crate) use quartet::restrict_to_quartet;
pub(crate) use family_scan::{Selection, scan_layer_families};
pub(crate) use minorant_selection::non_monomorphic_reindex_layer;
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
@@ -0,0 +1,121 @@
//! `--pair-balance`: a per-family sampling weight that favours the families
//! covering the pairs of genomes the index shares the fewest families between.
//!
//! The weight of a family is the **mean**, over the pairs of genomes carrying
//! it, of `1 / O(pair)`, where `O` is the index-wide number of variable
//! families the pair shares (the family-overlap annex, under the same
//! `--no-ambiguity` counting convention). It is the mean, not the sum: a sum
//! grows with the square of the number of carriers, so it would chase the
//! families carried by most genomes — in a skimming dataset typically
//! high-copy regions, sampled deeply for reasons unrelated to phylogeny —
//! instead of the ones covering rare pairs.
//!
//! Values are only meaningful relative to each other; the sampler rescales
//! them per layer (`SamplingSpec::pair_balance`).
use crate::siblings::FamilyOverlap;
use crate::siblings::helpers::is_carrier;
/// `1 / O` for every pair of genomes, as a dense symmetric `n × n` matrix
/// (diagonal and pairs sharing nothing hold `0.0`) — what
/// [`pair_balance_weight`] reads, built once per run from the persisted
/// family-overlap annex.
pub(crate) fn inverse_overlap_matrix(overlap: &FamilyOverlap) -> Vec<f32> {
let n = overlap.n_genomes();
let mut inv = vec![0.0f32; n * n];
for i in 0..n {
for j in (i + 1)..n {
let shared = overlap.get(i, j);
if shared > 0 {
let r = 1.0 / shared as f32;
inv[i * n + j] = r;
inv[j * n + i] = r;
}
}
}
inv
}
/// The pair-balance weight of one family from its raw per-genome masks:
/// mean of `inv_overlap` over the pairs of carriers (carriers per
/// [`is_carrier`], under `no_ambiguity`). `0.0` for a family with fewer than
/// two carriers — it covers no pair, so it can balance none.
pub(crate) fn pair_balance_weight(
genome_mask: &[u8],
no_ambiguity: bool,
inv_overlap: &[f32],
) -> f32 {
let n = genome_mask.len();
let carriers: Vec<usize> =
(0..n).filter(|&g| is_carrier(genome_mask[g], no_ambiguity)).collect();
let k = carriers.len();
if k < 2 {
return 0.0;
}
let mut sum = 0.0f64;
for (a, &i) in carriers.iter().enumerate() {
for &j in &carriers[a + 1..] {
sum += f64::from(inv_overlap[i * n + j]);
}
}
(sum / (k * (k - 1) / 2) as f64) as f32
}
#[cfg(test)]
mod tests {
use super::*;
/// 4 genomes; pair (0,1) shares 2 families, (0,2) shares 4, the others 1.
fn matrix() -> Vec<f32> {
let n = 4;
let mut m = vec![0.0f32; n * n];
let mut set = |i: usize, j: usize, shared: f32| {
m[i * n + j] = 1.0 / shared;
m[j * n + i] = 1.0 / shared;
};
set(0, 1, 2.0);
set(0, 2, 4.0);
for (i, j) in [(0, 3), (1, 2), (1, 3), (2, 3)] {
set(i, j, 1.0);
}
m
}
#[test]
fn single_pair_weight_is_the_inverse_overlap() {
let w = pair_balance_weight(&[1, 1, 0, 0], false, &matrix());
assert!((w - 0.5).abs() < 1e-6);
}
#[test]
fn weight_is_the_mean_not_the_sum_over_pairs() {
// carriers 0, 1, 2: pairs (0,1)=1/2, (0,2)=1/4, (1,2)=1 -> mean 7/12
let w = pair_balance_weight(&[1, 1, 1, 0], false, &matrix());
assert!((w - 7.0 / 12.0).abs() < 1e-6);
}
#[test]
fn fewer_than_two_carriers_weigh_nothing() {
assert_eq!(pair_balance_weight(&[1, 0, 0, 0], false, &matrix()), 0.0);
assert_eq!(pair_balance_weight(&[0, 0, 0, 0], false, &matrix()), 0.0);
}
#[test]
fn no_ambiguity_leaves_multi_member_genomes_out_of_the_pairs() {
let masks = [1u8, 1, 0b0011, 0];
// without the flag genome 2 carries the family: pairs (0,1), (0,2), (1,2)
let with = pair_balance_weight(&masks, false, &matrix());
// with it, only the pair (0,1) remains
let without = pair_balance_weight(&masks, true, &matrix());
assert!((with - 7.0 / 12.0).abs() < 1e-6);
assert!((without - 0.5).abs() < 1e-6);
}
#[test]
fn a_rare_pair_outweighs_a_common_one() {
// genomes (0,2) share many families, (1,3) few
let common = pair_balance_weight(&[1, 0, 1, 0], false, &matrix());
let rare = pair_balance_weight(&[0, 1, 0, 1], false, &matrix());
assert!(rare > common);
}
}
@@ -0,0 +1,45 @@
//! `--quartet`: restricting a layer's eligible families to the
//! quartet-informative ones, from the persisted quartet annex.
use std::path::Path;
use obicompactvec::{PersistentBitVec, TempBitVecBuilder};
use obikindex::{OKIError, OKIResult};
use super::minorant_selection::is_non_monomorphic_minorant;
use crate::siblings::quartet_annex::quartet_annex_file_name;
use crate::siblings::{ANNEX_FILE_NAME, SiblingAnnex};
/// Clears, in `eligible` (indexed by `family_idx`, monomorphic minorants
/// included), every family the quartet annex flags as not quartet-informative
/// under the `no_ambiguity` counting convention. The annex is indexed by the
/// compact non-monomorphic ordinal, so the sibling annex is walked in the same
/// order to translate one numbering into the other — annex-only, no
/// cross-partition resolution. The annex must already exist
/// ([`super::family_annexes::ensure_layer_family_annexes`] first).
pub(crate) fn restrict_to_quartet(
eligible: &mut TempBitVecBuilder,
layer_dir: &Path,
no_ambiguity: bool,
) -> OKIResult<()> {
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(no_ambiguity)))
.map_err(OKIError::Io)?;
let mut family_idx = 0usize;
let mut compact = 0usize;
for slot in 0..annex.len() {
let Some(mask) = annex.get(slot) else { continue };
if !mask.is_minorant() {
continue;
}
if is_non_monomorphic_minorant(mask) {
if !flags.get(compact) {
eligible.set(family_idx, false);
}
compact += 1;
}
family_idx += 1;
}
Ok(())
}
@@ -14,7 +14,7 @@ use obikidxcache::index_cache::IndexCache;
use obikindex::OKIResult;
use super::stats::sibling_family_size_histogram;
use super::subsample::{EntropyBias, SurvivingFamily, sample_index};
use super::subsample::{SamplingSpec, SurvivingFamily, sample_index};
/// One accumulator fed every layer's [`SurvivingFamily`] batch during a
/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of
@@ -42,26 +42,24 @@ fn resolve_target(cache: &IndexCache, n: Option<usize>) -> OKIResult<usize> {
}
}
/// Run one shared, possibly-subsampled/entropy-biased selection over
/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see
/// [`sample_index`]'s own docs for what each controls), feeding every
/// Run one shared, possibly-subsampled/entropy-biased/quartet-restricted
/// selection over `cache` (`n`/`spec`/`excluded` — see [`sample_index`]'s
/// own docs and [`SamplingSpec`] for what each controls), feeding every
/// surviving family of every visited layer to every reducer in
/// `reducers`, in the same order, before moving to the next layer. A
/// no-op (not an error) if the resolved target is `0`.
pub fn sampled_scan(
cache: &IndexCache,
n: Option<usize>,
free_loss: bool,
no_ambiguity: bool,
spec: SamplingSpec,
excluded: &[bool],
entropy_bias: Option<EntropyBias>,
reducers: &mut [&mut dyn LayerReducer],
) -> OKIResult<()> {
let target = resolve_target(cache, n)?;
if target == 0 {
return Ok(());
}
sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| {
sample_index(cache, target, spec, excluded, 0, |partition, layer, _raw_index, survivors| {
for reducer in reducers.iter_mut() {
reducer.reduce(partition, layer, &survivors);
}
@@ -51,12 +51,11 @@ use obikidxcache::index_cache::IndexCache;
use obikindex::OKIResult;
use rand::RngExt;
use super::entropy::iter_full_entropy;
use super::family_scan::{Selection, scan_layer_families};
use super::is_fast_mode;
use super::masking::{masked_state, survives_masking};
use super::minorant_selection::eligible_bitset_layer;
use crate::siblings::FamilyMask;
use super::masking::{masked_state, survives_masking, survives_quartet};
use super::minorant_selection::{eligible_bitset_layer, iter_full_scalar};
use super::{AnnexRequest, is_fast_mode, restrict_to_quartet};
use crate::siblings::{FamilyMask, ScalarKind};
use crate::siblings::extensions::SiblingBuilder;
/// Resolution rounds a single layer's sampling may take before giving up on
@@ -96,21 +95,61 @@ pub struct EntropyBias {
pub sigma: f64,
}
/// [`iter_full_entropy`], rotated to start at `start` and wrap once back to
/// it — the same `family_idx` order [`sample_layer`]'s own circular index
/// sequence walks, so the two can be `.zip()`-ed directly. Two more streamed
/// passes over the layer's (small) annex/entropy files, not a materialised
/// `Vec`/`HashMap` — `iter_full_entropy` itself never allocates more than
/// its own iterator state.
fn circular_entropy_values(
/// How a draw is conditioned: the character masking (`--free-loss`,
/// `--no-ambiguity`), the quartet restriction (`--quartet`: only families
/// that can separate two pairs of genomes) and the optional entropy bias.
/// `--no-ambiguity` also selects the counting convention of the persisted
/// entropy and quartet annexes the draw reads, so the draw and the alignment
/// it feeds always describe the same genomes. `pub`, not `pub(crate)`: part
/// of the public signature of [`super::scan::sampled_scan`].
#[derive(Clone, Copy)]
pub struct SamplingSpec {
pub free_loss: bool,
pub no_ambiguity: bool,
pub quartet: bool,
/// `--pair-balance`: weight each family by the mean, over the pairs of
/// genomes carrying it, of 1 / (number of families the pair shares), so
/// the draw favours the families covering the rarest pairs (see
/// `algorithms::pair_weight`).
pub pair_balance: bool,
pub entropy_bias: Option<EntropyBias>,
}
/// [`iter_full_scalar`] of `kind` (entropy, pair weight), rotated to start at
/// `start` and wrap once back to it — the same `family_idx` order
/// [`sample_layer`]'s own circular index sequence walks, so the streams can be
/// `.zip()`-ed directly. Two more streamed passes over the layer's (small)
/// annex files, not a materialised `Vec`/`HashMap` — `iter_full_scalar`
/// itself never allocates more than its own iterator state. `no_ambiguity`
/// selects the annex built under the same counting convention as the masking
/// the survivors go through afterwards.
fn circular_scalar_values(
layer_dir: &Path,
kind: ScalarKind,
start: usize,
no_ambiguity: bool,
) -> OKIResult<impl Iterator<Item = f32> + use<>> {
let head = iter_full_entropy(layer_dir)?.skip(start);
let tail = iter_full_entropy(layer_dir)?.take(start);
let head = iter_full_scalar(layer_dir, kind, no_ambiguity)?.skip(start);
let tail = iter_full_scalar(layer_dir, kind, no_ambiguity)?.take(start);
Ok(head.chain(tail))
}
/// One layer's place in a draw, planned before any layer is sampled: which of
/// its families may be drawn, and how much of the index-wide quota it
/// deserves.
struct LayerPlan {
partition: usize,
layer: usize,
/// One bit per minorant (`family_idx` order): still eligible to be drawn.
eligible: TempBitVecBuilder,
n_minorants: usize,
/// Total sampling weight of the eligible families: their number, or, with
/// `--pair-balance`, the sum of their pair weights. A layer's quota is its
/// share of this mass over every layer, so a layer rich in families that
/// cover rare pairs gets more of the draw.
mass: f64,
}
/// Samples up to `n` non-monomorphic families index-wide, proportionally
/// per layer, calling `on_layer(partition, layer, survivors)` once per
/// cached layer that contributed anything, with every family from that
@@ -159,10 +198,8 @@ fn circular_entropy_values(
pub(crate) fn sample_index(
cache: &IndexCache,
n: usize,
free_loss: bool,
no_ambiguity: bool,
spec: SamplingSpec,
excluded: &[bool],
entropy_bias: Option<EntropyBias>,
skip_layers: usize,
mut on_layer: impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
) -> OKIResult<usize> {
@@ -170,64 +207,79 @@ pub(crate) fn sample_index(
let fast_mode = is_fast_mode(cache);
// Entropy-biased mode reads each candidate's entropy from the entropy
// annex (`EntropyWeights::open`, per layer, below) — build any missing
// one now, once, rather than mid-walk.
if entropy_bias.is_some() {
cache.ensure_entropy_and_overlap_annexes()?;
// annex, `--quartet` restricts the candidates through the quartet annex,
// and `--pair-balance` weights them by the pair-weight annex (all per
// layer, below, each under this run's `--no-ambiguity` counting
// convention) — build any missing one now, once, rather than mid-walk.
let request = AnnexRequest {
entropy: spec.entropy_bias.map(|_| vec![spec.no_ambiguity]).unwrap_or_default(),
quartet: if spec.quartet { vec![spec.no_ambiguity] } else { Vec::new() },
overlap: Vec::new(),
pair_weight: if spec.pair_balance { vec![spec.no_ambiguity] } else { Vec::new() },
};
if !request.entropy.is_empty() || !request.quartet.is_empty() || !request.pair_weight.is_empty() {
cache.ensure_family_annexes(&request)?;
}
// Every cached layer's eligibility bitset, built up front — annex-only,
// no cross-partition resolution, so cheap even summed over the whole
// index — needed before any layer's quota can be computed (each
// layer's share depends on every other layer's own count).
let mut layers: Vec<(usize, usize, TempBitVecBuilder, usize)> = Vec::new();
for part in cache.partitions() {
let n_layer = cache.n_layer(part).unwrap_or(0);
for l in 0..n_layer {
// Every cached layer's plan, built up front — annex-only, no
// cross-partition resolution, so cheap even summed over the whole index
// — needed before any layer's quota can be computed (each layer's share
// depends on every other layer's own mass).
let mut plans: Vec<LayerPlan> = Vec::new();
for partition in cache.partitions() {
let n_layer = cache.n_layer(partition).unwrap_or(0);
for layer_idx in 0..n_layer {
let layer = cache
.get_layer(part, l)
.expect("cache-consistent: n_layer(part) already bounds l");
let eligible = eligible_bitset_layer(layer.dir())?;
.get_layer(partition, layer_idx)
.expect("cache-consistent: n_layer(partition) already bounds layer_idx");
let mut eligible = eligible_bitset_layer(layer.dir())?;
if spec.quartet {
restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?;
}
let n_minorants = eligible.view().len();
layers.push((part, l, eligible, n_minorants));
let mass = if spec.pair_balance {
iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)?
.enumerate()
.filter(|&(family_idx, _)| eligible.view().get(family_idx))
.map(|(_, weight)| f64::from(weight))
.sum()
} else {
eligible.view().count_ones() as f64
};
plans.push(LayerPlan { partition, layer: layer_idx, eligible, n_minorants, mass });
}
}
let total_eligible: u64 = layers
.iter()
.map(|(_, _, e, _)| e.view().count_ones())
.sum();
if total_eligible == 0 {
let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum();
if total_mass <= 0.0 {
return Ok(0);
}
// Quotas by cumulative rounding: layer `i` is allotted
// `floor(n × cumulative mass up to i / total) − what the previous layers
// were allotted`, so the quotas sum to exactly `n`. Rounding each layer's
// share down on its own would lose up to one site per layer — all of
// them, when `n` is smaller than the number of layers.
let mut total_kept = 0usize;
for (raw_index, (part, l, mut eligible, n_minorants)) in layers.into_iter().enumerate() {
if raw_index < skip_layers {
continue;
}
let count_layer = eligible.view().count_ones();
if count_layer == 0 {
continue;
}
let quota = ((n as u128 * count_layer as u128) / total_eligible as u128) as usize;
if quota == 0 {
let mut cumulative_mass = 0.0f64;
let mut allotted = 0usize;
for (raw_index, mut plan) in plans.into_iter().enumerate() {
cumulative_mass += plan.mass;
let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize;
let quota = cumulative_quota.saturating_sub(allotted);
allotted = allotted.max(cumulative_quota);
if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 {
continue;
}
total_kept += sample_layer(
cache,
part,
l,
raw_index,
&mut eligible,
n_minorants,
&mut plan,
quota,
n_genomes,
fast_mode,
free_loss,
no_ambiguity,
spec,
excluded,
entropy_bias,
&mut on_layer,
)?;
}
@@ -236,32 +288,28 @@ pub(crate) fn sample_index(
}
/// One layer's share of [`sample_index`] — see the module docs for the
/// per-family accept/reject/mark rules and the round-batching scheme.
#[allow(clippy::too_many_arguments)]
/// per-family accept/reject/mark rules and the round-batching scheme. Each
/// eligible family is accepted with probability
/// `quota / mass × pair weight × entropy kernel` (capped at 1): the pair
/// weight is `1` and `mass` the number of eligible families without
/// `--pair-balance`, and the kernel `1` without an entropy bias.
fn sample_layer(
cache: &IndexCache,
partition: usize,
layer_idx: usize,
raw_index: usize,
eligible: &mut TempBitVecBuilder,
n_minorants: usize,
plan: &mut LayerPlan,
quota: usize,
n_genomes: usize,
fast_mode: bool,
free_loss: bool,
no_ambiguity: bool,
spec: SamplingSpec,
excluded: &[bool],
entropy_bias: Option<EntropyBias>,
on_layer: &mut impl FnMut(usize, usize, usize, Arc<Vec<SurvivingFamily>>),
) -> OKIResult<usize> {
if n_minorants == 0 {
let LayerPlan { partition, layer: layer_idx, n_minorants, mass, .. } = *plan;
let eligible = &mut plan.eligible;
if n_minorants == 0 || mass <= 0.0 {
return Ok(0);
}
let count_layer = eligible.view().count_ones() as usize;
if count_layer == 0 {
return Ok(0);
}
let p0 = (quota as f64 / count_layer as f64).min(1.0);
let p0 = quota as f64 / mass;
let layer_dir = cache
.get_layer(partition, layer_idx)
.expect("cache-consistent")
@@ -301,34 +349,50 @@ fn sample_layer(
// This round's own circular walk, resuming exactly where the
// previous round left off (`start + visited`, wrapped) — zipped
// against that same rotation of the entropy stream when biased, so
// each candidate's own kernel weight is read in lockstep, never via
// a separate lookup structure. Unbiased draws skip the entropy
// stream entirely (a flat `1.0` kernel), never touching the entropy
// annex.
// against that same rotation of the entropy and pair-weight streams
// when they are used, so each candidate's own kernel and weight are
// read in lockstep, never via a separate lookup structure. A draw
// without the corresponding option skips its stream entirely (flat
// `1.0`), never touching the annex.
let round_start = (start + visited) % n_minorants;
let entropies: Box<dyn Iterator<Item = f32>> = match entropy_bias {
Some(_) => Box::new(circular_entropy_values(&layer_dir, round_start)?),
let entropies: Box<dyn Iterator<Item = f32>> = match spec.entropy_bias {
Some(_) => Box::new(circular_scalar_values(
&layer_dir,
ScalarKind::Entropy,
round_start,
spec.no_ambiguity,
)?),
None => Box::new(std::iter::repeat(0.0f32)), // never read: kernel forced to 1.0 below
};
let weights: Box<dyn Iterator<Item = f32>> = if spec.pair_balance {
Box::new(circular_scalar_values(
&layer_dir,
ScalarKind::PairWeight,
round_start,
spec.no_ambiguity,
)?)
} else {
Box::new(std::iter::repeat(1.0f32))
};
let candidates = (round_start..n_minorants)
.chain(0..round_start)
.zip(entropies);
.zip(entropies)
.zip(weights);
let mut accepted: HashSet<usize> = HashSet::new();
for (family_idx, entropy) in candidates {
for ((family_idx, entropy), weight) in candidates {
visited += 1;
if !eligible.view().get(family_idx) {
continue;
}
let kernel = match entropy_bias {
let kernel = match spec.entropy_bias {
Some(bias) => {
let d = (entropy as f64 - bias.mu) / bias.sigma;
(-0.5 * d * d).exp()
}
None => 1.0,
};
if rng.random::<f64>() < (p0 * kernel).min(1.0) {
if rng.random::<f64>() < (p0 * f64::from(weight) * kernel).min(1.0) {
eligible.set(family_idx, false);
accepted.insert(family_idx);
if accepted.len() >= target_this_round {
@@ -353,9 +417,13 @@ fn sample_layer(
mask.family_size() >= 2,
"eligible bitset must only accept non-monomorphic minorants"
);
let masked: Vec<Option<u8>> =
genome_mask.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect();
if survives_masking(&masked, excluded) {
let masked: Vec<Option<u8>> = genome_mask
.iter()
.map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity))
.collect();
if survives_masking(&masked, excluded)
&& (!spec.quartet || survives_quartet(&masked, excluded))
{
survivors.push(SurvivingFamily { family_idx, mask, masked });
kept += 1;
}
-162
View File
@@ -1,162 +0,0 @@
//! Persisted, per-layer Shannon entropy (`entropy15`, see
//! `algorithms::entropy`), one value per **non-monomorphic minorant
//! family** — compact: indexed by the dedicated ordinal
//! `algorithms::non_monomorphic_reindex_layer` computes (skips monomorphic
//! minorants entirely, ~98% of minorants on real data), never by
//! `scan_layer_families`'s own `family_idx` (which counts every minorant,
//! monomorphic included).
//!
//! Exists so repeated reads of the same layer's entropy values (e.g.
//! entropy-biased family selection, once reconnected) are a plain
//! positional mmap lookup instead of re-paying `scan_layer_families`'s
//! cross-partition resolution — building this file costs exactly that same
//! full scan the first time, it does not make any single computation of
//! entropy itself faster.
//!
//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat
//! per-entry data, read-only after build), just with `f32` payloads and a
//! different (compact) numbering — see the module docs above.
use std::fs::{File, OpenOptions};
use std::io;
use std::path::Path;
use memmap2::{Mmap, MmapMut};
const MAGIC: [u8; 4] = *b"PENT";
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
const HEADER_SIZE: usize = 16;
/// Sentinel: not yet computed. Entropy is always `>= 0.0`, so any negative
/// value is unambiguous — chosen over `NaN` to avoid `NaN`-comparison
/// footguns at every read site.
const SENTINEL: f32 = -1.0;
pub(crate) const ENTROPY_ANNEX_FILE_NAME: &str = "entropy.pent";
pub struct EntropyAnnex {
mmap: Mmap,
n: usize,
}
impl EntropyAnnex {
pub fn open(path: &Path) -> io::Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
if mmap.len() < HEADER_SIZE {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"PENT file too short",
));
}
if mmap[0..4] != MAGIC {
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PENT magic"));
}
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
if mmap.len() < HEADER_SIZE + n * 4 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"PENT file truncated",
));
}
Ok(Self { mmap, n })
}
pub fn len(&self) -> usize {
self.n
}
/// `None` for the sentinel (not yet computed — shouldn't happen against
/// a fully-built annex).
pub fn get(&self, idx: usize) -> Option<f32> {
let off = HEADER_SIZE + idx * 4;
let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap());
(v >= 0.0).then_some(v)
}
}
pub struct EntropyAnnexBuilder {
mmap: MmapMut,
}
impl EntropyAnnexBuilder {
/// Create a new annex of `n` entries (a layer's non-monomorphic
/// minorant count) at `path`, pre-initialised to the sentinel.
pub(crate) fn new(n: usize, path: &Path) -> io::Result<Self> {
let file_size = HEADER_SIZE + n * 4;
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
file.set_len(file_size as u64)?;
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
mmap[0..4].copy_from_slice(&MAGIC);
mmap[4..8].copy_from_slice(&[0u8; 4]);
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
for i in 0..n {
let off = HEADER_SIZE + i * 4;
mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes());
}
Ok(Self { mmap })
}
pub(crate) fn set(&mut self, idx: usize, value: f32) {
debug_assert!(value >= 0.0, "entropy must be non-negative, got {value}");
let off = HEADER_SIZE + idx * 4;
self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes());
}
pub(crate) fn close(self) -> io::Result<()> {
self.mmap.flush()
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn sentinel_is_negative_and_unset_entries_read_as_none() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.pent");
let builder = EntropyAnnexBuilder::new(4, &path).unwrap();
builder.close().unwrap();
let annex = EntropyAnnex::open(&path).unwrap();
for i in 0..4 {
assert_eq!(annex.get(i), None);
}
}
#[test]
fn roundtrip_including_zero_entropy() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.pent");
let mut builder = EntropyAnnexBuilder::new(4, &path).unwrap();
let values = [0.0f32, 1.5, 3.9, 0.0001];
for (i, &v) in values.iter().enumerate() {
builder.set(i, v);
}
builder.close().unwrap();
let annex = EntropyAnnex::open(&path).unwrap();
for (i, &v) in values.iter().enumerate() {
assert_eq!(annex.get(i), Some(v), "entry {i}");
}
}
#[test]
fn mixed_sentinel_and_real_values() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.pent");
let mut builder = EntropyAnnexBuilder::new(3, &path).unwrap();
builder.set(1, 2.0);
builder.close().unwrap();
let annex = EntropyAnnex::open(&path).unwrap();
assert_eq!(annex.get(0), None);
assert_eq!(annex.get(1), Some(2.0));
assert_eq!(annex.get(2), None);
assert_eq!(annex.len(), 3);
}
}
@@ -1,7 +1,8 @@
//! [`SiblingBuilder`] — construction triggered *internally*, as a side
//! effect of a [`crate::siblings::extensions::SiblingExt`] method (e.g.
//! `shannon_entropy_csv`/`family_overlap` ensuring their annexes exist
//! before reading them), never called directly from outside this crate.
//! `family_overlap` ensuring its annex exists before reading it, or the
//! sampler ensuring the entropy/quartet/pair-weight annexes it draws from),
//! never called directly from outside this crate.
//! Unlike `SiblingExt` (explicit, CLI-facing actions), nothing here is
//! meant to be its own CLI flag — same `pub`/`pub(crate)` split
//! `obikindexer` uses for its own `IndexBuilder`/`PrivateBuilder`.
@@ -10,44 +11,75 @@ use obikidxcache::index_cache::IndexCache;
use obikindex::{OKIError, OKIResult};
use obisys::progress_bar;
use crate::siblings::algorithms::{ensure_layer_entropy_annex, is_fast_mode};
use crate::siblings::{FAMILY_OVERLAP_FILE_NAME, FamilyOverlapAccumulator};
use crate::siblings::algorithms::{
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
inverse_overlap_matrix, is_fast_mode,
};
use crate::siblings::{FamilyOverlap, FamilyOverlapAccumulator, family_overlap_file_name};
pub(crate) trait SiblingBuilder {
/// Build every cached layer's entropy annex, and the whole index's
/// family-overlap annex, that don't already exist — one call covers
/// both, whichever of the two a caller actually needs
/// (`shannon_entropy_csv`/entropy-biased sampling on one side,
/// `family_overlap`/`--min-shared-family` on the other): a missing
/// annex always triggers a scan that feeds both, per
/// `ensure_layer_entropy_annex`'s own docs. A no-op, cheap
/// existence-check pass, once both already exist everywhere.
fn ensure_entropy_and_overlap_annexes(&self) -> OKIResult<()>;
/// Build, for every cached layer, the annexes `request` asks for that
/// don't already exist (entropy, quartet and pair weight, each per
/// `--no-ambiguity` counting convention), and the index-level
/// family-overlap annexes it asks for or that a pair weight is computed
/// from. One call covers all of them, whichever a caller actually needs:
/// a missing entropy, quartet or overlap annex triggers one scan that
/// feeds every missing one, per `ensure_layer_family_annexes`'s own docs,
/// then a second scan builds the missing pair weights, which need the
/// finished overlap matrix. A no-op, cheap existence-check pass, once
/// everything requested already exists.
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>;
}
impl SiblingBuilder for IndexCache {
fn ensure_entropy_and_overlap_annexes(&self) -> OKIResult<()> {
fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
let n_genomes = self.meta().genomes().len();
let fast_mode = is_fast_mode(self);
let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity));
let overlap_path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
let mut overlap_acc =
(!overlap_path.exists()).then(|| FamilyOverlapAccumulator::new(n_genomes));
let pb = progress_bar("entropy_overlap_annex", self.n_partition() as u64, "partitions");
// Sweep 1: entropy, quartet, and every family-overlap annex still missing.
let mut overlap_accs: Vec<(bool, FamilyOverlapAccumulator)> = request
.overlap_conventions()
.into_iter()
.filter(|&no_ambiguity| !overlap_path(no_ambiguity).exists())
.map(|no_ambiguity| (no_ambiguity, FamilyOverlapAccumulator::new(n_genomes, no_ambiguity)))
.collect();
let pb = progress_bar("family_annexes", self.n_partition() as u64, "partitions");
for part in self.partitions() {
let n_layer = self.n_layer(part).unwrap_or(0);
for l in 0..n_layer {
ensure_layer_entropy_annex(self, part, l, n_genomes, fast_mode, overlap_acc.as_mut())?;
ensure_layer_family_annexes(self, part, l, n_genomes, fast_mode, request, &mut overlap_accs)?;
}
pb.inc(1);
}
pb.finish_and_clear();
for (no_ambiguity, acc) in overlap_accs {
acc.write(&overlap_path(no_ambiguity)).map_err(OKIError::Io)?;
}
// Sweep 2: pair weights, from the now-complete overlap annexes.
if request.pair_weight.is_empty() {
return Ok(());
}
let matrices = request
.pair_weight_conventions()
.into_iter()
.map(|no_ambiguity| {
FamilyOverlap::open(&overlap_path(no_ambiguity))
.map(|overlap| (no_ambiguity, inverse_overlap_matrix(&overlap)))
.map_err(OKIError::Io)
})
.collect::<OKIResult<Vec<_>>>()?;
let pb = progress_bar("pair_weight_annex", self.n_partition() as u64, "partitions");
for part in self.partitions() {
let n_layer = self.n_layer(part).unwrap_or(0);
for l in 0..n_layer {
ensure_layer_pair_weight_annexes(self, part, l, n_genomes, fast_mode, &matrices)?;
}
pb.inc(1);
}
pb.finish_and_clear();
if let Some(acc) = overlap_acc {
acc.write(&overlap_path).map_err(OKIError::Io)?;
}
Ok(())
}
}
@@ -14,12 +14,15 @@ use obikindex::{OKIError, OKIResult};
use obisys::progress_bar;
use crate::siblings::algorithms::{
Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4,
is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, sibling_annex_stats,
sibling_family_size_histogram,
AnnexRequest, Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy,
family_entropy_4, is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families,
sibling_annex_stats, sibling_family_size_histogram,
};
use crate::siblings::extensions::SiblingBuilder;
use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap};
use crate::siblings::{
FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, QUARTET_ANNEX_FILE_NAMES, ScalarKind,
family_overlap_file_name,
};
pub trait SiblingExt {
/// Build the sibling-count/minorant annex for every layer of every
@@ -81,8 +84,10 @@ pub trait SiblingExt {
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
/// The Family Overlap annex — number of shared *variable* families per
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs).
/// Built (alongside the entropy annex) if missing, same as every other
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs),
/// under the `--no-ambiguity` counting convention `no_ambiguity` selects
/// (genomes carrying several members of a family do not count as
/// carrying it when `true`). Built if missing, same as every other
/// annex-backed method here. Feeds `--family-overlap`'s own matrix
/// output directly, and `--min-shared-family`'s auto-exclusion
/// (`FamilyOverlap::mean_row(g) < threshold` — same statistic
@@ -90,7 +95,7 @@ pub trait SiblingExt {
/// triggers adding `g` to the excluded-genome set consumed by
/// `algorithms::scan::sampled_scan`'s callers; this needs no
/// computation of its own beyond that check).
fn family_overlap(&self) -> OKIResult<FamilyOverlap>;
fn family_overlap(&self, no_ambiguity: bool) -> OKIResult<FamilyOverlap>;
}
impl SiblingExt for IndexCache {
@@ -115,16 +120,24 @@ impl SiblingExt for IndexCache {
.expect("cache-consistent: n_layer(part) already bounds l");
part_slots += build_layer_sibling_annex(self, &layer, l, n_parts, fast_mode)?;
// Invalidate this layer's entropy annex, if any — it was
// built against the sibling annex's *previous*
// `family_idx` numbering (`non_monomorphic_reindex_layer`
// reads the sibling annex), now stale.
// `ensure_entropy_and_overlap_annexes` below only builds
// what's *missing*, so the old file must be removed here
// for it to be rebuilt rather than silently left stale.
let entropy_path = layer.dir().join(ENTROPY_ANNEX_FILE_NAME);
if entropy_path.exists() {
std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?;
// Invalidate this layer's entropy and quartet annexes, if
// any — they were built against the sibling annex's
// *previous* `family_idx` numbering
// (`non_monomorphic_reindex_layer` reads the sibling
// annex), now stale. `ensure_family_annexes` below only
// builds what's *missing*, so the old files must be removed
// here for them to be rebuilt rather than silently left
// stale.
for name in ScalarKind::Entropy
.file_names()
.into_iter()
.chain(ScalarKind::PairWeight.file_names())
.chain(QUARTET_ANNEX_FILE_NAMES)
{
let entropy_path = layer.dir().join(name);
if entropy_path.exists() {
std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?;
}
}
}
total_slots += part_slots;
@@ -136,13 +149,16 @@ impl SiblingExt for IndexCache {
pb.finish_and_clear();
tracing::info!("sibling annex built — {total_slots} kmers across {n_parts} partitions");
// The family-overlap annex is index-level, not per-layer (see its
// own module docs) — any layer's sibling annex changing makes the
// whole thing stale, so it's invalidated once here rather than
// per-layer above.
let overlap_path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
if overlap_path.exists() {
std::fs::remove_file(&overlap_path).map_err(OKIError::Io)?;
// The family-overlap annexes are index-level, not per-layer (see
// their own module docs) — any layer's sibling annex changing makes
// them all stale (and, with them, every pair weight computed from
// them, removed per layer above), so they are invalidated once here
// rather than per-layer above.
for name in FAMILY_OVERLAP_FILE_NAMES {
let overlap_path = self.index().dir().join(name);
if overlap_path.exists() {
std::fs::remove_file(&overlap_path).map_err(OKIError::Io)?;
}
}
// Precompute the entropy and family-overlap annexes too, now that
@@ -152,19 +168,17 @@ impl SiblingExt for IndexCache {
// rather than leaving them to be built lazily on first
// `--shannon`/`--entropy-bias`/`--min-shared-family`/
// `--family-overlap` use.
tracing::info!("building entropy and family-overlap annexes");
self.ensure_entropy_and_overlap_annexes()?;
tracing::info!("building entropy and quartet (both --no-ambiguity conventions) and family-overlap annexes");
self.ensure_family_annexes(&AnnexRequest::everything())?;
Ok(())
}
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {
// Entropy/family-overlap annexes aren't this method's own output,
// but ensuring they exist is a cheap prerequisite check, not a
// reason to reuse their scan here — see below, this method
// deliberately runs its own, differently-scoped one.
self.ensure_entropy_and_overlap_annexes()?;
// Needs only the sibling annex (`non_monomorphic_reindex_layer`
// below): this method runs its own, differently-scoped scan and
// recomputes every value it writes, so it neither reads nor
// requires the entropy/family-overlap annexes.
let n_genomes = self.meta().genomes().len();
let fast_mode = is_fast_mode(self);
@@ -237,9 +251,9 @@ impl SiblingExt for IndexCache {
sibling_annex_stats(self)
}
fn family_overlap(&self) -> OKIResult<FamilyOverlap> {
self.ensure_entropy_and_overlap_annexes()?;
let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME);
fn family_overlap(&self, no_ambiguity: bool) -> OKIResult<FamilyOverlap> {
self.ensure_family_annexes(&AnnexRequest { overlap: vec![no_ambiguity], ..Default::default() })?;
let path = self.index().dir().join(family_overlap_file_name(no_ambiguity));
FamilyOverlap::open(&path).map_err(OKIError::Io)
}
}
+70 -12
View File
@@ -24,20 +24,43 @@ use std::path::Path;
use memmap2::Mmap;
use crate::siblings::helpers::triangle_index;
use crate::siblings::helpers::{is_carrier, triangle_index};
const MAGIC: [u8; 4] = *b"POVL";
// Header: magic(4) + _pad(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair,
// upper triangle) follows.
// Header: magic(4) + flags(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair,
// upper triangle) follows. `flags` bit 0 = `no_ambiguity` (see
// `FamilyOverlapAccumulator::new`); older files have `flags == 0`, i.e.
// `no_ambiguity == false`, which is what they were computed under.
const HEADER_SIZE: usize = 16;
const FLAG_NO_AMBIGUITY: u32 = 1;
pub(crate) const FAMILY_OVERLAP_FILE_NAME: &str = "family_overlap.povl";
/// Same annex, built with `no_ambiguity == true`.
pub(crate) const FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME: &str = "family_overlap.noamb.povl";
/// Every family-overlap file name, whatever its counting convention — for
/// invalidating them all together when the sibling annex they derive from is
/// rebuilt.
pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 2] =
[FAMILY_OVERLAP_FILE_NAME, FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME];
/// File name of the family-overlap annex built under the given
/// `--no-ambiguity` counting convention. The two never share a file.
pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str {
if no_ambiguity {
FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME
} else {
FAMILY_OVERLAP_FILE_NAME
}
}
/// Read-only, mmap-backed view of an already-built Family Overlap annex.
pub struct FamilyOverlap {
mmap: Mmap,
n_genomes: usize,
no_ambiguity: bool,
}
impl FamilyOverlap {
@@ -54,13 +77,22 @@ impl FamilyOverlap {
if mmap.len() < HEADER_SIZE + n_pairs * 8 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "POVL file truncated"));
}
Ok(Self { mmap, n_genomes })
let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap());
Ok(Self { mmap, n_genomes, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 })
}
pub fn n_genomes(&self) -> usize {
self.n_genomes
}
/// The counting convention these counts were computed under: `true`
/// when a genome carrying several members of a family was *not* counted
/// as carrying it (`--no-ambiguity`), `false` when any non-empty
/// state counts as carrying it.
pub fn no_ambiguity(&self) -> bool {
self.no_ambiguity
}
/// Number of variable families shared by genomes `i` and `j` — `0` for
/// `i == j` (not stored; a genome trivially "shares" every one of its
/// own families with itself, not a meaningful statistic here).
@@ -95,24 +127,30 @@ impl FamilyOverlap {
/// `EntropyAnnexBuilder` do for their own, much larger, per-k-mer data.
pub(crate) struct FamilyOverlapAccumulator {
n_genomes: usize,
no_ambiguity: bool,
counts: Vec<u64>,
}
impl FamilyOverlapAccumulator {
pub(crate) fn new(n_genomes: usize) -> Self {
/// `no_ambiguity` is the counting convention: `false` counts every
/// genome with a non-empty state as carrying the family, `true` only the
/// genomes carrying exactly one member of it — the others are missing
/// data in the alignment (`--no-ambiguity`), so they share nothing there.
pub(crate) fn new(n_genomes: usize, no_ambiguity: bool) -> Self {
let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2;
Self { n_genomes, counts: vec![0u64; n_pairs] }
Self { n_genomes, no_ambiguity, counts: vec![0u64; n_pairs] }
}
/// One variable family's contribution: every pair of genomes both
/// carrying it (`genome_mask[g] != 0`) gets its count incremented once.
/// carrying it (per the counting convention) gets its count incremented
/// once.
pub(crate) fn add_family(&mut self, genome_mask: &[u8]) {
for i in 0..self.n_genomes {
if genome_mask[i] == 0 {
if !is_carrier(genome_mask[i], self.no_ambiguity) {
continue;
}
for j in (i + 1)..self.n_genomes {
if genome_mask[j] != 0 {
if is_carrier(genome_mask[j], self.no_ambiguity) {
self.counts[triangle_index(self.n_genomes, i, j)] += 1;
}
}
@@ -122,7 +160,8 @@ impl FamilyOverlapAccumulator {
pub(crate) fn write(&self, path: &Path) -> io::Result<()> {
let mut buf = Vec::with_capacity(HEADER_SIZE + self.counts.len() * 8);
buf.extend_from_slice(&MAGIC);
buf.extend_from_slice(&[0u8; 4]);
let flags = if self.no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 };
buf.extend_from_slice(&flags.to_le_bytes());
buf.extend_from_slice(&(self.n_genomes as u64).to_le_bytes());
for &c in &self.counts {
buf.extend_from_slice(&c.to_le_bytes());
@@ -140,7 +179,7 @@ mod tests {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("test.povl");
let mut acc = FamilyOverlapAccumulator::new(3);
let mut acc = FamilyOverlapAccumulator::new(3, false);
acc.add_family(&[1, 1, 0]); // genomes 0,1 share this family
acc.add_family(&[1, 1, 1]); // all three share this one
acc.add_family(&[0, 1, 1]); // genomes 1,2 share this family
@@ -160,8 +199,27 @@ mod tests {
fn single_genome_mean_row_is_zero() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("test.povl");
FamilyOverlapAccumulator::new(1).write(&path).unwrap();
FamilyOverlapAccumulator::new(1, false).write(&path).unwrap();
let overlap = FamilyOverlap::open(&path).unwrap();
assert_eq!(overlap.mean_row(0), 0.0);
}
#[test]
fn no_ambiguity_does_not_count_multi_member_genomes_as_carriers() {
let dir = tempfile::tempdir().unwrap();
let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0100]];
for no_ambiguity in [false, true] {
let path = dir.path().join(family_overlap_file_name(no_ambiguity));
let mut acc = FamilyOverlapAccumulator::new(3, no_ambiguity);
for m in masks {
acc.add_family(m);
}
acc.write(&path).unwrap();
let overlap = FamilyOverlap::open(&path).unwrap();
assert_eq!(overlap.no_ambiguity(), no_ambiguity);
// genome 2 carries two members in the first family: only counted without the flag
assert_eq!(overlap.get(0, 2), if no_ambiguity { 1 } else { 2 });
assert_eq!(overlap.get(0, 1), 1 + 1);
}
}
}
@@ -0,0 +1,247 @@
//! Persisted, per-layer `f32` value per **non-monomorphic minorant family**:
//! the Shannon entropy of the family ([`ScalarKind::Entropy`], see
//! `algorithms::entropy`) or its pair-balance weight
//! ([`ScalarKind::PairWeight`], see `algorithms::pair_weight`). Compact:
//! indexed by the dedicated ordinal `algorithms::non_monomorphic_reindex_layer`
//! computes (skips monomorphic minorants entirely, ~98% of minorants on real
//! data), never by `scan_layer_families`'s own `family_idx` (which counts
//! every minorant, monomorphic included).
//!
//! Exists so repeated reads of the same layer's values (entropy-biased or
//! pair-balanced family selection) are a plain positional mmap lookup
//! instead of re-paying `scan_layer_families`'s cross-partition resolution —
//! building a file costs exactly that same full scan the first time, it does
//! not make any single computation of its value faster.
//!
//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat
//! per-entry data, read-only after build), just with `f32` payloads and a
//! different (compact) numbering. The two kinds share the format and differ
//! by magic and file name only, so one kind can never be read as the other.
use std::fs::{File, OpenOptions};
use std::io;
use std::path::Path;
use memmap2::{Mmap, MmapMut};
// Header: magic(4) + flags(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
// `flags` bit 0 = `no_ambiguity` (the counting convention the values were
// computed under, see `FamilyScalarAnnexBuilder::new`); every file written
// before the flag existed has `flags == 0`, i.e. `no_ambiguity == false`,
// which is what it was computed under.
const FLAG_NO_AMBIGUITY: u32 = 1;
const HEADER_SIZE: usize = 16;
/// Sentinel: not yet computed. Every stored value is `>= 0.0`, so any
/// negative value is unambiguous — chosen over `NaN` to avoid
/// `NaN`-comparison footguns at every read site.
const SENTINEL: f32 = -1.0;
/// What the stored `f32` means.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScalarKind {
/// Shannon entropy of the family (bits).
Entropy,
/// Mean, over the pairs of genomes carrying the family, of 1 / (number of
/// families the pair shares) — see `algorithms::pair_weight`.
PairWeight,
}
impl ScalarKind {
fn magic(self) -> [u8; 4] {
match self {
ScalarKind::Entropy => *b"PENT",
ScalarKind::PairWeight => *b"PPWT",
}
}
/// File name of this kind's annex built under the given `--no-ambiguity`
/// counting convention. The two conventions never share a file:
/// switching between them builds the missing annex without discarding the
/// other.
pub(crate) fn file_name(self, no_ambiguity: bool) -> &'static str {
match (self, no_ambiguity) {
(ScalarKind::Entropy, false) => "entropy.pent",
(ScalarKind::Entropy, true) => "entropy.noamb.pent",
(ScalarKind::PairWeight, false) => "pairweight.ppwt",
(ScalarKind::PairWeight, true) => "pairweight.noamb.ppwt",
}
}
/// Every file name of this kind, whatever its counting convention — for
/// invalidating them all together (e.g. when the sibling annex they were
/// numbered against is rebuilt).
pub(crate) fn file_names(self) -> [&'static str; 2] {
[self.file_name(false), self.file_name(true)]
}
}
pub struct FamilyScalarAnnex {
mmap: Mmap,
n: usize,
no_ambiguity: bool,
}
impl FamilyScalarAnnex {
pub(crate) fn open(path: &Path, kind: ScalarKind) -> io::Result<Self> {
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
if mmap.len() < HEADER_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file too short"));
}
if mmap[0..4] != kind.magic() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("bad magic: not a {kind:?} annex"),
));
}
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
if mmap.len() < HEADER_SIZE + n * 4 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file truncated"));
}
let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap());
Ok(Self { mmap, n, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 })
}
pub fn len(&self) -> usize {
self.n
}
/// The counting convention these values were computed under: `true`
/// when genomes carrying several members of a family were left out of
/// the count (`--no-ambiguity`), `false` when they were counted as a
/// state of their own.
pub fn no_ambiguity(&self) -> bool {
self.no_ambiguity
}
/// `None` for the sentinel (not yet computed — shouldn't happen against
/// a fully-built annex).
pub fn get(&self, idx: usize) -> Option<f32> {
let off = HEADER_SIZE + idx * 4;
let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap());
(v >= 0.0).then_some(v)
}
}
pub struct FamilyScalarAnnexBuilder {
mmap: MmapMut,
}
impl FamilyScalarAnnexBuilder {
/// Create a new annex of `n` entries (a layer's non-monomorphic
/// minorant count) at `path`, pre-initialised to the sentinel.
/// `no_ambiguity` records, in the header, the counting convention the
/// values will be computed under (see [`FamilyScalarAnnex::no_ambiguity`]).
pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, no_ambiguity: bool) -> io::Result<Self> {
let file_size = HEADER_SIZE + n * 4;
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(path)?;
file.set_len(file_size as u64)?;
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
mmap[0..4].copy_from_slice(&kind.magic());
let flags = if no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 };
mmap[4..8].copy_from_slice(&flags.to_le_bytes());
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
for i in 0..n {
let off = HEADER_SIZE + i * 4;
mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes());
}
Ok(Self { mmap })
}
pub(crate) fn set(&mut self, idx: usize, value: f32) {
debug_assert!(value >= 0.0, "stored values must be non-negative, got {value}");
let off = HEADER_SIZE + idx * 4;
self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes());
}
pub(crate) fn close(self) -> io::Result<()> {
self.mmap.flush()
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
const E: ScalarKind = ScalarKind::Entropy;
#[test]
fn sentinel_is_negative_and_unset_entries_read_as_none() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.pent");
FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap().close().unwrap();
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
for i in 0..4 {
assert_eq!(annex.get(i), None);
}
}
#[test]
fn roundtrip_including_zero_value() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.pent");
let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap();
let values = [0.0f32, 1.5, 3.9, 0.0001];
for (i, &v) in values.iter().enumerate() {
builder.set(i, v);
}
builder.close().unwrap();
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
for (i, &v) in values.iter().enumerate() {
assert_eq!(annex.get(i), Some(v), "entry {i}");
}
}
#[test]
fn mixed_sentinel_and_real_values() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.pent");
let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, false).unwrap();
builder.set(1, 2.0);
builder.close().unwrap();
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
assert_eq!(annex.get(0), None);
assert_eq!(annex.get(1), Some(2.0));
assert_eq!(annex.get(2), None);
assert_eq!(annex.len(), 3);
}
#[test]
fn counting_convention_roundtrips_through_the_header() {
let dir = tempdir().unwrap();
for kind in [ScalarKind::Entropy, ScalarKind::PairWeight] {
for no_ambiguity in [false, true] {
let path = dir.path().join(kind.file_name(no_ambiguity));
FamilyScalarAnnexBuilder::new(2, &path, kind, no_ambiguity).unwrap().close().unwrap();
assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().no_ambiguity(), no_ambiguity);
}
assert_ne!(kind.file_name(false), kind.file_name(true));
}
}
#[test]
fn one_kind_cannot_be_read_as_the_other() {
let dir = tempdir().unwrap();
let path = dir.path().join("w.ppwt");
FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, false).unwrap().close().unwrap();
assert!(FamilyScalarAnnex::open(&path, ScalarKind::Entropy).is_err());
assert!(FamilyScalarAnnex::open(&path, ScalarKind::PairWeight).is_ok());
}
#[test]
fn file_with_zero_flags_reads_as_ambiguity_counted() {
// files written before the flag existed carried four zero bytes there
let dir = tempdir().unwrap();
let path = dir.path().join("old.pent");
FamilyScalarAnnexBuilder::new(1, &path, E, false).unwrap().close().unwrap();
let bytes = std::fs::read(&path).unwrap();
assert_eq!(&bytes[4..8], &[0u8; 4]);
assert!(!FamilyScalarAnnex::open(&path, E).unwrap().no_ambiguity());
}
}
+12
View File
@@ -34,3 +34,15 @@ pub(super) fn triangle_index(n: usize, i: usize, j: usize) -> usize {
debug_assert!(j < n && i != j);
(i * (2 * n - i - 1)) / 2 + (j - i - 1)
}
/// Whether a genome with this per-family `mask` (bit `b` set iff it carries
/// the member whose central base is `b`) counts as carrying the family, under
/// the `--no-ambiguity` counting convention: any non-empty state does, except
/// — with `no_ambiguity` — a genome carrying several members, which is
/// missing data in the alignment and so shares nothing there. One definition
/// for every annex that counts carriers (family overlap, pair weight,
/// quartet), so they can never disagree.
#[inline]
pub(crate) fn is_carrier(mask: u8, no_ambiguity: bool) -> bool {
mask != 0 && !(no_ambiguity && mask.count_ones() > 1)
}
+10 -5
View File
@@ -20,17 +20,21 @@
//! reverse.
pub mod algorithms;
mod entropy_annex;
pub mod extensions;
mod family_overlap;
mod family_scalar_annex;
mod helpers;
pub mod iter;
mod quartet_annex;
mod siblingannex;
pub(crate) use entropy_annex::ENTROPY_ANNEX_FILE_NAME;
pub use entropy_annex::{EntropyAnnex, EntropyAnnexBuilder};
pub(crate) use family_overlap::{FAMILY_OVERLAP_FILE_NAME, FamilyOverlapAccumulator};
pub(crate) use family_overlap::{
FAMILY_OVERLAP_FILE_NAMES, FamilyOverlapAccumulator, family_overlap_file_name,
};
pub(crate) use family_scalar_annex::ScalarKind;
pub use family_scalar_annex::{FamilyScalarAnnex, FamilyScalarAnnexBuilder};
pub use family_overlap::FamilyOverlap;
pub(crate) use quartet_annex::QUARTET_ANNEX_FILE_NAMES;
pub use iter::{
MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt,
};
@@ -38,7 +42,8 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use algorithms::{
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment,
PairwiseReducer, RawSnpDistanceOutput, SamplingSpec, SankoffBundle, SiblingAnnexStats,
SnpAlignment,
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
pairwise_cost_matrix, sampled_scan,
};
@@ -0,0 +1,59 @@
//! Persisted, per-layer quartet-informativeness flag: one bit per
//! non-monomorphic minorant family, stored as a plain
//! [`obicompactvec::PersistentBitVec`] (no format of its own) and numbered
//! by the same compact ordinal as [`crate::siblings::EntropyAnnex`]
//! (`algorithms::non_monomorphic_reindex_layer`), never by
//! `scan_layer_families`'s own `family_idx`.
//!
//! A family is quartet-informative when its carrier genomes show at least
//! two distinct called states, each carried by at least two genomes: the
//! pattern that can separate two pairs of genomes from each other
//! (`AB|CD`). Anything weaker — a singleton variant, or a family carried by
//! fewer than four genomes — fits every topology and tells a tree nothing.
//! See `algorithms::masking::is_quartet_informative` for the exact rule.
//!
//! Which genomes count as carriers depends on `--no-ambiguity`, so each
//! convention has its own file, and **the file name is the parameter**: a
//! `PersistentBitVec` header has no room for a flag, and changing it would
//! change a shared structure for the sake of one annex.
//!
//! The flag is computed over every genome of the index. A later
//! `--exclude-genome` can only remove carriers, never create a bipartition,
//! so a family flagged `false` here is also uninformative once genomes are
//! excluded: the annex is a safe pre-filter, and the exact check
//! (`algorithms::masking::survives_quartet`) still runs on the resolved
//! family.
pub(crate) const QUARTET_ANNEX_FILE_NAME: &str = "quartet.pbiv";
/// Same annex, built with `no_ambiguity == true`.
pub(crate) const QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME: &str = "quartet.noamb.pbiv";
/// Every quartet-annex file name, whatever its counting convention — for
/// invalidating them all together when the sibling annex they were numbered
/// against is rebuilt.
pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 2] =
[QUARTET_ANNEX_FILE_NAME, QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME];
/// File name of the quartet annex built under the given `--no-ambiguity`
/// convention.
pub(crate) fn quartet_annex_file_name(no_ambiguity: bool) -> &'static str {
if no_ambiguity {
QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME
} else {
QUARTET_ANNEX_FILE_NAME
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_two_conventions_never_share_a_file() {
assert_ne!(quartet_annex_file_name(false), quartet_annex_file_name(true));
for no_ambiguity in [false, true] {
assert!(QUARTET_ANNEX_FILE_NAMES.contains(&quartet_annex_file_name(no_ambiguity)));
}
}
}