refactor: replace no_ambiguity bool with AmbiguityPolicy enum
Replace the binary `no_ambiguity` flag with an `AmbiguityPolicy` enum (`Keep`, `Forbid`, `OnlyDiploid`) to support multiple ambiguity conventions. This change updates CLI arguments, internal logic, and persistent storage formats to distinguish between keeping all genomes, forbidding ambiguous ones, or restricting analysis to diploid sites only. - Add `src/obikphylo/src/siblings/ambiguity.rs` defining the `AmbiguityPolicy` enum with variants `Keep`, `Forbid`, and `OnlyDiploid`, including methods for header flags and determining if a genome is dropped. - Update `PhyloArgs` in `args.rs` to derive `AmbiquityPolicy` from CLI flags and construct `AnnexSelection` for sibling annexes. - Modify `SamplingSpec` in `subsample.rs` to store `ambiguity: AmbiguityPolicy` instead of `no_ambiguity`, updating masking and scalar value logic accordingly. - Update `SiblingExt` in `sibling_ext.rs` so `build_sibling_annex` accepts an `AnnexSelection` to build only requested derived annexes, and `family_overlap` uses the specific policy. - Refactor `FamilyOverlap` in `family_overlap.rs` to store and use `AmbiguityPolicy`, replacing boolean checks with policy-based carrier determination. - Update `FamilyScalarAnnex` in `family_scalar_annex.rs` to store `AmbiguityPolicy` in headers, enabling distinct file names for each policy variant. - Modify `quartet_annex.rs` to generate distinct file names based on `AmbiguityPolicy`, ensuring no overlap between Keep, Forbid, and OnlyDiploid annexes. - Update `family_annexes.rs` to use `AnnexSelection` for building entropy, quartet, and pair weight annexes under specific policies instead of a single boolean convention. - Refactor algorithm modules (`entropy.rs`, `masking.rs`, `pair_weight.rs`, `quartet.rs`, `snp_distance.rs`) to accept `AmbiguityPolicy` arguments, replacing boolean checks with policy-specific logic (e.g., `OnlyDiploid` masking). - Update documentation in `phylo.md` and `evolutionary_distances.md` to describe the new `--only-diploid` option, the behavior of `--sibling-annex` with specific policies, and the separate storage of entropies per convention.
This commit is contained in:
1 parent
b55123a376
commit
5368c98875
21 files changed
+477
-278
No files matched your search
@@ -1,6 +1,7 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::Args;
|
||||
use obikphylo::siblings::{AmbiguityPolicy, AnnexSelection};
|
||||
use obikphylo::{DistanceMetric, GammaShape, SnpDistanceKind};
|
||||
|
||||
/// `--gamma-shape` CLI value: a fixed `alpha`, or `auto`/`estimate` to fit
|
||||
@@ -223,7 +224,13 @@ pub struct PhyloArgs {
|
||||
|
||||
/// Build (or rebuild) the sibling-count/minorant annex — independent of
|
||||
/// the distance metric below, meant to be run routinely, ahead of any
|
||||
/// SNP-family distance computation that will later consume it.
|
||||
/// SNP-family distance computation that will later consume it. Rebuilding
|
||||
/// it invalidates the annexes derived from it; those are rebuilt here
|
||||
/// only for the options given alongside (`--entropy` the entropy annex,
|
||||
/// `--quartet` the quartet one, `--pair-balance` the pair-weight and
|
||||
/// family-overlap ones, `--family-overlap`/`--min-shared-family` the
|
||||
/// family-overlap one), each under the `--no-ambiguity`/`--only-diploid`
|
||||
/// convention given — anything not asked for is built later, on demand.
|
||||
#[arg(long)]
|
||||
pub sibling_annex: bool,
|
||||
|
||||
@@ -526,6 +533,23 @@ pub struct PhyloArgs {
|
||||
}
|
||||
|
||||
impl PhyloArgs {
|
||||
/// How genomes carrying several members of a family are treated.
|
||||
pub fn ambiguity(&self) -> AmbiguityPolicy {
|
||||
AmbiguityPolicy::from_flags(self.no_ambiguity, self.only_diploid)
|
||||
}
|
||||
|
||||
/// The derived annexes the options of this invocation read, all under
|
||||
/// [`ambiguity`](Self::ambiguity).
|
||||
pub fn annex_selection(&self) -> AnnexSelection {
|
||||
AnnexSelection {
|
||||
policy: self.ambiguity(),
|
||||
entropy: self.entropy.is_some(),
|
||||
quartet: self.quartet,
|
||||
overlap: self.family_overlap || self.min_shared_family.is_some() || self.pair_balance,
|
||||
pair_weight: self.pair_balance,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a pseudo-alignment must be drawn and written (`--pseudo-alignment`,
|
||||
/// or any of its output-format flags).
|
||||
pub fn alignment_output(&self) -> bool {
|
||||
|
||||
@@ -85,7 +85,7 @@ pub fn run(args: PhyloArgs) {
|
||||
// so a concurrent run on the same index can't corrupt these writes.
|
||||
info!("building sibling-count/minorant annex");
|
||||
let t = Stage::start("sibling_annex");
|
||||
cache.build_sibling_annex().unwrap_or_else(|e| {
|
||||
cache.build_sibling_annex(&args.annex_selection()).unwrap_or_else(|e| {
|
||||
eprintln!("error building sibling annex: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
@@ -155,7 +155,7 @@ pub fn run(args: PhyloArgs) {
|
||||
// ── Family Overlap matrix (`--family-overlap`) ──────────────────────────────
|
||||
if args.family_overlap {
|
||||
let t = Stage::start("family_overlap");
|
||||
let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| {
|
||||
let overlap = cache.family_overlap(args.ambiguity()).unwrap_or_else(|e| {
|
||||
eprintln!("error computing family overlap: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
@@ -217,7 +217,7 @@ pub fn run(args: PhyloArgs) {
|
||||
// `args::PhyloArgs::min_shared_family`'s own docs on why).
|
||||
let snp_exclude_mask: Vec<bool> = match args.min_shared_family {
|
||||
Some(threshold) => {
|
||||
let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| {
|
||||
let overlap = cache.family_overlap(args.ambiguity()).unwrap_or_else(|e| {
|
||||
eprintln!("error computing family overlap: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
@@ -253,8 +253,7 @@ pub fn run(args: PhyloArgs) {
|
||||
};
|
||||
let sampling = SamplingSpec {
|
||||
free_loss: args.free_loss,
|
||||
no_ambiguity: args.no_ambiguity,
|
||||
only_diploid: args.only_diploid,
|
||||
ambiguity: args.ambiguity(),
|
||||
quartet: args.quartet,
|
||||
pair_balance: args.pair_balance,
|
||||
entropy_bias,
|
||||
|
||||
@@ -16,10 +16,16 @@
|
||||
/// real minorant, guarded against rather than dividing by zero). Second
|
||||
/// value: the denominator (`n_genomes_present`) used, for traceability.
|
||||
pub(crate) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> {
|
||||
family_entropy_with(genome_mask, AmbiguityPolicy::Keep)
|
||||
}
|
||||
|
||||
/// [`family_entropy`] over the genomes `policy` keeps as carriers
|
||||
/// ([`is_carrier`]).
|
||||
pub(crate) fn family_entropy_with(genome_mask: &[u8], policy: AmbiguityPolicy) -> Option<(f64, usize)> {
|
||||
let mut freq = [0u32; 16]; // index 0 (∅) never incremented, kept for direct `m as usize` indexing
|
||||
let mut present = 0u32;
|
||||
for &m in genome_mask {
|
||||
if m != 0 {
|
||||
if is_carrier(m, policy) {
|
||||
freq[m as usize] += 1;
|
||||
present += 1;
|
||||
}
|
||||
@@ -70,6 +76,9 @@ pub(crate) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> {
|
||||
Some((h, n_single as usize))
|
||||
}
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
use crate::siblings::helpers::is_carrier;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -104,10 +113,16 @@ mod tests {
|
||||
#[test]
|
||||
fn entropy_under_selects_the_counting_convention() {
|
||||
let masks = [0b0001u8, 0b0001, 0b0011]; // the last genome carries two members
|
||||
assert_eq!(family_entropy_under(&masks, true).unwrap().1, 2);
|
||||
assert_eq!(family_entropy_under(&masks, false).unwrap().1, 3);
|
||||
assert_eq!(family_entropy_under(&masks, true).unwrap().0, 0.0);
|
||||
assert!(family_entropy_under(&masks, false).unwrap().0 > 0.0);
|
||||
use AmbiguityPolicy::{Forbid, Keep, OnlyDiploid};
|
||||
assert_eq!(family_entropy_under(&masks, Forbid).unwrap().1, 2);
|
||||
assert_eq!(family_entropy_under(&masks, Keep).unwrap().1, 3);
|
||||
assert_eq!(family_entropy_under(&masks, OnlyDiploid).unwrap().1, 3);
|
||||
assert_eq!(family_entropy_under(&masks, Forbid).unwrap().0, 0.0);
|
||||
assert!(family_entropy_under(&masks, Keep).unwrap().0 > 0.0);
|
||||
// three members: kept by Keep, dropped by OnlyDiploid
|
||||
let masks = [0b0001u8, 0b0001, 0b0111];
|
||||
assert_eq!(family_entropy_under(&masks, Keep).unwrap().1, 3);
|
||||
assert_eq!(family_entropy_under(&masks, OnlyDiploid).unwrap().1, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -117,21 +132,24 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Shannon entropy (bits) of one family under the counting convention
|
||||
/// `--no-ambiguity` selects:
|
||||
/// - `false`: [`family_entropy`] — every genome carrying the family counts,
|
||||
/// Shannon entropy (bits) of one family under the counting convention an
|
||||
/// [`AmbiguityPolicy`] selects:
|
||||
/// - `Keep`: [`family_entropy`] — every genome carrying the family counts,
|
||||
/// a genome with several members being a state of its own;
|
||||
/// - `true`: [`family_entropy_4`] — only the genomes carrying exactly one
|
||||
/// member count, the others being missing data, exactly as the alignment
|
||||
/// itself will see them. Over those genomes the 15-state entropy can only
|
||||
/// ever take the four single-base states, so the two definitions coincide.
|
||||
/// - `Forbid` (`--no-ambiguity`): [`family_entropy_4`] — only the genomes
|
||||
/// carrying exactly one member count, the others being missing data,
|
||||
/// exactly as the alignment itself will see them. Over those genomes the
|
||||
/// 15-state entropy can only ever take the four single-base states, so the
|
||||
/// two definitions coincide;
|
||||
/// - `OnlyDiploid`: the 15-state entropy over the genomes with one or two
|
||||
/// members (ten states), the others being missing data.
|
||||
///
|
||||
/// Second value: the number of genomes the entropy was computed over.
|
||||
pub(crate) fn family_entropy_under(genome_mask: &[u8], no_ambiguity: bool) -> Option<(f64, usize)> {
|
||||
if no_ambiguity {
|
||||
family_entropy_4(genome_mask)
|
||||
} else {
|
||||
family_entropy(genome_mask)
|
||||
pub(crate) fn family_entropy_under(genome_mask: &[u8], policy: AmbiguityPolicy) -> Option<(f64, usize)> {
|
||||
match policy {
|
||||
AmbiguityPolicy::Keep => family_entropy(genome_mask),
|
||||
AmbiguityPolicy::Forbid => family_entropy_4(genome_mask),
|
||||
AmbiguityPolicy::OnlyDiploid => family_entropy_with(genome_mask, policy),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,34 +21,58 @@ use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::masking::is_quartet_informative;
|
||||
use super::minorant_selection::non_monomorphic_reindex_layer;
|
||||
use super::pair_weight::pair_balance_weight;
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
use crate::siblings::quartet_annex::quartet_annex_file_name;
|
||||
use crate::siblings::{
|
||||
FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind, family_overlap_file_name,
|
||||
};
|
||||
|
||||
/// Which annexes a caller needs, each listed once per `--no-ambiguity`
|
||||
/// counting convention (`false` = genomes carrying several members count as a
|
||||
/// state of their own, `true` = they are left out) — an empty list asks for
|
||||
/// none. A pair weight is computed from the family-overlap annex of its own
|
||||
/// convention, so asking for one asks for the other.
|
||||
/// Which annexes a caller needs, each listed once per ambiguity policy (how
|
||||
/// genomes carrying several members of a family are counted) — an empty list
|
||||
/// asks for none. A pair weight is computed from the family-overlap annex of
|
||||
/// its own policy, so asking for one asks for the other.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct AnnexRequest {
|
||||
pub entropy: Vec<bool>,
|
||||
pub quartet: Vec<bool>,
|
||||
pub overlap: Vec<bool>,
|
||||
pub pair_weight: Vec<bool>,
|
||||
pub entropy: Vec<AmbiguityPolicy>,
|
||||
pub quartet: Vec<AmbiguityPolicy>,
|
||||
pub overlap: Vec<AmbiguityPolicy>,
|
||||
pub pair_weight: Vec<AmbiguityPolicy>,
|
||||
}
|
||||
|
||||
/// The derived annexes a run asks to have ready, all under the one ambiguity
|
||||
/// `policy` of that run (`--sibling-annex` together with the options that
|
||||
/// say which annexes, and under which convention, the following computations
|
||||
/// will read). Nothing is built that is not asked for.
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct AnnexSelection {
|
||||
pub policy: AmbiguityPolicy,
|
||||
pub entropy: bool,
|
||||
pub quartet: bool,
|
||||
pub overlap: bool,
|
||||
pub pair_weight: bool,
|
||||
}
|
||||
|
||||
impl AnnexSelection {
|
||||
pub(crate) fn request(&self) -> AnnexRequest {
|
||||
let one = |wanted: bool| if wanted { vec![self.policy] } else { Vec::new() };
|
||||
AnnexRequest {
|
||||
entropy: one(self.entropy),
|
||||
quartet: one(self.quartet),
|
||||
overlap: one(self.overlap),
|
||||
pair_weight: one(self.pair_weight),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether any derived annex is asked for.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
!(self.entropy || self.quartet || self.overlap || self.pair_weight)
|
||||
}
|
||||
}
|
||||
|
||||
impl AnnexRequest {
|
||||
/// Everything under both conventions — what `--sibling-annex` precomputes.
|
||||
pub(crate) fn everything() -> Self {
|
||||
let both = vec![false, true];
|
||||
Self { entropy: both.clone(), quartet: both.clone(), overlap: both.clone(), pair_weight: both }
|
||||
}
|
||||
|
||||
/// The overlap conventions to ensure: the requested ones plus every one a
|
||||
/// requested pair weight is computed from, each once.
|
||||
pub(crate) fn overlap_conventions(&self) -> Vec<bool> {
|
||||
pub(crate) fn overlap_conventions(&self) -> Vec<AmbiguityPolicy> {
|
||||
dedup(self.overlap.iter().chain(&self.pair_weight).copied())
|
||||
}
|
||||
|
||||
@@ -59,25 +83,25 @@ impl AnnexRequest {
|
||||
let overlap = self
|
||||
.overlap_conventions()
|
||||
.into_iter()
|
||||
.all(|nb| cache.index().dir().join(family_overlap_file_name(nb)).exists());
|
||||
.all(|p| cache.index().dir().join(family_overlap_file_name(p)).exists());
|
||||
let pair_weight = self.pair_weight_conventions();
|
||||
overlap
|
||||
&& cache.iter_indexed().all(|(_, _, layer)| {
|
||||
let has = |name: &str| layer.dir().join(name).exists();
|
||||
self.entropy.iter().all(|&nb| has(ScalarKind::Entropy.file_name(nb)))
|
||||
&& self.quartet.iter().all(|&nb| has(quartet_annex_file_name(nb)))
|
||||
&& pair_weight.iter().all(|&nb| has(ScalarKind::PairWeight.file_name(nb)))
|
||||
self.entropy.iter().all(|&p| has(ScalarKind::Entropy.file_name(p)))
|
||||
&& self.quartet.iter().all(|&p| has(quartet_annex_file_name(p)))
|
||||
&& pair_weight.iter().all(|&p| has(ScalarKind::PairWeight.file_name(p)))
|
||||
})
|
||||
}
|
||||
|
||||
/// The pair-weight conventions requested, each once.
|
||||
pub(crate) fn pair_weight_conventions(&self) -> Vec<bool> {
|
||||
pub(crate) fn pair_weight_conventions(&self) -> Vec<AmbiguityPolicy> {
|
||||
dedup(self.pair_weight.iter().copied())
|
||||
}
|
||||
}
|
||||
|
||||
fn dedup(variants: impl IntoIterator<Item = bool>) -> Vec<bool> {
|
||||
let mut v: Vec<bool> = variants.into_iter().collect();
|
||||
fn dedup(variants: impl IntoIterator<Item = AmbiguityPolicy>) -> Vec<AmbiguityPolicy> {
|
||||
let mut v: Vec<AmbiguityPolicy> = variants.into_iter().collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
@@ -102,19 +126,21 @@ pub(crate) fn ensure_layer_family_annexes(
|
||||
n_genomes: usize,
|
||||
fast_mode: bool,
|
||||
request: &AnnexRequest,
|
||||
overlap_accs: &mut [(bool, FamilyOverlapAccumulator)],
|
||||
overlap_accs: &mut [(AmbiguityPolicy, FamilyOverlapAccumulator)],
|
||||
) -> OKIResult<()> {
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
|
||||
let missing = |name: &dyn Fn(bool) -> &'static str, variants: &[bool]| -> Vec<(bool, PathBuf)> {
|
||||
let missing = |name: &dyn Fn(AmbiguityPolicy) -> &'static str,
|
||||
variants: &[AmbiguityPolicy]|
|
||||
-> Vec<(AmbiguityPolicy, PathBuf)> {
|
||||
dedup(variants.iter().copied())
|
||||
.into_iter()
|
||||
.map(|no_ambiguity| (no_ambiguity, layer.dir().join(name(no_ambiguity))))
|
||||
.map(|policy| (policy, layer.dir().join(name(policy))))
|
||||
.filter(|(_, path)| !path.exists())
|
||||
.collect()
|
||||
};
|
||||
let missing_entropy = missing(&|nb| ScalarKind::Entropy.file_name(nb), &request.entropy);
|
||||
let missing_entropy = missing(&|p| ScalarKind::Entropy.file_name(p), &request.entropy);
|
||||
let missing_quartet = missing(&quartet_annex_file_name, &request.quartet);
|
||||
if missing_entropy.is_empty() && missing_quartet.is_empty() && overlap_accs.is_empty() {
|
||||
return Ok(());
|
||||
@@ -124,16 +150,16 @@ pub(crate) fn ensure_layer_family_annexes(
|
||||
let eligible: HashSet<usize> = reindex.keys().copied().collect();
|
||||
let mut entropy_builders = missing_entropy
|
||||
.iter()
|
||||
.map(|(no_ambiguity, path)| {
|
||||
FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *no_ambiguity)
|
||||
.map(|b| (*no_ambiguity, b))
|
||||
.map(|(policy, path)| {
|
||||
FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *policy)
|
||||
.map(|b| (*policy, b))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()
|
||||
.map_err(OKIError::Io)?;
|
||||
let mut quartet_builders = missing_quartet
|
||||
.iter()
|
||||
.map(|(no_ambiguity, path)| {
|
||||
PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*no_ambiguity, b))
|
||||
.map(|(policy, path)| {
|
||||
PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*policy, b))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()
|
||||
.map_err(OKIError::Io)?;
|
||||
@@ -151,13 +177,13 @@ pub(crate) fn ensure_layer_family_annexes(
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
let compact = reindex[&family_idx];
|
||||
for (no_ambiguity, builder) in entropy_builders.iter_mut() {
|
||||
if let Some((h, _)) = family_entropy_under(genome_mask, *no_ambiguity) {
|
||||
for (policy, builder) in entropy_builders.iter_mut() {
|
||||
if let Some((h, _)) = family_entropy_under(genome_mask, *policy) {
|
||||
builder.set(compact, h as f32);
|
||||
}
|
||||
}
|
||||
for (no_ambiguity, builder) in quartet_builders.iter_mut() {
|
||||
builder.set(compact, is_quartet_informative(genome_mask, *no_ambiguity));
|
||||
for (policy, builder) in quartet_builders.iter_mut() {
|
||||
builder.set(compact, is_quartet_informative(genome_mask, *policy));
|
||||
}
|
||||
for (_, acc) in overlap_accs.iter_mut() {
|
||||
acc.add_family(genome_mask);
|
||||
@@ -175,7 +201,7 @@ pub(crate) fn ensure_layer_family_annexes(
|
||||
}
|
||||
|
||||
/// Builds the pair-weight annexes of one layer that don't exist yet, one per
|
||||
/// `(no_ambiguity, inverse overlap matrix)` in `matrices` (see
|
||||
/// `(policy, inverse overlap matrix)` in `matrices` (see
|
||||
/// [`super::pair_weight::inverse_overlap_matrix`]), in one scan. A no-op (no
|
||||
/// scan) when they all exist. Needs the family-overlap annex of each
|
||||
/// convention to be complete: run after the sweep that builds it.
|
||||
@@ -185,14 +211,14 @@ pub(crate) fn ensure_layer_pair_weight_annexes(
|
||||
layer_idx: usize,
|
||||
n_genomes: usize,
|
||||
fast_mode: bool,
|
||||
matrices: &[(bool, Vec<f32>)],
|
||||
matrices: &[(AmbiguityPolicy, Vec<f32>)],
|
||||
) -> OKIResult<()> {
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
|
||||
let pending: Vec<&(bool, Vec<f32>)> = matrices
|
||||
let pending: Vec<&(AmbiguityPolicy, Vec<f32>)> = matrices
|
||||
.iter()
|
||||
.filter(|(no_ambiguity, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)).exists())
|
||||
.filter(|(policy, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*policy)).exists())
|
||||
.collect();
|
||||
if pending.is_empty() {
|
||||
return Ok(());
|
||||
@@ -202,10 +228,10 @@ pub(crate) fn ensure_layer_pair_weight_annexes(
|
||||
let eligible: HashSet<usize> = reindex.keys().copied().collect();
|
||||
let mut builders = pending
|
||||
.iter()
|
||||
.map(|(no_ambiguity, inverse)| {
|
||||
let path = layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity));
|
||||
FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *no_ambiguity)
|
||||
.map(|b| (*no_ambiguity, inverse, b))
|
||||
.map(|(policy, inverse)| {
|
||||
let path = layer.dir().join(ScalarKind::PairWeight.file_name(*policy));
|
||||
FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *policy)
|
||||
.map(|b| (*policy, inverse, b))
|
||||
})
|
||||
.collect::<io::Result<Vec<_>>>()
|
||||
.map_err(OKIError::Io)?;
|
||||
@@ -219,8 +245,8 @@ pub(crate) fn ensure_layer_pair_weight_annexes(
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, _kmer, _mask, genome_mask| {
|
||||
let compact = reindex[&family_idx];
|
||||
for (no_ambiguity, inverse, builder) in builders.iter_mut() {
|
||||
builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse));
|
||||
for (policy, inverse, builder) in builders.iter_mut() {
|
||||
builder.set(compact, pair_balance_weight(genome_mask, *policy, inverse));
|
||||
}
|
||||
},
|
||||
)?;
|
||||
|
||||
@@ -17,25 +17,22 @@
|
||||
//! waste sample quota on sites that die anyway once the excluded genome's
|
||||
//! data is dropped.
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
use crate::siblings::helpers::is_carrier;
|
||||
|
||||
/// This genome's masked character state, or `None` for "?" (missing). `m`
|
||||
/// is one `genome_mask` entry (bit `b` set iff this genome carries the
|
||||
/// family member whose own canonical central base is `b`; `0` = no member
|
||||
/// present in this genome, `∅` — a real state on its own unless
|
||||
/// `free_loss` says otherwise). `only_diploid` is the milder sibling of
|
||||
/// `no_ambiguity`: it keeps the genomes carrying one or two members (a base
|
||||
/// or a two-base IUPAC code) and drops those carrying three or four.
|
||||
/// `free_loss` says otherwise). `policy` makes missing the genomes carrying
|
||||
/// too many members ([`AmbiguityPolicy::drops`]).
|
||||
#[inline]
|
||||
pub(crate) fn masked_state(m: u8, free_loss: bool, no_ambiguity: bool, only_diploid: bool) -> Option<u8> {
|
||||
pub(crate) fn masked_state(m: u8, free_loss: bool, policy: AmbiguityPolicy) -> Option<u8> {
|
||||
if free_loss && m == 0 {
|
||||
return None; // absence -> missing, not a real state
|
||||
}
|
||||
if no_ambiguity && m.count_ones() > 1 {
|
||||
return None; // ambiguous (>1 member in this genome) -> missing
|
||||
}
|
||||
if only_diploid && m.count_ones() > 2 {
|
||||
return None; // more than a heterozygote's two members -> missing
|
||||
if policy.drops(m) {
|
||||
return None; // ambiguous beyond what the policy keeps -> missing
|
||||
}
|
||||
Some(m)
|
||||
}
|
||||
@@ -90,10 +87,10 @@ pub(crate) fn has_bipartition(counts: &[u32; 16]) -> bool {
|
||||
/// bipartition, never create one — so `false` is final, and `true` is only
|
||||
/// a necessary condition, re-checked exactly by [`survives_quartet`] once
|
||||
/// the family is resolved. This is what the persisted quartet annex stores.
|
||||
pub(crate) fn is_quartet_informative(genome_mask: &[u8], no_ambiguity: bool) -> bool {
|
||||
pub(crate) fn is_quartet_informative(genome_mask: &[u8], policy: AmbiguityPolicy) -> bool {
|
||||
let mut counts = [0u32; 16];
|
||||
for &m in genome_mask {
|
||||
if !is_carrier(m, no_ambiguity) {
|
||||
if !is_carrier(m, policy) {
|
||||
continue;
|
||||
}
|
||||
counts[(m & 0b1111) as usize] += 1;
|
||||
@@ -124,31 +121,31 @@ mod quartet_tests {
|
||||
|
||||
#[test]
|
||||
fn two_pairs_of_distinct_states_is_informative() {
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], false));
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], true));
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], AmbiguityPolicy::Keep));
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], AmbiguityPolicy::Forbid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn singleton_variant_is_not_informative() {
|
||||
// three genomes in one state plus one lone variant: fits any topology
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], false));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], AmbiguityPolicy::Keep));
|
||||
// fewer than four carriers can never be informative
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], false));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], false));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], AmbiguityPolicy::Keep));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], AmbiguityPolicy::Keep));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_genomes_are_not_a_state() {
|
||||
// two carriers of one base, two absences: absence must not form the 2nd pair
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], false));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], AmbiguityPolicy::Keep));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_drops_multi_member_genomes_from_the_count() {
|
||||
// the two-member genomes form a pair of their own state without the flag
|
||||
let masks = [0b0001u8, 0b0001, 0b0011, 0b0011];
|
||||
assert!(is_quartet_informative(&masks, false));
|
||||
assert!(!is_quartet_informative(&masks, true));
|
||||
assert!(is_quartet_informative(&masks, AmbiguityPolicy::Keep));
|
||||
assert!(!is_quartet_informative(&masks, AmbiguityPolicy::Forbid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -227,19 +224,20 @@ mod tests {
|
||||
/// `subsample::sample_layer` does once, up front, before calling
|
||||
/// [`survives_masking`].
|
||||
fn masked(raw: &[u8], free_loss: bool, no_ambiguity: bool) -> Vec<Option<u8>> {
|
||||
raw.iter().map(|&m| masked_state(m, free_loss, no_ambiguity, false)).collect()
|
||||
let policy = AmbiguityPolicy::from_flags(no_ambiguity, false);
|
||||
raw.iter().map(|&m| masked_state(m, free_loss, policy)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_diploid_keeps_up_to_two_members() {
|
||||
let m = |raw: u8| masked_state(raw, false, false, true);
|
||||
let m = |raw: u8| masked_state(raw, false, AmbiguityPolicy::OnlyDiploid);
|
||||
assert_eq!(m(0b0000), Some(0)); // absence stays a state (no --free-loss)
|
||||
assert_eq!(m(0b0100), Some(0b0100));
|
||||
assert_eq!(m(0b0101), Some(0b0101)); // R
|
||||
assert_eq!(m(0b0111), None); // V
|
||||
assert_eq!(m(0b1111), None); // N
|
||||
// the stricter flag wins when both are set
|
||||
assert_eq!(masked_state(0b0101, false, true, true), None);
|
||||
// `Forbid` is stricter: a heterozygote is missing there
|
||||
assert_eq!(masked_state(0b0101, false, AmbiguityPolicy::Forbid), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -11,7 +11,7 @@ use std::path::Path;
|
||||
use obicompactvec::TempBitVecBuilder;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use crate::siblings::{FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex};
|
||||
use crate::siblings::{AmbiguityPolicy, FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex};
|
||||
|
||||
pub(crate) fn is_non_monomorphic_minorant(mask: FamilyMask) -> bool {
|
||||
mask.is_minorant() && mask.siblings() >= 1
|
||||
@@ -92,7 +92,7 @@ pub(crate) fn eligible_bitset_layer(layer_dir: &Path) -> OKIResult<TempBitVecBui
|
||||
/// at each site.
|
||||
///
|
||||
/// Reads the `kind` annex (entropy, pair weight) built under the
|
||||
/// `no_ambiguity` counting convention; it must already exist
|
||||
/// `policy` counting convention; it must already exist
|
||||
/// (`ensure_layer_family_annexes` / `ensure_layer_pair_weight_annexes`
|
||||
/// first) — read positionally (`compact` advances only on non-monomorphic
|
||||
/// minorants, mirroring exactly how the annex was written), never
|
||||
@@ -101,15 +101,15 @@ pub(crate) fn eligible_bitset_layer(layer_dir: &Path) -> OKIResult<TempBitVecBui
|
||||
pub(crate) fn iter_full_scalar(
|
||||
layer_dir: &Path,
|
||||
kind: ScalarKind,
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
) -> OKIResult<impl Iterator<Item = f32> + use<>> {
|
||||
let annex = SiblingAnnex::open_in(layer_dir).map_err(OKIError::Io)?;
|
||||
let scalars =
|
||||
FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(no_ambiguity)), kind).map_err(OKIError::Io)?;
|
||||
if scalars.no_ambiguity() != no_ambiguity {
|
||||
FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(policy)), kind).map_err(OKIError::Io)?;
|
||||
if scalars.policy() != policy {
|
||||
return Err(OKIError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
format!("{kind:?} annex was built under the other --no-ambiguity convention"),
|
||||
format!("{kind:?} annex was built under {:?}, not {policy:?}", scalars.policy()),
|
||||
)));
|
||||
}
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ pub use subsample::{EntropyBias, LayerReport, SamplePlan, SamplingSpec, Survivin
|
||||
|
||||
pub(crate) use annex::build_layer_sibling_annex;
|
||||
pub(crate) use entropy_scan::{EntropySummary, scan_family_entropies};
|
||||
pub use family_annexes::AnnexSelection;
|
||||
pub(crate) use family_annexes::{
|
||||
AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes,
|
||||
};
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
//! Values are only meaningful relative to each other; the sampler rescales
|
||||
//! them per layer (`SamplingSpec::pair_balance`).
|
||||
|
||||
use crate::siblings::FamilyOverlap;
|
||||
use crate::siblings::{AmbiguityPolicy, FamilyOverlap};
|
||||
use crate::siblings::helpers::is_carrier;
|
||||
|
||||
/// `1 / O` for every pair of genomes, as a dense symmetric `n × n` matrix
|
||||
@@ -38,16 +38,16 @@ pub(crate) fn inverse_overlap_matrix(overlap: &FamilyOverlap) -> Vec<f32> {
|
||||
|
||||
/// The pair-balance weight of one family from its raw per-genome masks:
|
||||
/// mean of `inv_overlap` over the pairs of carriers (carriers per
|
||||
/// [`is_carrier`], under `no_ambiguity`). `0.0` for a family with fewer than
|
||||
/// [`is_carrier`], under `policy`). `0.0` for a family with fewer than
|
||||
/// two carriers — it covers no pair, so it can balance none.
|
||||
pub(crate) fn pair_balance_weight(
|
||||
genome_mask: &[u8],
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
inv_overlap: &[f32],
|
||||
) -> f32 {
|
||||
let n = genome_mask.len();
|
||||
let carriers: Vec<usize> =
|
||||
(0..n).filter(|&g| is_carrier(genome_mask[g], no_ambiguity)).collect();
|
||||
(0..n).filter(|&g| is_carrier(genome_mask[g], policy)).collect();
|
||||
let k = carriers.len();
|
||||
if k < 2 {
|
||||
return 0.0;
|
||||
@@ -83,30 +83,30 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn single_pair_weight_is_the_inverse_overlap() {
|
||||
let w = pair_balance_weight(&[1, 1, 0, 0], false, &matrix());
|
||||
let w = pair_balance_weight(&[1, 1, 0, 0], AmbiguityPolicy::Keep, &matrix());
|
||||
assert!((w - 0.5).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn weight_is_the_mean_not_the_sum_over_pairs() {
|
||||
// carriers 0, 1, 2: pairs (0,1)=1/2, (0,2)=1/4, (1,2)=1 -> mean 7/12
|
||||
let w = pair_balance_weight(&[1, 1, 1, 0], false, &matrix());
|
||||
let w = pair_balance_weight(&[1, 1, 1, 0], AmbiguityPolicy::Keep, &matrix());
|
||||
assert!((w - 7.0 / 12.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fewer_than_two_carriers_weigh_nothing() {
|
||||
assert_eq!(pair_balance_weight(&[1, 0, 0, 0], false, &matrix()), 0.0);
|
||||
assert_eq!(pair_balance_weight(&[0, 0, 0, 0], false, &matrix()), 0.0);
|
||||
assert_eq!(pair_balance_weight(&[1, 0, 0, 0], AmbiguityPolicy::Keep, &matrix()), 0.0);
|
||||
assert_eq!(pair_balance_weight(&[0, 0, 0, 0], AmbiguityPolicy::Keep, &matrix()), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_leaves_multi_member_genomes_out_of_the_pairs() {
|
||||
let masks = [1u8, 1, 0b0011, 0];
|
||||
// without the flag genome 2 carries the family: pairs (0,1), (0,2), (1,2)
|
||||
let with = pair_balance_weight(&masks, false, &matrix());
|
||||
let with = pair_balance_weight(&masks, AmbiguityPolicy::Keep, &matrix());
|
||||
// with it, only the pair (0,1) remains
|
||||
let without = pair_balance_weight(&masks, true, &matrix());
|
||||
let without = pair_balance_weight(&masks, AmbiguityPolicy::Forbid, &matrix());
|
||||
assert!((with - 7.0 / 12.0).abs() < 1e-6);
|
||||
assert!((without - 0.5).abs() < 1e-6);
|
||||
}
|
||||
@@ -114,8 +114,8 @@ mod tests {
|
||||
#[test]
|
||||
fn a_rare_pair_outweighs_a_common_one() {
|
||||
// genomes (0,2) share many families, (1,3) few
|
||||
let common = pair_balance_weight(&[1, 0, 1, 0], false, &matrix());
|
||||
let rare = pair_balance_weight(&[0, 1, 0, 1], false, &matrix());
|
||||
let common = pair_balance_weight(&[1, 0, 1, 0], AmbiguityPolicy::Keep, &matrix());
|
||||
let rare = pair_balance_weight(&[0, 1, 0, 1], AmbiguityPolicy::Keep, &matrix());
|
||||
assert!(rare > common);
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,11 @@ use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::minorant_selection::is_non_monomorphic_minorant;
|
||||
use crate::siblings::quartet_annex::quartet_annex_file_name;
|
||||
use crate::siblings::SiblingAnnex;
|
||||
use crate::siblings::{AmbiguityPolicy, SiblingAnnex};
|
||||
|
||||
/// Clears, in `eligible` (indexed by `family_idx`, monomorphic minorants
|
||||
/// included), every family the quartet annex flags as not quartet-informative
|
||||
/// under the `no_ambiguity` counting convention. The annex is indexed by the
|
||||
/// under the `policy` counting convention. The annex is indexed by the
|
||||
/// compact non-monomorphic ordinal, so the sibling annex is walked in the same
|
||||
/// order to translate one numbering into the other — annex-only, no
|
||||
/// cross-partition resolution. The annex must already exist
|
||||
@@ -20,10 +20,10 @@ use crate::siblings::SiblingAnnex;
|
||||
pub(crate) fn restrict_to_quartet(
|
||||
eligible: &mut TempBitVecBuilder,
|
||||
layer_dir: &Path,
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
) -> OKIResult<()> {
|
||||
let annex = SiblingAnnex::open_in(layer_dir).map_err(OKIError::Io)?;
|
||||
let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(no_ambiguity)))
|
||||
let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(policy)))
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
let mut family_idx = 0usize;
|
||||
|
||||
@@ -279,7 +279,7 @@ mod tests {
|
||||
/// something these tests need to exercise).
|
||||
fn tally_2genomes(pairs: &[(u8, u8)], n_shared: usize) -> PairwiseTally {
|
||||
// No masking requested (`free_loss = no_ambiguity = false`) —
|
||||
// `masked_state(m, false, false, false)` is always `Some(m)`, so the raw
|
||||
// `masked_state(m, false, AmbiguityPolicy::Keep)` is always `Some(m)`, so the raw
|
||||
// mask is exactly what `subsample::sample_layer` would have
|
||||
// computed here.
|
||||
let mut families = Vec::new();
|
||||
|
||||
@@ -57,7 +57,7 @@ use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::masking::{masked_state, survives_masking, survives_quartet};
|
||||
use super::minorant_selection::{eligible_bitset_layer, iter_full_scalar};
|
||||
use super::{AnnexRequest, is_fast_mode, restrict_to_quartet};
|
||||
use crate::siblings::{FamilyMask, ScalarKind};
|
||||
use crate::siblings::{AmbiguityPolicy, FamilyMask, ScalarKind};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
|
||||
/// Resolution rounds a single layer's sampling may take before giving up on
|
||||
@@ -109,13 +109,10 @@ pub struct EntropyBias {
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct SamplingSpec {
|
||||
pub free_loss: bool,
|
||||
pub no_ambiguity: bool,
|
||||
/// `--only-diploid`: like `no_ambiguity` but only genomes carrying three
|
||||
/// or four members of a family become missing. Applied by the post-hoc
|
||||
/// masking only — the persisted entropy/quartet/pair-weight annexes are
|
||||
/// read under the `no_ambiguity` convention (`false` here), which is a
|
||||
/// superset for the quartet pre-filter and only a weight otherwise.
|
||||
pub only_diploid: bool,
|
||||
/// How genomes carrying several members of a family are treated
|
||||
/// (`--no-ambiguity` / `--only-diploid`): by the post-hoc masking, and as
|
||||
/// the counting convention of the persisted annexes the draw reads.
|
||||
pub ambiguity: AmbiguityPolicy,
|
||||
pub quartet: bool,
|
||||
/// `--pair-balance`: weight each family by the mean, over the pairs of
|
||||
/// genomes carrying it, of 1 / (number of families the pair shares), so
|
||||
@@ -165,17 +162,17 @@ impl SamplePlan {
|
||||
/// [`sample_layer`]'s own circular index sequence walks, so the streams can be
|
||||
/// `.zip()`-ed directly. Two more streamed passes over the layer's (small)
|
||||
/// annex files, not a materialised `Vec`/`HashMap` — `iter_full_scalar`
|
||||
/// itself never allocates more than its own iterator state. `no_ambiguity`
|
||||
/// itself never allocates more than its own iterator state. `policy`
|
||||
/// selects the annex built under the same counting convention as the masking
|
||||
/// the survivors go through afterwards.
|
||||
fn circular_scalar_values(
|
||||
layer_dir: &Path,
|
||||
kind: ScalarKind,
|
||||
start: usize,
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
) -> OKIResult<impl Iterator<Item = f32> + use<>> {
|
||||
let head = iter_full_scalar(layer_dir, kind, no_ambiguity)?.skip(start);
|
||||
let tail = iter_full_scalar(layer_dir, kind, no_ambiguity)?.take(start);
|
||||
let head = iter_full_scalar(layer_dir, kind, policy)?.skip(start);
|
||||
let tail = iter_full_scalar(layer_dir, kind, policy)?.take(start);
|
||||
Ok(head.chain(tail))
|
||||
}
|
||||
|
||||
@@ -201,7 +198,7 @@ struct LayerPlan {
|
||||
/// per layer, calling `on_layer(partition, layer, survivors)` once per
|
||||
/// cached layer that contributed anything, with every family from that
|
||||
/// layer that was both drawn *and* still carries real polymorphism after
|
||||
/// `free_loss`/`no_ambiguity`/`excluded` masking
|
||||
/// `free_loss`/`ambiguity`/`excluded` masking
|
||||
/// (`algorithms::masking::survives_masking`) — i.e. every
|
||||
/// [`SurvivingFamily`] this produces is already a usable alignment site, no
|
||||
/// second resolution pass needed by the caller. `excluded[g]` (see
|
||||
@@ -262,10 +259,10 @@ pub(crate) fn sample_index(
|
||||
// layer, below, each under this run's `--no-ambiguity` counting
|
||||
// convention) — build any missing one now, once, rather than mid-walk.
|
||||
let request = AnnexRequest {
|
||||
entropy: spec.entropy_bias.map(|_| vec![spec.no_ambiguity]).unwrap_or_default(),
|
||||
quartet: if spec.quartet { vec![spec.no_ambiguity] } else { Vec::new() },
|
||||
entropy: spec.entropy_bias.map(|_| vec![spec.ambiguity]).unwrap_or_default(),
|
||||
quartet: if spec.quartet { vec![spec.ambiguity] } else { Vec::new() },
|
||||
overlap: Vec::new(),
|
||||
pair_weight: if spec.pair_balance { vec![spec.no_ambiguity] } else { Vec::new() },
|
||||
pair_weight: if spec.pair_balance { vec![spec.ambiguity] } else { Vec::new() },
|
||||
};
|
||||
if !request.entropy.is_empty() || !request.quartet.is_empty() || !request.pair_weight.is_empty() {
|
||||
cache.ensure_family_annexes(&request)?;
|
||||
@@ -285,12 +282,12 @@ pub(crate) fn sample_index(
|
||||
.expect("cache-consistent: n_layer(partition) already bounds layer_idx");
|
||||
let mut eligible = eligible_bitset_layer(layer.dir())?;
|
||||
if spec.quartet {
|
||||
restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?;
|
||||
restrict_to_quartet(&mut eligible, layer.dir(), spec.ambiguity)?;
|
||||
}
|
||||
let n_minorants = eligible.view().len();
|
||||
let eligible_count = eligible.view().count_ones();
|
||||
let mass = if spec.pair_balance {
|
||||
iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)?
|
||||
iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.ambiguity)?
|
||||
.enumerate()
|
||||
.filter(|&(family_idx, _)| eligible.view().get(family_idx))
|
||||
.map(|(_, weight)| f64::from(weight))
|
||||
@@ -435,7 +432,7 @@ fn sample_layer(
|
||||
&layer_dir,
|
||||
ScalarKind::Entropy,
|
||||
round_start,
|
||||
spec.no_ambiguity,
|
||||
spec.ambiguity,
|
||||
)?),
|
||||
None => Box::new(std::iter::repeat(0.0f32)), // never read: kernel forced to 1.0 below
|
||||
};
|
||||
@@ -444,7 +441,7 @@ fn sample_layer(
|
||||
&layer_dir,
|
||||
ScalarKind::PairWeight,
|
||||
round_start,
|
||||
spec.no_ambiguity,
|
||||
spec.ambiguity,
|
||||
)?)
|
||||
} else {
|
||||
Box::new(std::iter::repeat(1.0f32))
|
||||
@@ -494,7 +491,7 @@ fn sample_layer(
|
||||
);
|
||||
let masked: Vec<Option<u8>> = genome_mask
|
||||
.iter()
|
||||
.map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity, spec.only_diploid))
|
||||
.map(|&m| masked_state(m, spec.free_loss, spec.ambiguity))
|
||||
.collect();
|
||||
if survives_masking(&masked, excluded)
|
||||
&& (!spec.quartet || survives_quartet(&masked, excluded))
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
//! How a genome carrying several members of a family is treated — the one
|
||||
//! parameter behind `--no-ambiguity` / `--only-diploid`, and the key of every
|
||||
//! persisted annex whose values depend on which genomes count as carriers
|
||||
//! (entropy, quartet, pair weight, family overlap).
|
||||
|
||||
/// What becomes of a genome carrying more than one member of a family.
|
||||
///
|
||||
/// - [`Keep`](Self::Keep): nothing — its set of members is a state of its
|
||||
/// own (an IUPAC ambiguity code in the alignment).
|
||||
/// - [`OnlyDiploid`](Self::OnlyDiploid): kept up to two members (a
|
||||
/// heterozygote); three or four members make it missing data.
|
||||
/// - [`Forbid`](Self::Forbid) (`--no-ambiguity`): any genome with several
|
||||
/// members is missing data.
|
||||
///
|
||||
/// The order is the on-disk header code (see [`header_flags`](Self::header_flags)),
|
||||
/// kept so that annexes written when this was a `bool` stay valid.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum AmbiguityPolicy {
|
||||
#[default]
|
||||
Keep,
|
||||
Forbid,
|
||||
OnlyDiploid,
|
||||
}
|
||||
|
||||
impl AmbiguityPolicy {
|
||||
/// Every policy, in header-code order.
|
||||
pub const ALL: [AmbiguityPolicy; 3] = [Self::Keep, Self::Forbid, Self::OnlyDiploid];
|
||||
|
||||
/// From the two (mutually exclusive) CLI flags.
|
||||
pub fn from_flags(no_ambiguity: bool, only_diploid: bool) -> Self {
|
||||
debug_assert!(!(no_ambiguity && only_diploid), "--no-ambiguity and --only-diploid are exclusive");
|
||||
if no_ambiguity {
|
||||
Self::Forbid
|
||||
} else if only_diploid {
|
||||
Self::OnlyDiploid
|
||||
} else {
|
||||
Self::Keep
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a genome whose family presence mask is `mask` becomes missing
|
||||
/// data under this policy.
|
||||
#[inline]
|
||||
pub fn drops(self, mask: u8) -> bool {
|
||||
match self {
|
||||
Self::Keep => false,
|
||||
Self::OnlyDiploid => mask.count_ones() > 2,
|
||||
Self::Forbid => mask.count_ones() > 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `flags` value persisted in an annex header: `0` for `Keep` and `1`
|
||||
/// for `Forbid` are what the `no_ambiguity` bit used to be.
|
||||
pub(crate) fn header_flags(self) -> u32 {
|
||||
match self {
|
||||
Self::Keep => 0,
|
||||
Self::Forbid => 1,
|
||||
Self::OnlyDiploid => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Inverse of [`header_flags`](Self::header_flags); unknown codes read as
|
||||
/// `Keep`, as a file from before the flag existed.
|
||||
pub(crate) fn from_header_flags(flags: u32) -> Self {
|
||||
match flags {
|
||||
1 => Self::Forbid,
|
||||
2 => Self::OnlyDiploid,
|
||||
_ => Self::Keep,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn drops_follows_the_member_count() {
|
||||
for (mask, keep, diploid, forbid) in
|
||||
[(0b0000u8, false, false, false), (0b0100, false, false, false), (0b0101, false, false, true), (0b0111, false, true, true), (0b1111, false, true, true)]
|
||||
{
|
||||
assert_eq!(AmbiguityPolicy::Keep.drops(mask), keep);
|
||||
assert_eq!(AmbiguityPolicy::OnlyDiploid.drops(mask), diploid);
|
||||
assert_eq!(AmbiguityPolicy::Forbid.drops(mask), forbid);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_flags_roundtrip_and_stay_compatible_with_the_old_bool() {
|
||||
for p in AmbiguityPolicy::ALL {
|
||||
assert_eq!(AmbiguityPolicy::from_header_flags(p.header_flags()), p);
|
||||
}
|
||||
assert_eq!(AmbiguityPolicy::Keep.header_flags(), 0);
|
||||
assert_eq!(AmbiguityPolicy::Forbid.header_flags(), 1);
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ use crate::siblings::algorithms::{
|
||||
ensure_layer_pair_weight_annexes, inverse_overlap_matrix, is_fast_mode,
|
||||
};
|
||||
use crate::siblings::{
|
||||
ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, FamilyOverlapAccumulator,
|
||||
ANNEX_FILE_NAME, AmbiguityPolicy, FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, FamilyOverlapAccumulator,
|
||||
QUARTET_ANNEX_FILE_NAMES, ScalarKind, family_overlap_file_name,
|
||||
};
|
||||
|
||||
@@ -154,14 +154,14 @@ impl SiblingBuilder for IndexCache {
|
||||
fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> {
|
||||
let n_genomes = self.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(self);
|
||||
let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity));
|
||||
let overlap_path = |policy: AmbiguityPolicy| self.index().dir().join(family_overlap_file_name(policy));
|
||||
|
||||
// Sweep 1: entropy, quartet, and every family-overlap annex still missing.
|
||||
let mut overlap_accs: Vec<(bool, FamilyOverlapAccumulator)> = request
|
||||
let mut overlap_accs: Vec<(AmbiguityPolicy, FamilyOverlapAccumulator)> = request
|
||||
.overlap_conventions()
|
||||
.into_iter()
|
||||
.filter(|&no_ambiguity| !overlap_path(no_ambiguity).exists())
|
||||
.map(|no_ambiguity| (no_ambiguity, FamilyOverlapAccumulator::new(n_genomes, no_ambiguity)))
|
||||
.filter(|&policy| !overlap_path(policy).exists())
|
||||
.map(|policy| (policy, FamilyOverlapAccumulator::new(n_genomes, policy)))
|
||||
.collect();
|
||||
let pb = progress_bar("family_annexes", self.n_partition() as u64, "partitions");
|
||||
for part in self.partitions() {
|
||||
@@ -172,8 +172,8 @@ impl SiblingBuilder for IndexCache {
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
for (no_ambiguity, acc) in overlap_accs {
|
||||
acc.write(&overlap_path(no_ambiguity)).map_err(OKIError::Io)?;
|
||||
for (policy, acc) in overlap_accs {
|
||||
acc.write(&overlap_path(policy)).map_err(OKIError::Io)?;
|
||||
}
|
||||
|
||||
// Sweep 2: pair weights, from the now-complete overlap annexes.
|
||||
@@ -183,9 +183,9 @@ impl SiblingBuilder for IndexCache {
|
||||
let matrices = request
|
||||
.pair_weight_conventions()
|
||||
.into_iter()
|
||||
.map(|no_ambiguity| {
|
||||
FamilyOverlap::open(&overlap_path(no_ambiguity))
|
||||
.map(|overlap| (no_ambiguity, inverse_overlap_matrix(&overlap)))
|
||||
.map(|policy| {
|
||||
FamilyOverlap::open(&overlap_path(policy))
|
||||
.map(|overlap| (policy, inverse_overlap_matrix(&overlap)))
|
||||
.map_err(OKIError::Io)
|
||||
})
|
||||
.collect::<OKIResult<Vec<_>>>()?;
|
||||
|
||||
@@ -13,10 +13,10 @@ use obikidxcache::index_cache::IndexCache;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use crate::siblings::algorithms::{
|
||||
AnnexRequest, EntropySummary, SiblingAnnexStats, scan_family_entropies, sibling_annex_stats, sibling_family_size_histogram,
|
||||
AnnexRequest, AnnexSelection, EntropySummary, SiblingAnnexStats, scan_family_entropies, sibling_annex_stats, sibling_family_size_histogram,
|
||||
};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
use crate::siblings::{FamilyOverlap, family_overlap_file_name};
|
||||
use crate::siblings::{AmbiguityPolicy, FamilyOverlap, family_overlap_file_name};
|
||||
|
||||
pub trait SiblingExt {
|
||||
/// Build the sibling-count/minorant annex for every layer of every
|
||||
@@ -39,7 +39,14 @@ pub trait SiblingExt {
|
||||
/// Cross-partition/cross-layer lookups are required (a k-mer's siblings
|
||||
/// can live in any partition), but the layer loop itself — and thus the
|
||||
/// annex file this produces — stays local to one layer at a time.
|
||||
fn build_sibling_annex(&self) -> OKIResult<()>;
|
||||
///
|
||||
/// `derived` names the annexes derived from the sibling annex (entropy,
|
||||
/// quartet, pair weight, family overlap) to have ready afterwards, all
|
||||
/// under its one ambiguity policy — only those, so a routine rebuild
|
||||
/// pays for nothing the following computations will not read. Derived
|
||||
/// annexes that already existed are invalidated by the rebuild and are
|
||||
/// rebuilt on demand, by the computation that reads them, if not listed.
|
||||
fn build_sibling_annex(&self, derived: &AnnexSelection) -> OKIResult<()>;
|
||||
|
||||
/// Write a CSV (`partition,layer,family_idx,entropy15,entropy4,
|
||||
/// family_size,n_genomes_present,n_genomes_unambiguous`) of per-family
|
||||
@@ -91,9 +98,9 @@ pub trait SiblingExt {
|
||||
|
||||
/// The Family Overlap annex — number of shared *variable* families per
|
||||
/// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs),
|
||||
/// under the `--no-ambiguity` counting convention `no_ambiguity` selects
|
||||
/// (genomes carrying several members of a family do not count as
|
||||
/// carrying it when `true`). Built if missing, same as every other
|
||||
/// under the counting convention `policy` selects (which genomes
|
||||
/// carrying several members of a family still count as carrying it, see
|
||||
/// [`AmbiguityPolicy`]). Built if missing, same as every other
|
||||
/// annex-backed method here. Feeds `--family-overlap`'s own matrix
|
||||
/// output directly, and `--min-shared-family`'s auto-exclusion
|
||||
/// (`FamilyOverlap::mean_row(g) < threshold` — same statistic
|
||||
@@ -101,22 +108,22 @@ pub trait SiblingExt {
|
||||
/// triggers adding `g` to the excluded-genome set consumed by
|
||||
/// `algorithms::scan::sampled_scan`'s callers; this needs no
|
||||
/// computation of its own beyond that check).
|
||||
fn family_overlap(&self, no_ambiguity: bool) -> OKIResult<FamilyOverlap>;
|
||||
fn family_overlap(&self, policy: AmbiguityPolicy) -> OKIResult<FamilyOverlap>;
|
||||
}
|
||||
|
||||
impl SiblingExt for IndexCache {
|
||||
fn build_sibling_annex(&self) -> OKIResult<()> {
|
||||
fn build_sibling_annex(&self, derived: &AnnexSelection) -> OKIResult<()> {
|
||||
self.index().with_lock(|| {
|
||||
self.build_base_sibling_annex()?;
|
||||
// Precompute the entropy, quartet, pair-weight and family-overlap
|
||||
// annexes too, now that every layer's fresh sibling annex is on
|
||||
// disk (`non_monomorphic_reindex_layer`, which they depend on,
|
||||
// reads it) — same routine-rebuild footing as the sibling annex
|
||||
// itself, rather than leaving them to be built lazily on first
|
||||
// `--shannon`/`--entropy`/`--quartet`/`--pair-balance`/
|
||||
// `--min-shared-family`/`--family-overlap` use.
|
||||
tracing::info!("building entropy, quartet, pair-weight (both --no-ambiguity conventions) and family-overlap annexes");
|
||||
self.ensure_family_annexes(&AnnexRequest::everything())
|
||||
// Then exactly the derived annexes asked for, now that every
|
||||
// layer's fresh sibling annex is on disk
|
||||
// (`non_monomorphic_reindex_layer`, which they depend on, reads
|
||||
// it).
|
||||
if derived.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!("building the requested derived annexes ({:?})", derived.policy);
|
||||
self.ensure_family_annexes(&derived.request())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -160,9 +167,9 @@ impl SiblingExt for IndexCache {
|
||||
sibling_annex_stats(self)
|
||||
}
|
||||
|
||||
fn family_overlap(&self, no_ambiguity: bool) -> OKIResult<FamilyOverlap> {
|
||||
self.ensure_family_annexes(&AnnexRequest { overlap: vec![no_ambiguity], ..Default::default() })?;
|
||||
let path = self.index().dir().join(family_overlap_file_name(no_ambiguity));
|
||||
fn family_overlap(&self, policy: AmbiguityPolicy) -> OKIResult<FamilyOverlap> {
|
||||
self.ensure_family_annexes(&AnnexRequest { overlap: vec![policy], ..Default::default() })?;
|
||||
let path = self.index().dir().join(family_overlap_file_name(policy));
|
||||
FamilyOverlap::open(&path).map_err(OKIError::Io)
|
||||
}
|
||||
}
|
||||
@@ -25,35 +25,41 @@ use std::path::Path;
|
||||
use memmap2::Mmap;
|
||||
use obicompactvec::{building_path, publish_built};
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
use crate::siblings::helpers::{is_carrier, triangle_index};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"POVL";
|
||||
|
||||
// Header: magic(4) + flags(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair,
|
||||
// upper triangle) follows. `flags` bit 0 = `no_ambiguity` (see
|
||||
// upper triangle) follows. `flags` = `AmbiguityPolicy::header_flags` (see
|
||||
// `FamilyOverlapAccumulator::new`); older files have `flags == 0`, i.e.
|
||||
// `no_ambiguity == false`, which is what they were computed under.
|
||||
// `Keep`, which is what they were computed under.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
const FLAG_NO_AMBIGUITY: u32 = 1;
|
||||
|
||||
pub(crate) const FAMILY_OVERLAP_FILE_NAME: &str = "family_overlap.povl";
|
||||
|
||||
/// Same annex, built with `no_ambiguity == true`.
|
||||
/// Same annex, built under [`AmbiguityPolicy::Forbid`].
|
||||
pub(crate) const FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME: &str = "family_overlap.noamb.povl";
|
||||
|
||||
/// Same annex, built under [`AmbiguityPolicy::OnlyDiploid`].
|
||||
pub(crate) const FAMILY_OVERLAP_ONLY_DIPLOID_FILE_NAME: &str = "family_overlap.diploid.povl";
|
||||
|
||||
/// Every family-overlap file name, whatever its counting convention — for
|
||||
/// invalidating them all together when the sibling annex they derive from is
|
||||
/// rebuilt.
|
||||
pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 2] =
|
||||
[FAMILY_OVERLAP_FILE_NAME, FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME];
|
||||
pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 3] = [
|
||||
FAMILY_OVERLAP_FILE_NAME,
|
||||
FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME,
|
||||
FAMILY_OVERLAP_ONLY_DIPLOID_FILE_NAME,
|
||||
];
|
||||
|
||||
/// File name of the family-overlap annex built under the given
|
||||
/// `--no-ambiguity` counting convention. The two never share a file.
|
||||
pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str {
|
||||
if no_ambiguity {
|
||||
FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME
|
||||
} else {
|
||||
FAMILY_OVERLAP_FILE_NAME
|
||||
/// File name of the family-overlap annex built under the given ambiguity
|
||||
/// policy. The policies never share a file.
|
||||
pub(crate) fn family_overlap_file_name(policy: AmbiguityPolicy) -> &'static str {
|
||||
match policy {
|
||||
AmbiguityPolicy::Keep => FAMILY_OVERLAP_FILE_NAME,
|
||||
AmbiguityPolicy::Forbid => FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME,
|
||||
AmbiguityPolicy::OnlyDiploid => FAMILY_OVERLAP_ONLY_DIPLOID_FILE_NAME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +67,7 @@ pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str {
|
||||
pub struct FamilyOverlap {
|
||||
mmap: Mmap,
|
||||
n_genomes: usize,
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
}
|
||||
|
||||
impl FamilyOverlap {
|
||||
@@ -79,19 +85,17 @@ impl FamilyOverlap {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "POVL file truncated"));
|
||||
}
|
||||
let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap());
|
||||
Ok(Self { mmap, n_genomes, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 })
|
||||
Ok(Self { mmap, n_genomes, policy: AmbiguityPolicy::from_header_flags(flags) })
|
||||
}
|
||||
|
||||
pub fn n_genomes(&self) -> usize {
|
||||
self.n_genomes
|
||||
}
|
||||
|
||||
/// The counting convention these counts were computed under: `true`
|
||||
/// when a genome carrying several members of a family was *not* counted
|
||||
/// as carrying it (`--no-ambiguity`), `false` when any non-empty
|
||||
/// state counts as carrying it.
|
||||
pub fn no_ambiguity(&self) -> bool {
|
||||
self.no_ambiguity
|
||||
/// The counting convention these counts were computed under (see
|
||||
/// [`AmbiguityPolicy`]): which genomes count as carrying a family.
|
||||
pub fn policy(&self) -> AmbiguityPolicy {
|
||||
self.policy
|
||||
}
|
||||
|
||||
/// Number of variable families shared by genomes `i` and `j` — `0` for
|
||||
@@ -128,18 +132,17 @@ impl FamilyOverlap {
|
||||
/// `EntropyAnnexBuilder` do for their own, much larger, per-k-mer data.
|
||||
pub(crate) struct FamilyOverlapAccumulator {
|
||||
n_genomes: usize,
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
counts: Vec<u64>,
|
||||
}
|
||||
|
||||
impl FamilyOverlapAccumulator {
|
||||
/// `no_ambiguity` is the counting convention: `false` counts every
|
||||
/// genome with a non-empty state as carrying the family, `true` only the
|
||||
/// genomes carrying exactly one member of it — the others are missing
|
||||
/// data in the alignment (`--no-ambiguity`), so they share nothing there.
|
||||
pub(crate) fn new(n_genomes: usize, no_ambiguity: bool) -> Self {
|
||||
/// `policy` is the counting convention: a genome counts as carrying the
|
||||
/// family unless the policy makes it missing data in the alignment, in
|
||||
/// which case it shares nothing there (see [`AmbiguityPolicy::drops`]).
|
||||
pub(crate) fn new(n_genomes: usize, policy: AmbiguityPolicy) -> Self {
|
||||
let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2;
|
||||
Self { n_genomes, no_ambiguity, counts: vec![0u64; n_pairs] }
|
||||
Self { n_genomes, policy, counts: vec![0u64; n_pairs] }
|
||||
}
|
||||
|
||||
/// One variable family's contribution: every pair of genomes both
|
||||
@@ -147,11 +150,11 @@ impl FamilyOverlapAccumulator {
|
||||
/// once.
|
||||
pub(crate) fn add_family(&mut self, genome_mask: &[u8]) {
|
||||
for i in 0..self.n_genomes {
|
||||
if !is_carrier(genome_mask[i], self.no_ambiguity) {
|
||||
if !is_carrier(genome_mask[i], self.policy) {
|
||||
continue;
|
||||
}
|
||||
for j in (i + 1)..self.n_genomes {
|
||||
if is_carrier(genome_mask[j], self.no_ambiguity) {
|
||||
if is_carrier(genome_mask[j], self.policy) {
|
||||
self.counts[triangle_index(self.n_genomes, i, j)] += 1;
|
||||
}
|
||||
}
|
||||
@@ -161,7 +164,7 @@ impl FamilyOverlapAccumulator {
|
||||
pub(crate) fn write(&self, path: &Path) -> io::Result<()> {
|
||||
let mut buf = Vec::with_capacity(HEADER_SIZE + self.counts.len() * 8);
|
||||
buf.extend_from_slice(&MAGIC);
|
||||
let flags = if self.no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 };
|
||||
let flags = self.policy.header_flags();
|
||||
buf.extend_from_slice(&flags.to_le_bytes());
|
||||
buf.extend_from_slice(&(self.n_genomes as u64).to_le_bytes());
|
||||
for &c in &self.counts {
|
||||
@@ -181,7 +184,7 @@ mod tests {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("test.povl");
|
||||
|
||||
let mut acc = FamilyOverlapAccumulator::new(3, false);
|
||||
let mut acc = FamilyOverlapAccumulator::new(3, AmbiguityPolicy::Keep);
|
||||
acc.add_family(&[1, 1, 0]); // genomes 0,1 share this family
|
||||
acc.add_family(&[1, 1, 1]); // all three share this one
|
||||
acc.add_family(&[0, 1, 1]); // genomes 1,2 share this family
|
||||
@@ -201,26 +204,31 @@ mod tests {
|
||||
fn single_genome_mean_row_is_zero() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("test.povl");
|
||||
FamilyOverlapAccumulator::new(1, false).write(&path).unwrap();
|
||||
FamilyOverlapAccumulator::new(1, AmbiguityPolicy::Keep).write(&path).unwrap();
|
||||
let overlap = FamilyOverlap::open(&path).unwrap();
|
||||
assert_eq!(overlap.mean_row(0), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_does_not_count_multi_member_genomes_as_carriers() {
|
||||
fn the_policy_decides_which_multi_member_genomes_are_carriers() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0100]];
|
||||
for no_ambiguity in [false, true] {
|
||||
let path = dir.path().join(family_overlap_file_name(no_ambiguity));
|
||||
let mut acc = FamilyOverlapAccumulator::new(3, no_ambiguity);
|
||||
// genome 2: two members in the first family, three in the second
|
||||
let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0111]];
|
||||
for policy in AmbiguityPolicy::ALL {
|
||||
let path = dir.path().join(family_overlap_file_name(policy));
|
||||
let mut acc = FamilyOverlapAccumulator::new(3, policy);
|
||||
for m in masks {
|
||||
acc.add_family(m);
|
||||
}
|
||||
acc.write(&path).unwrap();
|
||||
let overlap = FamilyOverlap::open(&path).unwrap();
|
||||
assert_eq!(overlap.no_ambiguity(), no_ambiguity);
|
||||
// genome 2 carries two members in the first family: only counted without the flag
|
||||
assert_eq!(overlap.get(0, 2), if no_ambiguity { 1 } else { 2 });
|
||||
assert_eq!(overlap.policy(), policy);
|
||||
let shared_with_2 = match policy {
|
||||
AmbiguityPolicy::Keep => 2,
|
||||
AmbiguityPolicy::OnlyDiploid => 1,
|
||||
AmbiguityPolicy::Forbid => 0,
|
||||
};
|
||||
assert_eq!(overlap.get(0, 2), shared_with_2);
|
||||
assert_eq!(overlap.get(0, 1), 1 + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,12 +25,13 @@ use std::path::{Path, PathBuf};
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
use obicompactvec::{building_path, publish_built};
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
|
||||
// Header: magic(4) + flags(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
|
||||
// `flags` bit 0 = `no_ambiguity` (the counting convention the values were
|
||||
// computed under, see `FamilyScalarAnnexBuilder::new`); every file written
|
||||
// before the flag existed has `flags == 0`, i.e. `no_ambiguity == false`,
|
||||
// which is what it was computed under.
|
||||
const FLAG_NO_AMBIGUITY: u32 = 1;
|
||||
// `flags` = `AmbiguityPolicy::header_flags` (the counting convention the
|
||||
// values were computed under, see `FamilyScalarAnnexBuilder::new`); every file
|
||||
// written before the flag existed has `flags == 0`, i.e. `Keep`, which is what
|
||||
// it was computed under.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// Sentinel: not yet computed. Every stored value is `>= 0.0`, so any
|
||||
@@ -56,31 +57,32 @@ impl ScalarKind {
|
||||
}
|
||||
}
|
||||
|
||||
/// File name of this kind's annex built under the given `--no-ambiguity`
|
||||
/// counting convention. The two conventions never share a file:
|
||||
/// switching between them builds the missing annex without discarding the
|
||||
/// other.
|
||||
pub(crate) fn file_name(self, no_ambiguity: bool) -> &'static str {
|
||||
match (self, no_ambiguity) {
|
||||
(ScalarKind::Entropy, false) => "entropy.pent",
|
||||
(ScalarKind::Entropy, true) => "entropy.noamb.pent",
|
||||
(ScalarKind::PairWeight, false) => "pairweight.ppwt",
|
||||
(ScalarKind::PairWeight, true) => "pairweight.noamb.ppwt",
|
||||
/// File name of this kind's annex built under the given ambiguity
|
||||
/// policy. The policies never share a file: switching between them builds
|
||||
/// the missing annex without discarding the others.
|
||||
pub(crate) fn file_name(self, policy: AmbiguityPolicy) -> &'static str {
|
||||
match (self, policy) {
|
||||
(ScalarKind::Entropy, AmbiguityPolicy::Keep) => "entropy.pent",
|
||||
(ScalarKind::Entropy, AmbiguityPolicy::Forbid) => "entropy.noamb.pent",
|
||||
(ScalarKind::Entropy, AmbiguityPolicy::OnlyDiploid) => "entropy.diploid.pent",
|
||||
(ScalarKind::PairWeight, AmbiguityPolicy::Keep) => "pairweight.ppwt",
|
||||
(ScalarKind::PairWeight, AmbiguityPolicy::Forbid) => "pairweight.noamb.ppwt",
|
||||
(ScalarKind::PairWeight, AmbiguityPolicy::OnlyDiploid) => "pairweight.diploid.ppwt",
|
||||
}
|
||||
}
|
||||
|
||||
/// Every file name of this kind, whatever its counting convention — for
|
||||
/// invalidating them all together (e.g. when the sibling annex they were
|
||||
/// numbered against is rebuilt).
|
||||
pub(crate) fn file_names(self) -> [&'static str; 2] {
|
||||
[self.file_name(false), self.file_name(true)]
|
||||
pub(crate) fn file_names(self) -> [&'static str; 3] {
|
||||
AmbiguityPolicy::ALL.map(|p| self.file_name(p))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FamilyScalarAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
no_ambiguity: bool,
|
||||
policy: AmbiguityPolicy,
|
||||
}
|
||||
|
||||
impl FamilyScalarAnnex {
|
||||
@@ -100,19 +102,16 @@ impl FamilyScalarAnnex {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file truncated"));
|
||||
}
|
||||
let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap());
|
||||
Ok(Self { mmap, n, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 })
|
||||
Ok(Self { mmap, n, policy: AmbiguityPolicy::from_header_flags(flags) })
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.n
|
||||
}
|
||||
|
||||
/// The counting convention these values were computed under: `true`
|
||||
/// when genomes carrying several members of a family were left out of
|
||||
/// the count (`--no-ambiguity`), `false` when they were counted as a
|
||||
/// state of their own.
|
||||
pub fn no_ambiguity(&self) -> bool {
|
||||
self.no_ambiguity
|
||||
/// The counting convention these values were computed under.
|
||||
pub fn policy(&self) -> AmbiguityPolicy {
|
||||
self.policy
|
||||
}
|
||||
|
||||
/// `None` for the sentinel (not yet computed — shouldn't happen against
|
||||
@@ -132,9 +131,9 @@ pub struct FamilyScalarAnnexBuilder {
|
||||
impl FamilyScalarAnnexBuilder {
|
||||
/// Create a new annex of `n` entries (a layer's non-monomorphic
|
||||
/// minorant count) at `path`, pre-initialised to the sentinel.
|
||||
/// `no_ambiguity` records, in the header, the counting convention the
|
||||
/// values will be computed under (see [`FamilyScalarAnnex::no_ambiguity`]).
|
||||
pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, no_ambiguity: bool) -> io::Result<Self> {
|
||||
/// `policy` records, in the header, the counting convention the values
|
||||
/// will be computed under (see [`FamilyScalarAnnex::policy`]).
|
||||
pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, policy: AmbiguityPolicy) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n * 4;
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
@@ -145,7 +144,7 @@ impl FamilyScalarAnnexBuilder {
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&kind.magic());
|
||||
let flags = if no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 };
|
||||
let flags = policy.header_flags();
|
||||
mmap[4..8].copy_from_slice(&flags.to_le_bytes());
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
for i in 0..n {
|
||||
@@ -181,7 +180,7 @@ mod tests {
|
||||
fn sentinel_is_negative_and_unset_entries_read_as_none() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap().close().unwrap();
|
||||
FamilyScalarAnnexBuilder::new(4, &path, E, AmbiguityPolicy::Keep).unwrap().close().unwrap();
|
||||
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
|
||||
for i in 0..4 {
|
||||
assert_eq!(annex.get(i), None);
|
||||
@@ -192,7 +191,7 @@ mod tests {
|
||||
fn roundtrip_including_zero_value() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap();
|
||||
let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, AmbiguityPolicy::Keep).unwrap();
|
||||
let values = [0.0f32, 1.5, 3.9, 0.0001];
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
builder.set(i, v);
|
||||
@@ -208,7 +207,7 @@ mod tests {
|
||||
fn mixed_sentinel_and_real_values() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, false).unwrap();
|
||||
let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, AmbiguityPolicy::Keep).unwrap();
|
||||
builder.set(1, 2.0);
|
||||
builder.close().unwrap();
|
||||
let annex = FamilyScalarAnnex::open(&path, E).unwrap();
|
||||
@@ -222,12 +221,13 @@ mod tests {
|
||||
fn counting_convention_roundtrips_through_the_header() {
|
||||
let dir = tempdir().unwrap();
|
||||
for kind in [ScalarKind::Entropy, ScalarKind::PairWeight] {
|
||||
for no_ambiguity in [false, true] {
|
||||
let path = dir.path().join(kind.file_name(no_ambiguity));
|
||||
FamilyScalarAnnexBuilder::new(2, &path, kind, no_ambiguity).unwrap().close().unwrap();
|
||||
assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().no_ambiguity(), no_ambiguity);
|
||||
for policy in AmbiguityPolicy::ALL {
|
||||
let path = dir.path().join(kind.file_name(policy));
|
||||
FamilyScalarAnnexBuilder::new(2, &path, kind, policy).unwrap().close().unwrap();
|
||||
assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().policy(), policy);
|
||||
}
|
||||
assert_ne!(kind.file_name(false), kind.file_name(true));
|
||||
let names = kind.file_names();
|
||||
assert!(names[0] != names[1] && names[1] != names[2] && names[0] != names[2]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,7 +235,7 @@ mod tests {
|
||||
fn one_kind_cannot_be_read_as_the_other() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("w.ppwt");
|
||||
FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, false).unwrap().close().unwrap();
|
||||
FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, AmbiguityPolicy::Keep).unwrap().close().unwrap();
|
||||
assert!(FamilyScalarAnnex::open(&path, ScalarKind::Entropy).is_err());
|
||||
assert!(FamilyScalarAnnex::open(&path, ScalarKind::PairWeight).is_ok());
|
||||
}
|
||||
@@ -245,9 +245,9 @@ mod tests {
|
||||
// files written before the flag existed carried four zero bytes there
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("old.pent");
|
||||
FamilyScalarAnnexBuilder::new(1, &path, E, false).unwrap().close().unwrap();
|
||||
FamilyScalarAnnexBuilder::new(1, &path, E, AmbiguityPolicy::Keep).unwrap().close().unwrap();
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
assert_eq!(&bytes[4..8], &[0u8; 4]);
|
||||
assert!(!FamilyScalarAnnex::open(&path, E).unwrap().no_ambiguity());
|
||||
assert_eq!(FamilyScalarAnnex::open(&path, E).unwrap().policy(), AmbiguityPolicy::Keep);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use super::FamilyMask;
|
||||
use super::{AmbiguityPolicy, FamilyMask};
|
||||
|
||||
/// Central-position base of a canonical k-mer, in the fixed 0=A/1=C/2=G/3=T
|
||||
/// encoding — the mask's bit index. `k` must be odd (project invariant).
|
||||
@@ -43,6 +43,6 @@ pub(super) fn triangle_index(n: usize, i: usize, j: usize) -> usize {
|
||||
/// for every annex that counts carriers (family overlap, pair weight,
|
||||
/// quartet), so they can never disagree.
|
||||
#[inline]
|
||||
pub(crate) fn is_carrier(mask: u8, no_ambiguity: bool) -> bool {
|
||||
mask != 0 && !(no_ambiguity && mask.count_ones() > 1)
|
||||
pub(crate) fn is_carrier(mask: u8, policy: AmbiguityPolicy) -> bool {
|
||||
mask != 0 && !policy.drops(mask)
|
||||
}
|
||||
@@ -20,6 +20,7 @@
|
||||
//! reverse.
|
||||
|
||||
pub mod algorithms;
|
||||
mod ambiguity;
|
||||
pub mod extensions;
|
||||
mod family_overlap;
|
||||
mod family_scalar_annex;
|
||||
@@ -35,13 +36,14 @@ pub(crate) use family_scalar_annex::ScalarKind;
|
||||
pub use family_scalar_annex::{FamilyScalarAnnex, FamilyScalarAnnexBuilder};
|
||||
pub use family_overlap::FamilyOverlap;
|
||||
pub(crate) use quartet_annex::QUARTET_ANNEX_FILE_NAMES;
|
||||
pub use ambiguity::AmbiguityPolicy;
|
||||
pub use iter::{
|
||||
MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt,
|
||||
};
|
||||
pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
|
||||
pub use algorithms::{
|
||||
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
|
||||
AlignmentReducer, AnnexSelection, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
|
||||
LayerReport, PairwiseReducer, RawSnpDistanceOutput, SamplePlan, SampleStats,
|
||||
SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SiteLocus, SnpAlignment,
|
||||
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
//! fewer than four genomes — fits every topology and tells a tree nothing.
|
||||
//! See `algorithms::masking::is_quartet_informative` for the exact rule.
|
||||
//!
|
||||
//! Which genomes count as carriers depends on `--no-ambiguity`, so each
|
||||
//! convention has its own file, and **the file name is the parameter**: a
|
||||
//! Which genomes count as carriers depends on the ambiguity policy
|
||||
//! (`--no-ambiguity`, `--only-diploid`), so each policy has its own file, and **the file name is the parameter**: a
|
||||
//! `PersistentBitVec` header has no room for a flag, and changing it would
|
||||
//! change a shared structure for the sake of one annex.
|
||||
//!
|
||||
@@ -24,24 +24,31 @@
|
||||
//! (`algorithms::masking::survives_quartet`) still runs on the resolved
|
||||
//! family.
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAME: &str = "quartet.pbiv";
|
||||
|
||||
/// Same annex, built with `no_ambiguity == true`.
|
||||
/// Same annex, built under [`AmbiguityPolicy::Forbid`].
|
||||
pub(crate) const QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME: &str = "quartet.noamb.pbiv";
|
||||
|
||||
/// Same annex, built under [`AmbiguityPolicy::OnlyDiploid`].
|
||||
pub(crate) const QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME: &str = "quartet.diploid.pbiv";
|
||||
|
||||
/// Every quartet-annex file name, whatever its counting convention — for
|
||||
/// invalidating them all together when the sibling annex they were numbered
|
||||
/// against is rebuilt.
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 2] =
|
||||
[QUARTET_ANNEX_FILE_NAME, QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME];
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 3] = [
|
||||
QUARTET_ANNEX_FILE_NAME,
|
||||
QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME,
|
||||
QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME,
|
||||
];
|
||||
|
||||
/// File name of the quartet annex built under the given `--no-ambiguity`
|
||||
/// convention.
|
||||
pub(crate) fn quartet_annex_file_name(no_ambiguity: bool) -> &'static str {
|
||||
if no_ambiguity {
|
||||
QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME
|
||||
} else {
|
||||
QUARTET_ANNEX_FILE_NAME
|
||||
/// File name of the quartet annex built under the given ambiguity policy.
|
||||
pub(crate) fn quartet_annex_file_name(policy: AmbiguityPolicy) -> &'static str {
|
||||
match policy {
|
||||
AmbiguityPolicy::Keep => QUARTET_ANNEX_FILE_NAME,
|
||||
AmbiguityPolicy::Forbid => QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME,
|
||||
AmbiguityPolicy::OnlyDiploid => QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,10 +57,11 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_two_conventions_never_share_a_file() {
|
||||
assert_ne!(quartet_annex_file_name(false), quartet_annex_file_name(true));
|
||||
for no_ambiguity in [false, true] {
|
||||
assert!(QUARTET_ANNEX_FILE_NAMES.contains(&quartet_annex_file_name(no_ambiguity)));
|
||||
fn the_policies_never_share_a_file() {
|
||||
let names = AmbiguityPolicy::ALL.map(quartet_annex_file_name);
|
||||
assert!(names[0] != names[1] && names[1] != names[2] && names[0] != names[2]);
|
||||
for name in names {
|
||||
assert!(QUARTET_ANNEX_FILE_NAMES.contains(&name));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user