refactor: treat absence as state unless --free-loss and update quartet annexes
Refactor the handling of genome absence to be a distinct state unless `--free-loss` is specified. This change introduces `QuartetConvention` to pair ambiguity policies with the free-loss flag, updating file naming conventions and logic across sampling, masking, and stats modules. - Update `--exclude-genome` description in `pair_coverage_sampling.md` to clarify absence as a state unless `--free-loss` is specified. - Move multi-member genome handling convention from `pair_coverage_sampling.md` to file name conventions. - Clarify persisted annex structure in `pair_coverage_sampling.md` as `PersistentBitVec` per non-monomorphic family with counting convention carried by file name. - Fix initial version's absence handling in `has_bipartition`, the annex, and `--sample-stats` to correctly count index 0 when absence is a state. - Replace entropy-only builder with `AnnexRequest` and `ensure_layer_family_annexes`. - Consolidate sampling parameters into a single `SamplingSpec`. - Update family constraint description in `sampling_design.md`. - Modify uniform draws and `--quartet` removal text in `sampling_design.md`. - Clarify state counting in tests for `--free-loss` option in `sampling_design.md`. - State that absence forms a pair without `--free-loss` but never with it in `sampling_design.md`. - Update `--quartet` option to require at least two distinct states, each carried by at least two genomes. - Define absence as a state unless `--free-loss` is given for family carrier definition. - Clarify `--pair-balance` favors families covering pairs with fewest shared families. - Modify logic for families fitting every tree topology equally well in `phylo.md`. - Change `--quartet` restriction logic to apply after pre-checks involving `--no-ambiguity` and `--exclude-genome`. - Update `obikmer` dependency version from 1.4.4 to 1.4.5 in `Cargo.lock` and `Cargo.toml`. - Modify `annex_selection` in `args/mod.rs` to derive `free_loss`, `entropy`, `quartet`, and `overlap` from `SamplingSpec`. - Update `SamplingArgs::quartet` documentation in `sampling.rs` to describe state determination based on `--free-loss`. - Add `free_loss` field to `AnnexSelection` struct in `family_annexes.rs`. - Change `AnnexRequest` to use `Vec<QuartetConvention>` instead of `Vec<AmbiguityPolicy>`. - Update `request` logic in `family_annexes.rs` to conditionally create `QuartetConvention` based on `self.quartet`. - Modify `dedup` signature in `family_annexes.rs` to accept iterator of type `T: Ord`. - Update `ensure_layer_family_annexes` in `family_annexes.rs` to use `missing_files` and `QuartetConvention`. - Add `missing_files` definition to `family_annexes.rs`. - Change `has_bipartition` in `masking.rs` to filter counts where count is `>= 2`. - Update `is_quartet_informative` in `masking.rs` to accept `free_loss` boolean parameter. - Modify `two_pairs_of_distinct_states_is_informative` and `singleton_variant_is_not_informative` in `masking.rs` to test both `free_loss` policies. - Add `under_free_loss_absence_is_missing_and_forms_no_pair` and `without_free_loss_absence_is_a_state_that_can_form_a_pair` definitions to `masking.rs`. - Update `no_ambiguity_drops_multi_member_genomes_from_the_count` in `masking.rs` to check `is_quartet_informative` under `free_loss=true`. - Add `the_annex_rule_and_the_exact_check_agree_on_the_same_family` definition to `masking.rs`. - Remove `absent_genomes_are_not_a_state` definition from `masking.rs`. - Update `restrict_to_quartet` in `quartet.rs` to accept `QuartetConvention` instead of `AmbiquityPolicy`. - Modify `add_site` in `sample_stats.rs` to handle state 0 as a state participating in bipartition. - Add `absence_as_a_state_makes_a_site_informative` definition to `sample_stats.rs`. - Add `quartet_convention` function to `SamplingSpec` in `subsample.rs`. - Update `sample_index` in `subsample.rs` to use `spec.quartet_convention()` for `AnnexRequest`. - Update `restrict_to_quartet` call in `subsample.rs` to use `spec.quartet_convention()`. - Add comment regarding `--no-ambiguity` counting convention in `helpers.rs`. - Import `QUARTET_ANNEX_FILE_NAMES` and `QuartetConvention` from `quartet_annex` in `mod.rs`. - Add `QuartetConvention` struct with `policy` and `free_loss` fields to `quartet_annex.rs`. - Modify `quartet_annex_file_name` in `quartet_annex.rs` to take `QuartetConvention`. - Add `the_conventions_never_share_a_file` function to `quartet_annex.rs`. - Remove `the_policies_never_share_a_file` definition from `quartet_annex.rs`.
This commit is contained in:
1 parent
c93616b432
commit
5db7b440da
15 files changed
+218
-101
No files matched your search
Generated
+1
-1
@@ -1531,7 +1531,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "obikmer"
|
||||
version = "1.4.4"
|
||||
version = "1.4.5"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"csv",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "obikmer"
|
||||
version = "1.4.4"
|
||||
version = "1.4.5"
|
||||
edition = "2024"
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -115,6 +115,7 @@ impl PhyloArgs {
|
||||
pub fn annex_selection(&self) -> AnnexSelection {
|
||||
AnnexSelection {
|
||||
policy: self.ambiguity(),
|
||||
free_loss: self.sample.free_loss,
|
||||
entropy: self.sample.entropy.is_some(),
|
||||
quartet: self.sample.quartet,
|
||||
overlap: self.annex.family_overlap
|
||||
|
||||
@@ -61,8 +61,12 @@ pub struct SamplingArgs {
|
||||
/// a family carried by fewer than four genomes, fits every arrangement
|
||||
/// of the genomes and teaches nothing about the tree.
|
||||
///
|
||||
/// Which genomes count as carriers follows --no-ambiguity. Applies to
|
||||
/// pseudo-alignments, Sankoff analyses and SNP-based distances.
|
||||
/// The states are those of the alignment. Without --free-loss, absence
|
||||
/// is a state of its own, so two genomes with a base and two genomes
|
||||
/// lacking the family form two pairs. With --free-loss, absence is
|
||||
/// missing data and never forms a pair. Which genomes count as carriers
|
||||
/// also follows --no-ambiguity. Applies to pseudo-alignments, Sankoff
|
||||
/// analyses and SNP-based distances.
|
||||
#[arg(long)]
|
||||
pub quartet: bool,
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use obicompactvec::PersistentBitVecBuilder;
|
||||
use obikidxcache::index_cache::IndexCache;
|
||||
@@ -22,19 +22,20 @@ use super::masking::is_quartet_informative;
|
||||
use super::minorant_selection::non_monomorphic_reindex_layer;
|
||||
use super::pair_weight::pair_balance_weight;
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
use crate::siblings::quartet_annex::quartet_annex_file_name;
|
||||
use crate::siblings::quartet_annex::{QuartetConvention, quartet_annex_file_name};
|
||||
use crate::siblings::{
|
||||
FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind, family_overlap_file_name,
|
||||
};
|
||||
|
||||
/// Which annexes a caller needs, each listed once per ambiguity policy (how
|
||||
/// genomes carrying several members of a family are counted) — an empty list
|
||||
/// asks for none. A pair weight is computed from the family-overlap annex of
|
||||
/// asks for none (the quartet annex also depends on `--free-loss`, so its
|
||||
/// convention is a [`QuartetConvention`]). A pair weight is computed from the family-overlap annex of
|
||||
/// its own policy, so asking for one asks for the other.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct AnnexRequest {
|
||||
pub entropy: Vec<AmbiguityPolicy>,
|
||||
pub quartet: Vec<AmbiguityPolicy>,
|
||||
pub quartet: Vec<QuartetConvention>,
|
||||
pub overlap: Vec<AmbiguityPolicy>,
|
||||
pub pair_weight: Vec<AmbiguityPolicy>,
|
||||
}
|
||||
@@ -46,6 +47,9 @@ pub(crate) struct AnnexRequest {
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct AnnexSelection {
|
||||
pub policy: AmbiguityPolicy,
|
||||
/// `--free-loss`: part of the quartet annex's convention (whether absence
|
||||
/// counts as a state), irrelevant to the other annexes.
|
||||
pub free_loss: bool,
|
||||
pub entropy: bool,
|
||||
pub quartet: bool,
|
||||
pub overlap: bool,
|
||||
@@ -57,7 +61,11 @@ impl AnnexSelection {
|
||||
let one = |wanted: bool| if wanted { vec![self.policy] } else { Vec::new() };
|
||||
AnnexRequest {
|
||||
entropy: one(self.entropy),
|
||||
quartet: one(self.quartet),
|
||||
quartet: if self.quartet {
|
||||
vec![QuartetConvention { policy: self.policy, free_loss: self.free_loss }]
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
overlap: one(self.overlap),
|
||||
pair_weight: one(self.pair_weight),
|
||||
}
|
||||
@@ -100,8 +108,22 @@ impl AnnexRequest {
|
||||
}
|
||||
}
|
||||
|
||||
fn dedup(variants: impl IntoIterator<Item = AmbiguityPolicy>) -> Vec<AmbiguityPolicy> {
|
||||
let mut v: Vec<AmbiguityPolicy> = variants.into_iter().collect();
|
||||
/// The annexes among `conventions` (each once) whose file does not exist yet
|
||||
/// in `dir`, with their path.
|
||||
fn missing_files<T: Ord + Copy>(
|
||||
dir: &Path,
|
||||
name: impl Fn(T) -> &'static str,
|
||||
conventions: &[T],
|
||||
) -> Vec<(T, PathBuf)> {
|
||||
dedup(conventions.iter().copied())
|
||||
.into_iter()
|
||||
.map(|convention| (convention, dir.join(name(convention))))
|
||||
.filter(|(_, path)| !path.exists())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn dedup<T: Ord>(variants: impl IntoIterator<Item = T>) -> Vec<T> {
|
||||
let mut v: Vec<T> = variants.into_iter().collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
@@ -131,17 +153,9 @@ pub(crate) fn ensure_layer_family_annexes(
|
||||
let layer = cache
|
||||
.get_layer(partition, layer_idx)
|
||||
.expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)");
|
||||
let missing = |name: &dyn Fn(AmbiguityPolicy) -> &'static str,
|
||||
variants: &[AmbiguityPolicy]|
|
||||
-> Vec<(AmbiguityPolicy, PathBuf)> {
|
||||
dedup(variants.iter().copied())
|
||||
.into_iter()
|
||||
.map(|policy| (policy, layer.dir().join(name(policy))))
|
||||
.filter(|(_, path)| !path.exists())
|
||||
.collect()
|
||||
};
|
||||
let missing_entropy = missing(&|p| ScalarKind::Entropy.file_name(p), &request.entropy);
|
||||
let missing_quartet = missing(&quartet_annex_file_name, &request.quartet);
|
||||
let missing_entropy =
|
||||
missing_files(layer.dir(), |p| ScalarKind::Entropy.file_name(p), &request.entropy);
|
||||
let missing_quartet = missing_files(layer.dir(), quartet_annex_file_name, &request.quartet);
|
||||
if missing_entropy.is_empty() && missing_quartet.is_empty() && overlap_accs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -182,8 +196,11 @@ pub(crate) fn ensure_layer_family_annexes(
|
||||
builder.set(compact, h as f32);
|
||||
}
|
||||
}
|
||||
for (policy, builder) in quartet_builders.iter_mut() {
|
||||
builder.set(compact, is_quartet_informative(genome_mask, *policy));
|
||||
for (convention, builder) in quartet_builders.iter_mut() {
|
||||
builder.set(
|
||||
compact,
|
||||
is_quartet_informative(genome_mask, convention.policy, convention.free_loss),
|
||||
);
|
||||
}
|
||||
for (_, acc) in overlap_accs.iter_mut() {
|
||||
acc.add_family(genome_mask);
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
//! data is dropped.
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
use crate::siblings::helpers::is_carrier;
|
||||
|
||||
/// This genome's masked character state, or `None` for "?" (missing). `m`
|
||||
/// is one `genome_mask` entry (bit `b` set iff this genome carries the
|
||||
@@ -69,39 +68,43 @@ pub(crate) fn survives_masking(masked: &[Option<u8>], excluded: &[bool]) -> bool
|
||||
}
|
||||
|
||||
/// Whether `counts` — the number of genomes per *called* state, indexed by
|
||||
/// the `0..=15` state value, index `0` (absence) never counted — shows a
|
||||
/// bipartition: at least two distinct states each carried by at least two
|
||||
/// genomes. That is the pattern that can separate two pairs of genomes from
|
||||
/// each other (`AB|CD`); a singleton variant, or a family carried by fewer
|
||||
/// than four genomes, fits every topology and informs none.
|
||||
/// the `0..=15` state value — shows a bipartition: at least two distinct
|
||||
/// states each carried by at least two genomes. That is the pattern that can
|
||||
/// separate two pairs of genomes from each other (`AB|CD`); a singleton
|
||||
/// variant, or a family carried by fewer than four genomes, fits every
|
||||
/// topology and informs none.
|
||||
///
|
||||
/// Index `0` is absence (`∅`). It holds genomes only when absence is a real
|
||||
/// state of the alignment (no `--free-loss`), and is then a state like any
|
||||
/// other; under `--free-loss` absent genomes are missing data, are never
|
||||
/// counted, and the index stays empty.
|
||||
pub(crate) fn has_bipartition(counts: &[u32; 16]) -> bool {
|
||||
counts[1..].iter().filter(|&&c| c >= 2).count() >= 2
|
||||
counts.iter().filter(|&&c| c >= 2).count() >= 2
|
||||
}
|
||||
|
||||
/// Quartet-informativeness of a family from its raw per-genome masks alone,
|
||||
/// under the `--no-ambiguity` counting convention: absent genomes (`0`)
|
||||
/// never count, and with `no_ambiguity` a genome carrying several members
|
||||
/// doesn't either — otherwise its combination of bases is a state of its
|
||||
/// own. Ignores `--exclude-genome` and `--free-loss` on purpose: absence is
|
||||
/// never a called state here, and removing genomes can only destroy a
|
||||
/// bipartition, never create one — so `false` is final, and `true` is only
|
||||
/// a necessary condition, re-checked exactly by [`survives_quartet`] once
|
||||
/// the family is resolved. This is what the persisted quartet annex stores.
|
||||
pub(crate) fn is_quartet_informative(genome_mask: &[u8], policy: AmbiguityPolicy) -> bool {
|
||||
/// under one counting convention: each genome is read through
|
||||
/// [`masked_state`] exactly as in the alignment — absence is a state of its
|
||||
/// own unless `free_loss`, and `policy` makes the genomes carrying too many
|
||||
/// members missing. Ignores `--exclude-genome` on purpose: removing genomes
|
||||
/// can only destroy a bipartition, never create one — so `false` is final, and
|
||||
/// `true` is only a necessary condition, re-checked exactly by
|
||||
/// [`survives_quartet`] once the family is resolved. This is what the
|
||||
/// persisted quartet annex stores.
|
||||
pub(crate) fn is_quartet_informative(genome_mask: &[u8], policy: AmbiguityPolicy, free_loss: bool) -> bool {
|
||||
let mut counts = [0u32; 16];
|
||||
for &m in genome_mask {
|
||||
if !is_carrier(m, policy) {
|
||||
continue;
|
||||
if let Some(state) = masked_state(m, free_loss, policy) {
|
||||
counts[(state & 0b1111) as usize] += 1;
|
||||
}
|
||||
counts[(m & 0b1111) as usize] += 1;
|
||||
}
|
||||
has_bipartition(&counts)
|
||||
}
|
||||
|
||||
/// Exact quartet check on an already-[`masked_state`]d family, over the
|
||||
/// *included* genomes (`excluded` as in [`survives_masking`]). Only called
|
||||
/// states count: a genome whose masked state is `0` (absence kept as a real
|
||||
/// state, i.e. no `--free-loss`) is not a carrier of anything.
|
||||
/// *included* genomes (`excluded` as in [`survives_masking`]). Missing
|
||||
/// genomes (`None`) count for nothing; absence kept as a real state (no
|
||||
/// `--free-loss`, masked state `0`) is a state like any other.
|
||||
pub(crate) fn survives_quartet(masked: &[Option<u8>], excluded: &[bool]) -> bool {
|
||||
let mut counts = [0u32; 16];
|
||||
for (g, &state) in masked.iter().enumerate() {
|
||||
@@ -119,33 +122,48 @@ pub(crate) fn survives_quartet(masked: &[Option<u8>], excluded: &[bool]) -> bool
|
||||
mod quartet_tests {
|
||||
use super::*;
|
||||
|
||||
const KEEP: AmbiguityPolicy = AmbiguityPolicy::Keep;
|
||||
const FORBID: AmbiguityPolicy = AmbiguityPolicy::Forbid;
|
||||
|
||||
#[test]
|
||||
fn two_pairs_of_distinct_states_is_informative() {
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], AmbiguityPolicy::Keep));
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], AmbiguityPolicy::Forbid));
|
||||
for free_loss in [true, false] {
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], KEEP, free_loss));
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], FORBID, free_loss));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn singleton_variant_is_not_informative() {
|
||||
// three genomes in one state plus one lone variant: fits any topology
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], AmbiguityPolicy::Keep));
|
||||
// fewer than four carriers can never be informative
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], AmbiguityPolicy::Keep));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], AmbiguityPolicy::Keep));
|
||||
for free_loss in [true, false] {
|
||||
// three genomes in one state plus one lone variant: fits any topology
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], KEEP, free_loss));
|
||||
// fewer than four genomes with a state can never be informative
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], KEEP, free_loss));
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], KEEP, free_loss));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_genomes_are_not_a_state() {
|
||||
// two carriers of one base, two absences: absence must not form the 2nd pair
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], AmbiguityPolicy::Keep));
|
||||
fn under_free_loss_absence_is_missing_and_forms_no_pair() {
|
||||
// two carriers of one base, two absences: absence is missing data
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], KEEP, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_free_loss_absence_is_a_state_that_can_form_a_pair() {
|
||||
// A A ∅ ∅ separates two pairs of genomes once ∅ is a character state
|
||||
assert!(is_quartet_informative(&[0b0001, 0b0001, 0, 0], KEEP, false));
|
||||
// ... but a single absent genome is a lone state, like any singleton
|
||||
assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0], KEEP, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_ambiguity_drops_multi_member_genomes_from_the_count() {
|
||||
// the two-member genomes form a pair of their own state without the flag
|
||||
let masks = [0b0001u8, 0b0001, 0b0011, 0b0011];
|
||||
assert!(is_quartet_informative(&masks, AmbiguityPolicy::Keep));
|
||||
assert!(!is_quartet_informative(&masks, AmbiguityPolicy::Forbid));
|
||||
assert!(is_quartet_informative(&masks, KEEP, true));
|
||||
assert!(!is_quartet_informative(&masks, FORBID, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -154,8 +172,19 @@ mod quartet_tests {
|
||||
assert!(survives_quartet(&masked, &[]));
|
||||
// excluding one genome of the second pair breaks the bipartition
|
||||
assert!(!survives_quartet(&masked, &[false, false, true]));
|
||||
// an absence kept as state 0 (no --free-loss) is not a called state
|
||||
assert!(!survives_quartet(&[Some(1), Some(1), Some(0), Some(0)], &[]));
|
||||
// an absence kept as state 0 (no --free-loss) is a state: ∅ ∅ is a pair
|
||||
assert!(survives_quartet(&[Some(1), Some(1), Some(0), Some(0)], &[]));
|
||||
// the same genomes under --free-loss are missing: no second pair
|
||||
assert!(!survives_quartet(&[Some(1), Some(1), None, None], &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_annex_rule_and_the_exact_check_agree_on_the_same_family() {
|
||||
let raw = [0b0001u8, 0b0001, 0, 0];
|
||||
for free_loss in [true, false] {
|
||||
let masked: Vec<_> = raw.iter().map(|&m| masked_state(m, free_loss, KEEP)).collect();
|
||||
assert_eq!(is_quartet_informative(&raw, KEEP, free_loss), survives_quartet(&masked, &[]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,12 +7,12 @@ use obicompactvec::{PersistentBitVec, TempBitVecBuilder};
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::minorant_selection::is_non_monomorphic_minorant;
|
||||
use crate::siblings::quartet_annex::quartet_annex_file_name;
|
||||
use crate::siblings::{AmbiguityPolicy, SiblingAnnex};
|
||||
use crate::siblings::quartet_annex::{QuartetConvention, quartet_annex_file_name};
|
||||
use crate::siblings::SiblingAnnex;
|
||||
|
||||
/// Clears, in `eligible` (indexed by `family_idx`, monomorphic minorants
|
||||
/// included), every family the quartet annex flags as not quartet-informative
|
||||
/// under the `policy` counting convention. The annex is indexed by the
|
||||
/// under the `convention` counting convention. The annex is indexed by the
|
||||
/// compact non-monomorphic ordinal, so the sibling annex is walked in the same
|
||||
/// order to translate one numbering into the other — annex-only, no
|
||||
/// cross-partition resolution. The annex must already exist
|
||||
@@ -20,10 +20,10 @@ use crate::siblings::{AmbiguityPolicy, SiblingAnnex};
|
||||
pub(crate) fn restrict_to_quartet(
|
||||
eligible: &mut TempBitVecBuilder,
|
||||
layer_dir: &Path,
|
||||
policy: AmbiguityPolicy,
|
||||
convention: QuartetConvention,
|
||||
) -> OKIResult<()> {
|
||||
let annex = SiblingAnnex::open_in(layer_dir).map_err(OKIError::Io)?;
|
||||
let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(policy)))
|
||||
let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(convention)))
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
let mut family_idx = 0usize;
|
||||
|
||||
@@ -84,8 +84,12 @@ impl SampleStatsReducer {
|
||||
let mut counts = [0u32; 16];
|
||||
let mut carriers: Vec<usize> = Vec::new();
|
||||
for (g, &state) in masked.iter().enumerate() {
|
||||
if let Some(state) = state.filter(|&s| s != 0 && !self.excluded[g]) {
|
||||
counts[(state & 0b1111) as usize] += 1;
|
||||
let Some(state) = state.filter(|_| !self.excluded[g]) else { continue };
|
||||
// Absence (state 0, only present without `--free-loss`) is a state
|
||||
// of the alignment, so it takes part in the bipartition — but its
|
||||
// genome carries none of the family's members.
|
||||
counts[(state & 0b1111) as usize] += 1;
|
||||
if state != 0 {
|
||||
carriers.push(g);
|
||||
}
|
||||
}
|
||||
@@ -140,6 +144,19 @@ mod tests {
|
||||
assert_eq!(s.overlap(3, 0), s.overlap(0, 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absence_as_a_state_makes_a_site_informative() {
|
||||
let mut r = SampleStatsReducer::new(4, &[]);
|
||||
// A A ∅ ∅ (no --free-loss): two pairs of states, though only two carriers
|
||||
r.add_site(&[Some(1), Some(1), Some(0), Some(0)]);
|
||||
// the same genomes under --free-loss: the absences are missing
|
||||
r.add_site(&[Some(1), Some(1), None, None]);
|
||||
let s = r.finish();
|
||||
assert_eq!(s.sites_by_carriers[2], 2);
|
||||
assert_eq!(s.informative_by_carriers[2], 1);
|
||||
assert_eq!(s.calls, vec![2, 2, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excluded_genomes_count_nowhere() {
|
||||
let mut r = SampleStatsReducer::new(4, &[false, false, false, true]);
|
||||
|
||||
@@ -57,7 +57,7 @@ use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::masking::{masked_state, survives_masking, survives_quartet};
|
||||
use super::minorant_selection::{eligible_bitset_layer, iter_full_scalar};
|
||||
use super::{AnnexRequest, is_fast_mode, restrict_to_quartet};
|
||||
use crate::siblings::{AmbiguityPolicy, FamilyMask, ScalarKind};
|
||||
use crate::siblings::{AmbiguityPolicy, FamilyMask, QuartetConvention, ScalarKind};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
|
||||
/// Resolution rounds a single layer's sampling may take before giving up on
|
||||
@@ -122,6 +122,15 @@ pub struct SamplingSpec {
|
||||
pub entropy_bias: Option<EntropyBias>,
|
||||
}
|
||||
|
||||
impl SamplingSpec {
|
||||
/// The counting convention of the quartet annex this draw reads: how
|
||||
/// genomes with several members count, and whether absence is a state —
|
||||
/// the same two choices the post-hoc masking makes.
|
||||
pub(crate) fn quartet_convention(&self) -> QuartetConvention {
|
||||
QuartetConvention { policy: self.ambiguity, free_loss: self.free_loss }
|
||||
}
|
||||
}
|
||||
|
||||
/// What one layer contributed to a draw — one row of `--sample-plan`.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LayerReport {
|
||||
@@ -260,7 +269,7 @@ pub(crate) fn sample_index(
|
||||
// convention) — build any missing one now, once, rather than mid-walk.
|
||||
let request = AnnexRequest {
|
||||
entropy: spec.entropy_bias.map(|_| vec![spec.ambiguity]).unwrap_or_default(),
|
||||
quartet: if spec.quartet { vec![spec.ambiguity] } else { Vec::new() },
|
||||
quartet: if spec.quartet { vec![spec.quartet_convention()] } else { Vec::new() },
|
||||
overlap: Vec::new(),
|
||||
pair_weight: if spec.pair_balance { vec![spec.ambiguity] } else { Vec::new() },
|
||||
};
|
||||
@@ -282,7 +291,7 @@ pub(crate) fn sample_index(
|
||||
.expect("cache-consistent: n_layer(partition) already bounds layer_idx");
|
||||
let mut eligible = eligible_bitset_layer(layer.dir())?;
|
||||
if spec.quartet {
|
||||
restrict_to_quartet(&mut eligible, layer.dir(), spec.ambiguity)?;
|
||||
restrict_to_quartet(&mut eligible, layer.dir(), spec.quartet_convention())?;
|
||||
}
|
||||
let n_minorants = eligible.view().len();
|
||||
let eligible_count = eligible.view().count_ones();
|
||||
|
||||
@@ -40,8 +40,9 @@ pub(super) fn triangle_index(n: usize, i: usize, j: usize) -> usize {
|
||||
/// the `--no-ambiguity` counting convention: any non-empty state does, except
|
||||
/// — with `no_ambiguity` — a genome carrying several members, which is
|
||||
/// missing data in the alignment and so shares nothing there. One definition
|
||||
/// for every annex that counts carriers (family overlap, pair weight,
|
||||
/// quartet), so they can never disagree.
|
||||
/// for every annex that counts carriers (family overlap, pair weight), so they
|
||||
/// can never disagree. (The quartet annex does not use it: absence can be a
|
||||
/// state there, so it reads genomes through `masking::masked_state`.)
|
||||
#[inline]
|
||||
pub(crate) fn is_carrier(mask: u8, policy: AmbiguityPolicy) -> bool {
|
||||
mask != 0 && !policy.drops(mask)
|
||||
|
||||
@@ -35,7 +35,7 @@ pub(crate) use family_overlap::{
|
||||
pub(crate) use family_scalar_annex::ScalarKind;
|
||||
pub use family_scalar_annex::{FamilyScalarAnnex, FamilyScalarAnnexBuilder};
|
||||
pub use family_overlap::FamilyOverlap;
|
||||
pub(crate) use quartet_annex::QUARTET_ANNEX_FILE_NAMES;
|
||||
pub(crate) use quartet_annex::{QUARTET_ANNEX_FILE_NAMES, QuartetConvention};
|
||||
pub use ambiguity::AmbiguityPolicy;
|
||||
pub use iter::{
|
||||
MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt,
|
||||
|
||||
@@ -12,10 +12,15 @@
|
||||
//! fewer than four genomes — fits every topology and tells a tree nothing.
|
||||
//! See `algorithms::masking::is_quartet_informative` for the exact rule.
|
||||
//!
|
||||
//! Which genomes count as carriers depends on the ambiguity policy
|
||||
//! (`--no-ambiguity`, `--only-diploid`), so each policy has its own file, and **the file name is the parameter**: a
|
||||
//! `PersistentBitVec` header has no room for a flag, and changing it would
|
||||
//! change a shared structure for the sake of one annex.
|
||||
//! Absence counts as a state exactly when it does in the alignment: without
|
||||
//! `--free-loss` a genome carrying none of the family's members has the state
|
||||
//! `∅`, which can form a pair like any base; with `--free-loss` it is missing
|
||||
//! data and counts for nothing. Which genomes count as carriers also depends on
|
||||
//! the ambiguity policy (`--no-ambiguity`, `--only-diploid`). Each
|
||||
//! [`QuartetConvention`] (ambiguity policy × `--free-loss`) has its own file,
|
||||
//! and **the file name is the parameter**: a `PersistentBitVec` header has no
|
||||
//! room for a flag, and changing it would change a shared structure for the
|
||||
//! sake of one annex.
|
||||
//!
|
||||
//! The flag is computed over every genome of the index. A later
|
||||
//! `--exclude-genome` can only remove carriers, never create a bipartition,
|
||||
@@ -26,29 +31,59 @@
|
||||
|
||||
use crate::siblings::AmbiguityPolicy;
|
||||
|
||||
/// How a quartet annex counts the genomes of a family: one annex per value.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub(crate) struct QuartetConvention {
|
||||
/// How genomes carrying several members of the family are counted.
|
||||
pub policy: AmbiguityPolicy,
|
||||
/// `--free-loss`: absence is missing data instead of a state of its own.
|
||||
pub free_loss: bool,
|
||||
}
|
||||
|
||||
impl QuartetConvention {
|
||||
/// Every convention, in a fixed order.
|
||||
#[cfg(test)]
|
||||
pub(crate) const ALL: [QuartetConvention; 6] = [
|
||||
QuartetConvention { policy: AmbiguityPolicy::Keep, free_loss: true },
|
||||
QuartetConvention { policy: AmbiguityPolicy::Forbid, free_loss: true },
|
||||
QuartetConvention { policy: AmbiguityPolicy::OnlyDiploid, free_loss: true },
|
||||
QuartetConvention { policy: AmbiguityPolicy::Keep, free_loss: false },
|
||||
QuartetConvention { policy: AmbiguityPolicy::Forbid, free_loss: false },
|
||||
QuartetConvention { policy: AmbiguityPolicy::OnlyDiploid, free_loss: false },
|
||||
];
|
||||
}
|
||||
|
||||
// The three `--free-loss` files keep the names the annex had when absence was
|
||||
// never counted: that rule is exactly the `--free-loss` one, so annexes already
|
||||
// on disk stay valid under it. The files for absence-as-a-state are new names,
|
||||
// so an annex built under the old rule can never be read as one of them.
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAME: &str = "quartet.pbiv";
|
||||
|
||||
/// Same annex, built under [`AmbiguityPolicy::Forbid`].
|
||||
pub(crate) const QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME: &str = "quartet.noamb.pbiv";
|
||||
|
||||
/// Same annex, built under [`AmbiguityPolicy::OnlyDiploid`].
|
||||
pub(crate) const QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME: &str = "quartet.diploid.pbiv";
|
||||
pub(crate) const QUARTET_ANNEX_ABSENCE_FILE_NAME: &str = "quartet.abs.pbiv";
|
||||
pub(crate) const QUARTET_ANNEX_ABSENCE_NO_AMBIGUITY_FILE_NAME: &str = "quartet.abs.noamb.pbiv";
|
||||
pub(crate) const QUARTET_ANNEX_ABSENCE_ONLY_DIPLOID_FILE_NAME: &str = "quartet.abs.diploid.pbiv";
|
||||
|
||||
/// Every quartet-annex file name, whatever its counting convention — for
|
||||
/// invalidating them all together when the sibling annex they were numbered
|
||||
/// against is rebuilt.
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 3] = [
|
||||
/// Every quartet-annex file name, whatever its convention — for invalidating
|
||||
/// them all when the sibling annex they are numbered against is rebuilt.
|
||||
pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 6] = [
|
||||
QUARTET_ANNEX_FILE_NAME,
|
||||
QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME,
|
||||
QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME,
|
||||
QUARTET_ANNEX_ABSENCE_FILE_NAME,
|
||||
QUARTET_ANNEX_ABSENCE_NO_AMBIGUITY_FILE_NAME,
|
||||
QUARTET_ANNEX_ABSENCE_ONLY_DIPLOID_FILE_NAME,
|
||||
];
|
||||
|
||||
/// File name of the quartet annex built under the given ambiguity policy.
|
||||
pub(crate) fn quartet_annex_file_name(policy: AmbiguityPolicy) -> &'static str {
|
||||
match policy {
|
||||
AmbiguityPolicy::Keep => QUARTET_ANNEX_FILE_NAME,
|
||||
AmbiguityPolicy::Forbid => QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME,
|
||||
AmbiguityPolicy::OnlyDiploid => QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME,
|
||||
/// File name of the quartet annex built under the given convention.
|
||||
pub(crate) fn quartet_annex_file_name(convention: QuartetConvention) -> &'static str {
|
||||
match (convention.free_loss, convention.policy) {
|
||||
(true, AmbiguityPolicy::Keep) => QUARTET_ANNEX_FILE_NAME,
|
||||
(true, AmbiguityPolicy::Forbid) => QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME,
|
||||
(true, AmbiguityPolicy::OnlyDiploid) => QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME,
|
||||
(false, AmbiguityPolicy::Keep) => QUARTET_ANNEX_ABSENCE_FILE_NAME,
|
||||
(false, AmbiguityPolicy::Forbid) => QUARTET_ANNEX_ABSENCE_NO_AMBIGUITY_FILE_NAME,
|
||||
(false, AmbiguityPolicy::OnlyDiploid) => QUARTET_ANNEX_ABSENCE_ONLY_DIPLOID_FILE_NAME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,11 +92,13 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_policies_never_share_a_file() {
|
||||
let names = AmbiguityPolicy::ALL.map(quartet_annex_file_name);
|
||||
assert!(names[0] != names[1] && names[1] != names[2] && names[0] != names[2]);
|
||||
for name in names {
|
||||
assert!(QUARTET_ANNEX_FILE_NAMES.contains(&name));
|
||||
fn the_conventions_never_share_a_file() {
|
||||
let mut names: Vec<&str> = QuartetConvention::ALL.map(quartet_annex_file_name).to_vec();
|
||||
for name in &names {
|
||||
assert!(QUARTET_ANNEX_FILE_NAMES.contains(name));
|
||||
}
|
||||
names.sort_unstable();
|
||||
names.dedup();
|
||||
assert_eq!(names.len(), QuartetConvention::ALL.len());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user