diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 94b179e0..45c64119 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -275,11 +275,9 @@ pub fn run(args: PhyloArgs) { std::process::exit(1); } - let mut alignment_reducer = need_alignment.then(|| { - obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity) - }); - let mut pairwise_reducer = - need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n, args.free_loss, args.no_ambiguity)); + let mut alignment_reducer = + need_alignment.then(|| obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask)); + let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n)); if need_alignment || need_pairwise { let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new(); diff --git a/src/obikphylo/src/siblings/algorithms/alignment.rs b/src/obikphylo/src/siblings/algorithms/alignment.rs index 1371fc41..fb59a1a4 100644 --- a/src/obikphylo/src/siblings/algorithms/alignment.rs +++ b/src/obikphylo/src/siblings/algorithms/alignment.rs @@ -7,7 +7,7 @@ use obikidxcache::index_cache::IndexCache; use rkyv::{Archive, Deserialize, Serialize}; -use super::masking::{iupac_code, masked_state}; +use super::masking::iupac_code; use super::scan::LayerReducer; use super::subsample::SurvivingFamily; @@ -34,21 +34,22 @@ pub struct SnpAlignment { /// it can run alongside any other reducer (e.g. /// `algorithms::pairwise::PairwiseReducer`) over the exact same batches in /// one shared scan, instead of each needing its own `sample_index` call. +/// No `free_loss`/`no_ambiguity` of its own: `SurvivingFamily::masked` has +/// already applied them, once, upstream in `subsample::sample_layer` — see +/// that field's own docs. pub struct AlignmentReducer { genome_indices: Vec, sequences: Vec>, - free_loss: bool, - no_ambiguity: bool, } impl AlignmentReducer { - pub fn new(cache: &IndexCache, excluded: &[bool], free_loss: bool, no_ambiguity: bool) -> Self { + pub fn new(cache: &IndexCache, excluded: &[bool]) -> Self { let n_genomes = cache.meta().genomes().len(); let genome_indices: Vec = (0..n_genomes) .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) .collect(); let sequences = vec![Vec::new(); genome_indices.len()]; - Self { genome_indices, sequences, free_loss, no_ambiguity } + Self { genome_indices, sequences } } pub fn finish(self) -> SnpAlignment { @@ -58,25 +59,18 @@ impl AlignmentReducer { impl LayerReducer for AlignmentReducer { fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { - reduce_alignment(survivors, &self.genome_indices, self.free_loss, self.no_ambiguity, &mut self.sequences); + reduce_alignment(survivors, &self.genome_indices, &mut self.sequences); } } /// Appends one layer's worth of columns onto `sequences` — self-contained -/// (no sampling/masking logic of its own, just character encoding), the -/// actual reduce [`AlignmentReducer`] wraps. -fn reduce_alignment( - survivors: &[SurvivingFamily], - genome_indices: &[usize], - free_loss: bool, - no_ambiguity: bool, - sequences: &mut [Vec], -) { +/// (no sampling/masking logic of its own, just character encoding over +/// `family.masked`, already computed upstream), the actual reduce +/// [`AlignmentReducer`] wraps. +fn reduce_alignment(survivors: &[SurvivingFamily], genome_indices: &[usize], sequences: &mut [Vec]) { for family in survivors { for (row, &g) in genome_indices.iter().enumerate() { - let ch = masked_state(family.genome_mask[g], free_loss, no_ambiguity) - .map(iupac_code) - .unwrap_or(b'?'); + let ch = family.masked[g].map(iupac_code).unwrap_or(b'?'); sequences[row].push(ch); } } diff --git a/src/obikphylo/src/siblings/algorithms/masking.rs b/src/obikphylo/src/siblings/algorithms/masking.rs index 9ae56b34..afc788d6 100644 --- a/src/obikphylo/src/siblings/algorithms/masking.rs +++ b/src/obikphylo/src/siblings/algorithms/masking.rs @@ -33,28 +33,31 @@ pub(crate) fn masked_state(m: u8, free_loss: bool, no_ambiguity: bool) -> Option Some(m) } -/// Whether a family's resolved `genome_mask` still carries real -/// polymorphism (`>=2` distinct masked states) once [`masked_state`] is -/// applied to every *included* genome — i.e. whether it remains a usable -/// alignment site rather than one that collapsed to monomorphic (or fully -/// missing) once uninformative states, and excluded genomes entirely, are -/// dropped from consideration. `excluded[g]` (`false` if `g` is beyond -/// `excluded`'s own length — a caller that never excludes anyone can pass -/// an empty slice) means genome `g` is skipped outright, not merely masked -/// to "?": it never contributes to `seen`, whether or not `free_loss`/ -/// `no_ambiguity` would also have masked it. -pub(crate) fn survives_masking( - genome_mask: &[u8], - excluded: &[bool], - free_loss: bool, - no_ambiguity: bool, -) -> bool { +/// Whether a family's already-[`masked_state`]d per-genome states still +/// carry real polymorphism (`>=2` distinct states) among *included* +/// genomes — i.e. whether it remains a usable alignment site rather than +/// one that collapsed to monomorphic (or fully missing) once uninformative +/// states, and excluded genomes entirely, are dropped from consideration. +/// Takes `masked` — one [`masked_state`] result per genome — rather than +/// the raw `genome_mask` and the masking flags themselves: the caller +/// (`subsample::sample_layer`) computes `masked_state` exactly once per +/// (family, genome) and reuses that same array for this check, for +/// building the eventual alignment column, and for folding the pairwise +/// tally, instead of every consumer recomputing it independently (which +/// used to let their masking semantics silently drift apart — see +/// `algorithms::pairwise::reduce_pairwise`'s own docs for the bug that +/// caused). `excluded[g]` (`false` if `g` is beyond `excluded`'s own +/// length — a caller that never excludes anyone can pass an empty slice) +/// means genome `g` is skipped outright, not merely masked to "?": it +/// never contributes to `seen`, whether or not `masked[g]` is already +/// `None`. +pub(crate) fn survives_masking(masked: &[Option], excluded: &[bool]) -> bool { let mut seen: u16 = 0; - for (g, &m) in genome_mask.iter().enumerate() { + for (g, &state) in masked.iter().enumerate() { if excluded.get(g).copied().unwrap_or(false) { continue; } - if let Some(state) = masked_state(m, free_loss, no_ambiguity) { + if let Some(state) = state { seen |= 1 << state; } } @@ -96,18 +99,25 @@ pub(crate) fn iupac_code(state: u8) -> u8 { mod tests { use super::*; + /// One [`masked_state`] call per raw mask — mirrors what + /// `subsample::sample_layer` does once, up front, before calling + /// [`survives_masking`]. + fn masked(raw: &[u8], free_loss: bool, no_ambiguity: bool) -> Vec> { + raw.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect() + } + #[test] fn plain_no_masking_counts_absence_as_its_own_state() { // Two genomes: one absent (0), one carrying base A (1) — 2 distinct // states without free-loss, so this survives. - assert!(survives_masking(&[0, 1], &[], false, false)); + assert!(survives_masking(&masked(&[0, 1], false, false), &[])); } #[test] fn free_loss_drops_absence_and_can_collapse_to_monomorphic() { // Same data as above, but with free-loss the absent genome becomes // "?" (excluded) — only one real state (A) remains. - assert!(!survives_masking(&[0, 1], &[], true, false)); + assert!(!survives_masking(&masked(&[0, 1], true, false), &[])); } #[test] @@ -116,8 +126,8 @@ mod tests { // carries only A. Without no-ambiguity: 2 distinct states (0b0011, // 0b0001) -> survives. With no-ambiguity: genome 0 becomes "?", // only one real state left -> collapses. - assert!(survives_masking(&[0b0011, 0b0001], &[], false, false)); - assert!(!survives_masking(&[0b0011, 0b0001], &[], false, true)); + assert!(survives_masking(&masked(&[0b0011, 0b0001], false, false), &[])); + assert!(!survives_masking(&masked(&[0b0011, 0b0001], false, true), &[])); } #[test] @@ -126,7 +136,8 @@ mod tests { // this survives (2 distinct states, matching the plain test above). // Excluding genome 1 (the only real signal) leaves just genome 0's // absence — a single state, collapses. - assert!(survives_masking(&[0, 1], &[false, false], false, false)); - assert!(!survives_masking(&[0, 1], &[false, true], false, false)); + let m = masked(&[0, 1], false, false); + assert!(survives_masking(&m, &[false, false])); + assert!(!survives_masking(&m, &[false, true])); } } diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index db738bd5..86f31921 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -407,29 +407,23 @@ impl PartitionDispersion { /// own docs for why the partition axis must be captured here rather than /// after the fact) — the "reduce" half of the map/reduce /// [`super::subsample::sample_index`] drives, self-contained (no -/// sampling logic of its own) so it can run alongside any other reduce -/// over the same batch (see `algorithms::alignment::reduce_alignment`, +/// sampling/masking logic of its own) so it can run alongside any other +/// reduce over the same batch (see `algorithms::alignment::reduce_alignment`, /// run alongside this one whenever both an alignment and a tally are /// wanted from the same [`super::scan::sampled_scan`] call). /// -/// `free_loss`/`no_ambiguity`: the same masking -/// [`algorithms::alignment::reduce_alignment`] applies per genome via -/// `masking::masked_state`, applied here too so the tally agrees with -/// what the alignment actually shows for the same family/genome — used -/// directly for `cardinality` (a masked-away genome contributes `0`, -/// matching its `?` in the alignment, not its raw, pre-masking bit -/// count). `subst`/`same_all`/`same_variable` don't need to look at -/// `masked_state` separately: `single_form`'s own "exactly one bit" -/// requirement already agrees with "masked, then exactly one bit" for -/// every possible mask, for any combination of these two flags (masking -/// only ever turns an already-`None` case — 0 bits or ≥2 bits — into a -/// differently-shaped `None`, never a `Some` into a `None` or vice -/// versa), so routing them through `masked_state` too would be a no-op. +/// Reads `family.masked` directly — [`super::subsample::SurvivingFamily`]'s +/// own docs explain why that's already the masked (`--free-loss`/ +/// `--no-ambiguity`-applied) per-genome state, computed once upstream, the +/// same array [`algorithms::alignment::reduce_alignment`] reads too: this +/// is what guarantees `cardinality` agrees with what the alignment +/// actually shows for the same family/genome (a masked-away genome +/// contributes `0`, matching its `?`). `subst`/`same_all`/`same_variable` +/// don't need anything beyond `single_form`'s "exactly one bit" check on +/// the already-masked state. pub(crate) fn reduce_pairwise( survivors: &[SurvivingFamily], partition: usize, - free_loss: bool, - no_ambiguity: bool, tally: &mut PairwiseTally, dispersion: &mut PartitionDispersion, ) { @@ -438,15 +432,10 @@ pub(crate) fn reduce_pairwise( let mut partition_eligible: u64 = 0; for family in survivors { let variable = family.mask.family_size() >= 2; - let genome_mask = &family.genome_mask; let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - let masked_cardinality = |g: usize| -> usize { - super::masking::masked_state(genome_mask[g], free_loss, no_ambiguity) - .map_or(0, |m| m.count_ones() as usize) + family.masked[g].filter(|m| m.count_ones() == 1).map(|m| m.trailing_zeros() as u8) }; + let masked_cardinality = |g: usize| -> usize { family.masked[g].map_or(0, |m| m.count_ones() as usize) }; for i in 0..n_genomes { let bi = single_form(i); @@ -491,23 +480,14 @@ pub(crate) fn reduce_pairwise( pub struct PairwiseReducer { tally: PairwiseTally, dispersion: PartitionDispersion, - free_loss: bool, - no_ambiguity: bool, } impl PairwiseReducer { - /// `free_loss`/`no_ambiguity`: the same masking flags passed to - /// `AlignmentReducer::new` — must match when both run over the same - /// [`super::scan::sampled_scan`] call, so the cardinality tally agrees - /// with what the alignment actually shows (see [`reduce_pairwise`]'s - /// own docs). - pub fn new(n_genomes: usize, free_loss: bool, no_ambiguity: bool) -> Self { - Self { - tally: PairwiseTally::new(n_genomes), - dispersion: PartitionDispersion::default(), - free_loss, - no_ambiguity, - } + /// No `free_loss`/`no_ambiguity` of its own: `SurvivingFamily::masked` + /// has already applied them, once, upstream in + /// `subsample::sample_layer` — see that field's own docs. + pub fn new(n_genomes: usize) -> Self { + Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() } } pub fn finish(self) -> (PairwiseTally, PartitionDispersion) { @@ -517,14 +497,7 @@ impl PairwiseReducer { impl super::scan::LayerReducer for PairwiseReducer { fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { - reduce_pairwise( - survivors, - partition, - self.free_loss, - self.no_ambiguity, - &mut self.tally, - &mut self.dispersion, - ); + reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion); } } @@ -537,20 +510,20 @@ mod cardinality_masking_tests { // Genome 0 is ambiguous (carries both A and C, mask 0b0011 = cardinality // 2); genome 1 carries only A (mask 0b0001 = cardinality 1). The family // is variable (both A and C are observed somewhere), so it contributes - // to `cardinality` regardless of masking. - fn ambiguous_vs_single() -> Vec { - vec![SurvivingFamily { - family_idx: 0, - mask: FamilyMask::EMPTY.with(0).with(1), // A, C observed - genome_mask: vec![0b0011, 0b0001], - }] + // to `cardinality` regardless of masking. `masked` is what + // `subsample::sample_layer` would have computed upstream (see + // `SurvivingFamily::masked`'s own docs) — the masking decision is baked + // in here, not passed to `reduce_pairwise` itself anymore. + fn ambiguous_vs_single(masked: Vec>) -> Vec { + vec![SurvivingFamily { family_idx: 0, mask: FamilyMask::EMPTY.with(0).with(1), masked }] } #[test] fn cardinality_uses_raw_mask_without_no_ambiguity() { let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&ambiguous_vs_single(), 0, false, false, &mut tally, &mut dispersion); + let families = ambiguous_vs_single(vec![Some(0b0011), Some(0b0001)]); + reduce_pairwise(&families, 0, &mut tally, &mut dispersion); let counts = tally.cardinality_tally(&[]).counts; assert_eq!(counts[2][1], 1, "genome 0's raw cardinality (2) should be tallied"); assert_eq!(counts[0][1], 0, "no masking requested — nothing should collapse to 0"); @@ -558,12 +531,14 @@ mod cardinality_masking_tests { #[test] fn cardinality_masks_ambiguous_genome_to_zero_under_no_ambiguity() { - // Matches what `AlignmentReducer`/`masked_state` would show for - // genome 0 at this site under `--no-ambiguity`: `?`, i.e. no - // information — cardinality 0, not its raw 2. + // `masked[0] == None` matches what `sample_layer` would have + // computed for genome 0 at this site under `--no-ambiguity`: `?` + // in the alignment, i.e. no information — cardinality 0, not its + // raw 2. let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&ambiguous_vs_single(), 0, false, true, &mut tally, &mut dispersion); + let families = ambiguous_vs_single(vec![None, Some(0b0001)]); + reduce_pairwise(&families, 0, &mut tally, &mut dispersion); let counts = tally.cardinality_tally(&[]).counts; assert_eq!(counts[2][1], 0, "raw cardinality must not leak through under --no-ambiguity"); assert_eq!(counts[0][1], 1, "the ambiguous genome should be masked to cardinality 0"); diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 099ff3c8..01d62177 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -278,12 +278,16 @@ mod tests { /// single-base composition, an edge case irrelevant to real data, not /// something these tests need to exercise). fn tally_2genomes(pairs: &[(u8, u8)], n_shared: usize) -> PairwiseTally { + // No masking requested (`free_loss = no_ambiguity = false`) — + // `masked_state(m, false, false)` is always `Some(m)`, so the raw + // mask is exactly what `subsample::sample_layer` would have + // computed here. let mut families = Vec::new(); for &(a, b) in pairs { families.push(SurvivingFamily { family_idx: families.len(), mask: FamilyMask::EMPTY.with(a.trailing_zeros() as u8).with(b.trailing_zeros() as u8), - genome_mask: vec![a, b], + masked: vec![Some(a), Some(b)], }); } const BASES: [u8; 4] = [A, C, G, T]; @@ -293,12 +297,12 @@ mod tests { families.push(SurvivingFamily { family_idx: families.len(), mask: FamilyMask::EMPTY.with(b.trailing_zeros() as u8).with(other.trailing_zeros() as u8), - genome_mask: vec![b, b], + masked: vec![Some(b), Some(b)], }); } let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&families, 0, false, false, &mut tally, &mut dispersion); + reduce_pairwise(&families, 0, &mut tally, &mut dispersion); tally } diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index 31ee836e..9ad7a2a9 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -54,7 +54,7 @@ use rand::RngExt; use super::entropy::iter_full_entropy; use super::family_scan::{Selection, scan_layer_families}; use super::is_fast_mode; -use super::masking::survives_masking; +use super::masking::{masked_state, survives_masking}; use super::minorant_selection::eligible_bitset_layer; use crate::siblings::FamilyMask; use crate::siblings::extensions::SiblingBuilder; @@ -66,15 +66,24 @@ const MAX_TOPUP_ROUNDS: usize = 5; /// One surviving family — already drawn (Bernoulli/entropy-bias), already /// resolved (`scan_layer_families`), already passed post-hoc masking -/// (`super::masking::survives_masking`). `genome_mask` is an owned copy: the -/// borrowed slice `scan_layer_families` hands its own callback doesn't -/// outlive that callback, but a `SurvivingFamily` is meant to be handed to -/// [`sample_index`]'s caller as part of a whole layer's batch, well after -/// that callback returns. +/// (`super::masking::survives_masking`). `masked[g]` is genome `g`'s +/// [`super::masking::masked_state`] result — computed exactly once, here, +/// from the raw (per-family, per-genome) resolved state, *before* +/// `--free-loss`/`--no-ambiguity` are applied — never the raw state +/// itself: every consumer downstream (this family's own survival check, +/// its eventual alignment column, its contribution to the pairwise tally) +/// needs the masked view, and computing it once and sharing it is what +/// guarantees they can never disagree about what a genome's state was at +/// this site (see `algorithms::pairwise::reduce_pairwise`'s own docs for +/// the inconsistency that arose from each consumer masking independently). +/// An owned `Vec`: the borrowed slice `scan_layer_families` hands its own +/// callback doesn't outlive that callback, but a `SurvivingFamily` is meant +/// to be handed to [`sample_index`]'s caller as part of a whole layer's +/// batch, well after that callback returns. pub struct SurvivingFamily { pub family_idx: usize, pub mask: FamilyMask, - pub genome_mask: Vec, + pub masked: Vec>, } /// Gaussian-kernel entropy bias parameters — `mu`/`sigma` are the @@ -111,9 +120,10 @@ fn circular_entropy_values( /// [`SurvivingFamily`] this produces is already a usable alignment site, no /// second resolution pass needed by the caller. `excluded[g]` (see /// `survives_masking`'s own docs) never counts genome `g` toward that check -/// — `genome_mask` itself still carries every genome's own resolved state, -/// excluded or not, so a caller building per-genome output (e.g. an -/// alignment row) must apply the same exclusion itself when consuming it. +/// — `SurvivingFamily::masked` itself still carries every genome's own +/// masked state, excluded or not, so a caller building per-genome output +/// (e.g. an alignment row) must apply the same exclusion itself when +/// consuming it. /// /// `survivors` is a single `Arc>` — one allocation per /// layer (bounded by that layer's own quota, never the whole index), handed @@ -348,12 +358,10 @@ fn sample_layer( mask.family_size() >= 2, "eligible bitset must only accept non-monomorphic minorants" ); - if survives_masking(genome_mask, excluded, free_loss, no_ambiguity) { - survivors.push(SurvivingFamily { - family_idx, - mask, - genome_mask: genome_mask.to_vec(), - }); + let masked: Vec> = + genome_mask.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect(); + if survives_masking(&masked, excluded) { + survivors.push(SurvivingFamily { family_idx, mask, masked }); kept += 1; } },