diff --git a/src/obikmer/src/cmd/phylo/args.rs b/src/obikmer/src/cmd/phylo/args.rs index 305607bb..8226533f 100644 --- a/src/obikmer/src/cmd/phylo/args.rs +++ b/src/obikmer/src/cmd/phylo/args.rs @@ -35,8 +35,9 @@ fn parse_gamma_shape(s: &str) -> Result { /// `--distance` value — either one of `obikphylo::DistanceMetric`'s /// whole-index metrics (routed to `IndexCache::distance`) or one of /// `obikphylo::SnpDistanceKind`'s `snp-*` corrections (routed to -/// `SiblingExt::snp_distance`, the sibling-annex pipeline) — two genuinely -/// different code paths behind one CLI vocabulary, see +/// `obikphylo::siblings::distance_matrix`, over a tally built by +/// `sampled_scan`/`PairwiseReducer`, the sibling-annex pipeline) — two +/// genuinely different code paths behind one CLI vocabulary, see /// `DevDocMD/theory/evolutionary_distances.md`, "`--distance` unification". #[derive(clap::ValueEnum, Clone, Copy, Debug)] pub enum DistanceArg { @@ -121,10 +122,10 @@ pub struct PhyloArgs { /// Exclude a genome (by its exact label) — from `--pseudo-alignment`'s /// sampling (a family whose only polymorphism lived in an excluded /// genome is discarded during sampling, not filtered afterward — see - /// `obikphylo::siblings::extensions::SiblingExt::snp_pseudo_alignment`'s - /// own docs) and from the distance matrix / shared-kmer matrix CSV - /// output (row and column both dropped; the underlying computation - /// itself is unaffected). Repeatable. + /// `obikphylo::siblings::sampled_scan`'s own docs) and from the + /// distance matrix / shared-kmer matrix CSV output (row and column + /// both dropped; the underlying computation itself is unaffected). + /// Repeatable. #[arg(long = "exclude-genome", value_name = "LABEL")] pub exclude_genome: Vec, @@ -194,10 +195,9 @@ pub struct PhyloArgs { /// `--pseudo-alignment`/`--sankoff` (mandatory for both) and for a /// `snp-*` `--distance` value (optional there: omitted means exhaustive /// — every non-monomorphic minorant of the whole index, not an - /// approximation, see `obikphylo::siblings::SiblingExt::snp_distance`'s - /// own docs) — a target, not a guarantee when given (proportional - /// per-layer sampling; see - /// `obikphylo::siblings::SiblingExt::snp_pseudo_alignment`'s own docs). + /// approximation) — a target, not a guarantee when given (proportional + /// per-layer sampling; see `obikphylo::siblings::sampled_scan`'s own + /// docs). #[arg(long)] pub subsample: Option, @@ -329,7 +329,8 @@ pub struct PhyloArgs { /// the central-position SNP substitution spectrum (`snp-raw`, `snp-jc`, /// `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv`) — /// the latter route to a different computation entirely - /// (`SiblingExt::snp_distance`, requires `--sibling-annex` first; see + /// (`obikphylo::siblings::sampled_scan` + `distance_matrix`, requires + /// `--sibling-annex` first; see /// `DevDocMD/theory/evolutionary_distances.md`, "`--distance` /// unification" for the full catalog and why LogDet/Tajima-Nei/F84/ /// HKY85 aren't offered yet). diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 45c64119..43e451bf 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -45,10 +45,10 @@ pub fn run(args: PhyloArgs) { let n = labels.len(); // ── Genome exclusion (`--exclude-genome`) ─────────────────────────────────── - // Resolved once, up front: `snp_pseudo_alignment` needs it baked into - // sampling itself (see its own docs), and the distance/shared-kmer CSV - // writers below just skip these rows/columns at write time — the - // underlying `cache.distance(...)` computation is unaffected either way. + // Resolved once, up front: `sampled_scan` needs it baked into sampling + // itself (see its own docs), and the distance/shared-kmer CSV writers + // below just skip these rows/columns at write time — the underlying + // `cache.distance(...)` computation is unaffected either way. let exclude_mask: Vec = { let mut mask = vec![false; n]; for label in &args.exclude_genome { diff --git a/src/obikmer/src/cmd/phylo/sankoff.rs b/src/obikmer/src/cmd/phylo/sankoff.rs index 0a412a86..9168c80f 100644 --- a/src/obikmer/src/cmd/phylo/sankoff.rs +++ b/src/obikmer/src/cmd/phylo/sankoff.rs @@ -1,5 +1,5 @@ //! Output writers for `--sankoff` — no calibration logic here, just -//! formatting: `obikphylo::siblings::SiblingExt::sankoff_bundle` and the +//! formatting: `obikphylo::siblings::SankoffBundle::finish` and the //! `cardinality_transition_probs`/`composition_transition_probs`/ //! `pairwise_cost_matrix` calibration functions do all the actual work in //! `mod.rs`, this module only serialises their results. @@ -13,8 +13,8 @@ use tracing::info; // ── Sankoff pseudo-alignment → FASTA ──────────────────────────────────────── // -// Same data as `--pseudo-alignment`'s output (`SnpAlignment`/ -// `snp_pseudo_alignment`), re-coded so its symbols match the accompanying +// Same data as `--pseudo-alignment`'s own output (`SnpAlignment`, built by +// `AlignmentReducer`), re-coded so its symbols match the accompanying // `--sankoff` matrix output exactly: `0` for the empty/absent state instead // of `-`, which TNT/PhyG would otherwise read as their own gap character // rather than our "family absent" state. Unless `free_loss` (`--free-loss`) diff --git a/src/obikphylo/src/lib.rs b/src/obikphylo/src/lib.rs index 7e80a1bc..8610d84b 100644 --- a/src/obikphylo/src/lib.rs +++ b/src/obikphylo/src/lib.rs @@ -14,12 +14,15 @@ //! to a lower-level crate if an indexing-side consumer (e.g. default-`k` //! guidance) appears. //! -//! `siblings` is being reconnected piece by piece while `obikindex`'s -//! stateless audit is caught up with — see the project memory on this. So -//! far it only carries the annex-construction path -//! (`SiblingExt::build_sibling_annex`); the rest of the old -//! `siblings_old` (SNP distance, cardinality, pseudo-alignment, stats) is -//! still disconnected. +//! `siblings` carries the full pipeline today: annex construction +//! (`SiblingExt::build_sibling_annex`), Shannon entropy/family-overlap +//! annexes, and the sampled SNP path (pseudo-alignment, `snp-*` +//! `--distance`, Sankoff cost-matrix calibration) driven through a single +//! shared scan (`siblings::sampled_scan` + `LayerReducer` impls) — see +//! `siblings::algorithms::scan`'s own module docs. `siblings_old` is its +//! predecessor, fully superseded and not compiled into this crate (no +//! `mod siblings_old;` anywhere) — dead source left on disk, not an +//! in-progress migration target. mod distance; pub mod karlin; diff --git a/src/obikphylo/src/siblings/algorithms/family_scan.rs b/src/obikphylo/src/siblings/algorithms/family_scan.rs index 692b7272..4afa908b 100644 --- a/src/obikphylo/src/siblings/algorithms/family_scan.rs +++ b/src/obikphylo/src/siblings/algorithms/family_scan.rs @@ -78,9 +78,13 @@ fn family_batch_size(n_genomes: usize) -> usize { } /// Restricts [`scan_layer_families`] to a subset of a layer's families, -/// identified by their iteration-order index. `All` (the only variant any -/// current caller uses) keeps a full, unsampled scan; `Some` is exercised -/// once sampling (`--subsample`/`--entropy-bias`) is reconnected. +/// identified by their iteration-order index. `All` keeps a full, +/// unsampled scan (`SiblingExt::shannon_entropy_csv`, +/// `sibling_annex_stats`); `Some` bounds it to a caller-picked set — +/// `subsample::sample_layer`'s own accepted-candidates batch, or +/// `entropy::ensure_layer_entropy_annex`'s non-monomorphic-only +/// restriction — skipping the expensive cross-partition resolution +/// entirely for every family outside the set. pub enum Selection<'a> { All, Some(&'a std::collections::HashSet), diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 7eff4aff..24b62f4c 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -43,6 +43,7 @@ pub use subsample::{EntropyBias, SurvivingFamily}; pub(crate) use annex::build_layer_sibling_annex; pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4}; pub(crate) use family_scan::{Selection, scan_layer_families}; +pub(crate) use minorant_selection::non_monomorphic_reindex_layer; pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram}; /// Whether every layer number in `cache` fits in a `FamilyMask` field diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index 86f31921..c358e2e8 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -22,15 +22,19 @@ use rkyv::{Archive, Deserialize, Serialize}; use super::subsample::SurvivingFamily; -/// One pair's own tallies, unfiltered — `subst` is populated only by -/// variable families (a substitution, `bi != bj`, is only possible when the -/// family varies *somewhere* in the index; see the module docs), so it -/// needs no separate `variable` gate of its own. `same_all` counts every -/// agreement (`bi == bj`, any family); `same_variable` is the subset from -/// variable families only — [`RawSnpDistanceOutput`] wants the former, -/// [`BasePairTally`] the latter (they are *not* the same population — see -/// `BasePairTally::same`'s own docs). `cardinality` is restricted to -/// variable families only, matching [`CardinalityTally`]'s own scope. +/// One pair's own tallies, unfiltered. Both `subst` and `same_all` are +/// populated by every family reaching [`reduce_pairwise`] — no separate +/// "variable families only" gate needed for either: [`super::subsample::sample_layer`] +/// only ever draws from `minorant_selection::eligible_bitset_layer`, which +/// contains exclusively non-monomorphic minorants (`family_size() >= 2`, +/// enforced by a `debug_assert!` right where a [`SurvivingFamily`] is +/// built), so *every* family folded here already varies somewhere in the +/// index. `same_all` is what both [`RawSnpDistanceOutput`] and +/// [`BasePairTally`] read (there used to be a second, `same_variable` +/// field restricted to variable families for the latter — removed once it +/// was confirmed, via the invariant above, to always hold the exact same +/// values as `same_all`). `cardinality` is populated for every family too, +/// same reasoning. /// /// `Archive`/`Serialize`/`Deserialize`: part of [`PairwiseTally`]'s own /// `--session` persistence (see that struct's docs) — `rkyv` needs every @@ -40,7 +44,6 @@ use super::subsample::SurvivingFamily; struct PairStats { subst: [[u64; 4]; 4], same_all: [u64; 4], - same_variable: [u64; 4], cardinality: [[u64; 5]; 5], } @@ -145,7 +148,7 @@ impl PairwiseTally { } let stats = self.pair(i, j); for a in 0..4 { - same[a] += stats.same_variable[a]; + same[a] += stats.same_all[a]; for b in 0..4 { counts[a][b] += stats.subst[a][b]; } @@ -267,19 +270,17 @@ pub struct BasePairTally { /// `same`, not here. pub counts: [[u64; 4]; 4], /// `same[a]` = number of eligible loci, pooled over included genome - /// pairs, where both genomes' single forms are `a` **and** the family - /// varies somewhere in the index — a fully invariant family's - /// agreement is genome-wide background, not SNP-adjacent signal (see - /// `PairStats::same_all` vs `same_variable`). + /// pairs, where both genomes' single forms are `a` — every family + /// folded into [`PairwiseTally`] already varies somewhere in the index + /// by construction (see `PairStats`'s own docs), so there is no + /// further "variable families only" restriction to apply here. pub same: [u64; 4], } /// Cardinality (`0..=4` members of a family) co-occurrence, pooled over /// every genome pair except `--exclude-genome` ones (see /// [`PairwiseTally::cardinality_tally`]'s own docs for why this is *not* -/// gated by `ratio_ceiling` the way [`BasePairTally`] is), restricted to -/// variable families (`family_size() >= 2`) — matching -/// `snp_pseudo_alignment`'s own scope. `pub`: part of +/// gated by `ratio_ceiling` the way [`BasePairTally`] is). `pub`: part of /// [`super::SankoffBundle`]'s public signature. pub struct CardinalityTally { /// `counts[a][b] == counts[b][a]` = number of family sites, pooled over @@ -418,9 +419,9 @@ impl PartitionDispersion { /// same array [`algorithms::alignment::reduce_alignment`] reads too: this /// is what guarantees `cardinality` agrees with what the alignment /// actually shows for the same family/genome (a masked-away genome -/// contributes `0`, matching its `?`). `subst`/`same_all`/`same_variable` -/// don't need anything beyond `single_form`'s "exactly one bit" check on -/// the already-masked state. +/// contributes `0`, matching its `?`). No "variable families only" gate +/// on any field: see [`PairStats`]'s own docs for why every family reaching +/// this function already varies somewhere in the index, by construction. pub(crate) fn reduce_pairwise( survivors: &[SurvivingFamily], partition: usize, @@ -431,7 +432,6 @@ pub(crate) fn reduce_pairwise( let mut partition_subst: u64 = 0; let mut partition_eligible: u64 = 0; for family in survivors { - let variable = family.mask.family_size() >= 2; let single_form = |g: usize| -> Option { family.masked[g].filter(|m| m.count_ones() == 1).map(|m| m.trailing_zeros() as u8) }; @@ -451,18 +451,13 @@ pub(crate) fn reduce_pairwise( stats.subst[bj as usize][bi as usize] += 1; } else { stats.same_all[bi as usize] += 1; - if variable { - stats.same_variable[bi as usize] += 1; - } } } - if variable { - let card_j = masked_cardinality(j); - stats.cardinality[card_i][card_j] += 1; - if card_i != card_j { - stats.cardinality[card_j][card_i] += 1; - } + let card_j = masked_cardinality(j); + stats.cardinality[card_i][card_j] += 1; + if card_i != card_j { + stats.cardinality[card_j][card_i] += 1; } } } diff --git a/src/obikphylo/src/siblings/algorithms/stats.rs b/src/obikphylo/src/siblings/algorithms/stats.rs index e8417ef8..b0536788 100644 --- a/src/obikphylo/src/siblings/algorithms/stats.rs +++ b/src/obikphylo/src/siblings/algorithms/stats.rs @@ -93,7 +93,7 @@ pub(crate) fn sibling_family_size_histogram(cache: &IndexCache) -> OKIResult<[u6 /// at a finer grain, across partitions — stacking another parallel layer /// loop on top would fight it for locality instead of helping, exactly the /// same reasoning `SiblingExt::build_sibling_annex`/`shannon_entropy_csv`/ -/// `snp_pseudo_alignment` already follow). +/// `algorithms::scan::sampled_scan` already follow). pub(crate) fn sibling_annex_stats(cache: &IndexCache) -> OKIResult { let n_genomes = cache.meta().genomes().len(); let fast_mode = is_fast_mode(cache); diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index 9ad7a2a9..c33cde1e 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -89,7 +89,7 @@ pub struct SurvivingFamily { /// Gaussian-kernel entropy bias parameters — `mu`/`sigma` are the /// `--entropy`/`--entropy-sd` CLI values (defaulted by the caller, not /// here). `pub`, not `pub(crate)`: part of the public signature of -/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`]. +/// [`super::scan::sampled_scan`]. #[derive(Clone, Copy)] pub struct EntropyBias { pub mu: f64, @@ -138,24 +138,19 @@ fn circular_entropy_values( /// function's own iteration order (`cache.partitions()` then `0..n_layer`, /// both deterministic for an unchanged index), to skip *entirely* — /// neither their (cheap) eligibility-bitset lookup's `sample_layer` call -/// nor `on_layer` runs for them. Exists for `--session` resume: quotas are -/// computed from `total_eligible` (summed over *every* layer regardless of -/// `skip_layers`), so a resumed run reproduces the exact same *per-layer -/// quotas* an uninterrupted one would have — i.e. how many sites each -/// remaining layer contributes stays correct. It does **not** reproduce -/// the exact same *sample*: `sample_layer`'s draws come from `rand::rng()` -/// (`Self::sample_layer`'s own docs), a per-process, OS-seeded generator -/// with no continuity across a process boundary, so the layers processed -/// after a resume draw different (but equally valid) random outcomes than -/// an uninterrupted run would have. Confirmed by direct test (kill mid-run, -/// resume, compare to an uninterrupted run against the same index/params: -/// different distance matrices, both legitimate samples). This is -/// deliberate, not a defect: run-to-run sampling variance is wanted (see -/// `DevDocMD/theory/evolutionary_distances.md`, "`--session`" section) — -/// making resume bit-reproducible would require seeding deterministically, -/// which would also remove that variance for ordinary repeated runs -/// unless scoped carefully, and was explicitly declined for now. `0` for -/// every caller not doing checkpointed resume. +/// nor `on_layer` runs for them. Built for checkpointed-resume support +/// (quotas are computed from `total_eligible`, summed over *every* layer +/// regardless of `skip_layers`, so a resumed run reproduces the exact same +/// *per-layer quotas* an uninterrupted one would have — i.e. how many +/// sites each remaining layer contributes stays correct — without +/// reproducing the exact same *sample*: `sample_layer`'s draws come from +/// `rand::rng()`, a per-process, OS-seeded generator with no continuity +/// across a process boundary). No current caller passes anything but `0`: +/// the `--session`-based resume mechanism this was built for was removed +/// (see `algorithms::scan`'s own module docs) pending a rebuild on top of +/// [`sampled_scan`](super::scan::sampled_scan) — the parameter stays, as +/// the seam a future resume mechanism would hook into, rather than being +/// ripped out and re-added later. /// /// Returns the actual number of *newly processed* sites kept (not /// counting whatever `skip_layers` layers already contributed on an diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index c80bf521..7cc26ba3 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -15,7 +15,8 @@ use obisys::progress_bar; use crate::siblings::algorithms::{ Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4, - is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram, + is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, sibling_annex_stats, + sibling_family_size_histogram, }; use crate::siblings::extensions::SiblingBuilder; use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap}; @@ -51,8 +52,12 @@ pub trait SiblingExt { /// every cached layer, from an already-built sibling annex (run /// [`build_sibling_annex`](Self::build_sibling_annex) first). /// - /// Full, unsampled scan of every cached layer (`Selection::All`) — no - /// `--subsample`/`--entropy-bias` yet (see the project memory on this). + /// Full, unsampled scan of every non-monomorphic minorant of every + /// cached layer (`Selection::Some`, restricted the same way + /// `ensure_layer_entropy_annex` is — monomorphic minorants have + /// nothing to report here, so never pay `scan_layer_families`'s + /// cross-partition resolution for them) — no `--subsample`/ + /// `--entropy-bias` yet (see the project memory on this). fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>; /// The global family-size histogram alone (`SiblingAnnexStats::counts`, @@ -152,11 +157,9 @@ impl SiblingExt for IndexCache { fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> { // Entropy/family-overlap annexes aren't this method's own output, - // but building them is exactly the same full scan this method is - // about to run anyway — see - // `SiblingBuilder::ensure_entropy_and_overlap_annexes`'s own docs - // for why that's a side effect worth taking here rather than a - // separate, explicit step. + // but ensuring they exist is a cheap prerequisite check, not a + // reason to reuse their scan here — see below, this method + // deliberately runs its own, differently-scoped one. self.ensure_entropy_and_overlap_annexes()?; let n_genomes = self.meta().genomes().len(); @@ -170,6 +173,19 @@ impl SiblingExt for IndexCache { for part in self.partitions() { let n_layer = self.n_layer(part).unwrap_or(0); for l in 0..n_layer { + let layer = self + .get_layer(part, l) + .expect("cache-consistent: n_layer(part) already bounds l"); + // Bounded to non-monomorphic minorants only, same reasoning + // as `ensure_layer_entropy_annex`'s own docs: monomorphism + // is already known from the annex bits alone (~98% of a + // layer's minorants, measured on real data), so there is no + // reason to pay `scan_layer_families`'s expensive + // cross-partition resolution for families this method + // skips right below anyway. + let reindex = non_monomorphic_reindex_layer(layer.dir())?; + let eligible: std::collections::HashSet = reindex.keys().copied().collect(); + let mut write_err = None; scan_layer_families( self, @@ -177,14 +193,15 @@ impl SiblingExt for IndexCache { l, n_genomes, fast_mode, - &Selection::All, + &Selection::Some(&eligible), |family_idx, mask, genome_mask| { if write_err.is_some() { return; } - if mask.family_size() < 2 { - return; // monomorphic family — entropy trivially 0, no signal, skip - } + debug_assert!( + mask.family_size() >= 2, + "Selection::Some(eligible) must only visit non-monomorphic minorants" + ); let Some((h15, n_present)) = family_entropy(genome_mask) else { return; }; diff --git a/src/obikphylo/src/siblings_old/alignment.rs b/src/obikphylo/src/siblings_old/alignment.rs deleted file mode 100644 index 7efbfff5..00000000 --- a/src/obikphylo/src/siblings_old/alignment.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::sync::Arc; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::family_scan::{Selection, scan_layer_families}; -use super::subsample::EntropyBias; - -/// IUPAC ambiguity code for a per-genome family presence mask (bit `b` set -/// iff the genome carries the member whose own central base is `b`): -/// single bit -> the plain base; 2 or 3 bits -> the matching IUPAC -/// ambiguity code (preserves partial information instead of collapsing to -/// `N`, the same convention used for diploid heterozygous VCF/FASTA sites); -/// all 4 bits -> `N`; no bits (genome carries none of the family's observed -/// members) -> `-` (no data at this locus for this genome). -pub(super) fn iupac_code(mask: u8) -> u8 { - match mask & 0b1111 { - 0b0000 => b'-', - 0b0001 => b'A', - 0b0010 => b'C', - 0b0100 => b'G', - 0b1000 => b'T', - 0b0101 => b'R', // A/G - 0b1010 => b'Y', // C/T - 0b0110 => b'S', // C/G - 0b1001 => b'W', // A/T - 0b1100 => b'K', // G/T - 0b0011 => b'M', // A/C - 0b1110 => b'B', // C/G/T - 0b1101 => b'D', // A/G/T - 0b1011 => b'H', // A/C/T - 0b0111 => b'V', // A/C/G - 0b1111 => b'N', - _ => unreachable!("masked to 4 bits"), - } -} - -/// A SNP-only pseudo-alignment: one row (byte sequence, IUPAC-coded) per -/// genome, one column per variable family (`family_size() >= 2` — monomorphic -/// families carry no signal and are skipped, unlike `raw_snp_distance`'s -/// tally which does count them as `shared`). Column order is the same, -/// deterministic sweep order as the annex build (partition, then layer, then -/// slot) — arbitrary but stable and identical across genomes, which is all a -/// pseudo-alignment needs (there is no natural genomic coordinate to sort by -/// once flanks are dropped). See `DevDocMD/theory/evolutionary_distances.md`, -/// "Multi-genome framing: family as pseudo-alignment column". -pub struct SnpAlignment { - /// `sequences[g]` = genome `g`'s IUPAC-coded row, same length for every - /// genome (`sequences.len()` columns). - pub sequences: Vec>, -} - -/// Adds [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) to `KmerIndex` — -/// phylo-domain functionality, kept out of `obikindex` itself (see -/// `DevDocMD/architecture/siblings.md`). -pub trait SnpAlignmentExt { - /// Build the SNP-only pseudo-alignment from an already-built sibling - /// annex (run [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) first). - /// - /// `subsample`: `--subsample N` — cap the number of variable families - /// retained to (approximately) `N`, sampled proportionally per layer - /// among non-monomorphic minorants (see - /// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`"). - /// `None` keeps today's behaviour: every non-monomorphic minorant of - /// every layer. `entropy_bias`: `--entropy`/`--entropy-sd` — weight - /// that sample (or, with `subsample = None`, filter the whole index) - /// by a Gaussian kernel on each family's entropy — see - /// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". - fn snp_pseudo_alignment( - &self, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult; -} - -impl SnpAlignmentExt for KmerIndex { - fn snp_pseudo_alignment( - &self, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - let selections = - super::subsample::compute_selections(self, &layer_dirs, subsample, entropy_bias)?; - - let pb = progress_bar("snp_pseudo_alignment", layer_dirs.len() as u64, "layers"); - // One layer at a time, not `par_iter()` over layers — same - // rationale as `build_sibling_annex`: running many layers' - // `scan_layer_families` concurrently would each group their own - // lookups by partition internally, but interleave those sweeps - // across layers at the OS level, scattering page-cache access over - // every partition at once again and defeating the whole point of - // the grouping. Columns appended straight into `sequences` as each - // family comes back from `scan_layer_families` (bounded-batch, not - // whole-layer) — no intermediate per-layer buffer, layer order - // giving a single deterministic column order across the whole index. - let mut sequences: Vec> = vec![Vec::new(); n_genomes]; - for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |_family_idx, mask, genome_mask| { - if mask.family_size() < 2 { - return; // monomorphic family — no signal, skip - } - for (g, &m) in genome_mask.iter().enumerate() { - sequences[g].push(iupac_code(m)); - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(SnpAlignment { sequences }) - } -} diff --git a/src/obikphylo/src/siblings_old/build.rs b/src/obikphylo/src/siblings_old/build.rs deleted file mode 100644 index 4ed4a703..00000000 --- a/src/obikphylo/src/siblings_old/build.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::Ordering; - -use rayon::prelude::*; - -use obikseq::CanonicalKmer; -use obikindex::layer::MphfLayer; -use obipipeline::ThrottleGuard; -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::OKIResult; - -use super::cache::PartitionCache; -use super::helpers::{central_base, is_minorant}; -use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder}; - -// ── obipipeline data types ───────────────────────────────────────────────── - -/// A batch of this layer's distinct k-mers (iteration-order index + k-mer), -/// the pipeline's source item — batched, not one k-mer per item, so that -/// pipeline messages and their synchronisation cost stay amortised over -/// thousands of lookups (see `build_layer_sibling_annex`'s comment on -/// `BATCH_SIZE`). Carries the throttle permit for the whole batch, moved -/// (not cloned) into the corresponding `VariantBatch` — a 1-to-1 transform, -/// unlike the fan-out `Flat` stage this replaced, needs no `Arc` sharing. -/// -/// The `usize` is this k-mer's position in `iter_kmers()`'s enumeration -/// order, **not** an MPHF slot — see `DevDocMD/architecture/siblings.md`. It -/// is the same index the annex file is written under. -struct SourceBatch { - items: Vec<(usize, CanonicalKmer)>, - _permit: ThrottleGuard, -} - -/// One batch's worth of central-substitution variants (up to 3 per source -/// k-mer), each already routed to its destination partition and carrying -/// its own central base (0=A/1=C/2=G/3=T) — the mask bit it will set on a -/// hit. `(dest_partition, variant, source_order, base)` per entry, where -/// `source_order` is the source k-mer's iteration-order index (see -/// `SourceBatch`), not an MPHF slot. -struct VariantBatch { - items: Vec<(usize, CanonicalKmer, usize, u8)>, - _permit: ThrottleGuard, -} - -enum SibData { - Batch(SourceBatch), - Variants(VariantBatch), -} - -/// Adds [`build_sibling_annex`](Self::build_sibling_annex) to `KmerIndex`. -pub trait SiblingAnnexBuildExt { - /// Build the sibling-count/minorant annex for every layer of every - /// partition of this (already built) index, writing one annex file per - /// layer alongside its existing index files. Safe to call again later - /// (e.g. after a fresh `merge`) — each run simply overwrites the annex - /// files of the index it is called on. - /// - /// Construction only — no statistics gathered here on purpose: this is - /// meant to run routinely (it is the artefact the SNP-family distances - /// will consume), while the sibling-count distribution - /// ([`sibling_annex_stats`](super::stats::SiblingStatsExt::sibling_annex_stats)) - /// is a separate, occasional diagnostic pass over the result, not run - /// every time. - /// - /// Cross-partition/cross-layer lookups are required (a k-mer's siblings - /// can live in any partition), but the layer loop itself — and thus the - /// annex file this produces — stays local to one layer at a time. - fn build_sibling_annex(&self) -> OKIResult<()>; -} - -impl SiblingAnnexBuildExt for KmerIndex { - fn build_sibling_annex(&self) -> OKIResult<()> { - let n_parts = self.n_partitions(); - - tracing::info!("opening {n_parts} partition(s) for the sibling-annex sweep"); - let cache = Arc::new(PartitionCache::build( - self, - n_parts, - self.meta().config.with_counts, - )?); - - let pb = progress_bar("sibling_annex", n_parts as u64, "partitions"); - let mut total_slots: u64 = 0; - for part in 0..n_parts { - let index_dir = self.index_dir(part); - if !index_dir.exists() { - pb.inc(1); - continue; - } - let meta = self.partition_meta(part)?; - - let mut part_slots: u64 = 0; - for l in 0..meta.n_layers { - part_slots += build_layer_sibling_annex( - self, - &self.layer_dir(part, l), - n_parts, - l, - &cache, - )?; - } - total_slots += part_slots; - pb.inc(1); - pb.set_message(format!( - "partition {part}: {part_slots} kmers ({total_slots} total)" - )); - } - pb.finish_and_clear(); - tracing::info!("sibling annex built — {total_slots} kmers across {n_parts} partitions"); - - Ok(()) - } -} - -/// Returns the number of distinct k-mers (annex entries) processed, for -/// progress reporting. A free function, not a `KmerIndex` method — called -/// only from `build_sibling_annex` above, in the same file. -fn build_layer_sibling_annex( - index: &KmerIndex, - layer_dir: &Path, - n_parts: usize, - l: usize, - cache: &Arc, -) -> OKIResult { - let mphf = MphfLayer::open(layer_dir)?; - let k = index.kmer_size(); - let n = mphf.n(); - - // Whether this layer's family-member fields can carry a real layer - // number (exploited by `family_scan::scan_layer_families`'s fast path) - // or must stay presence-only — a single fact about the whole index, - // decided once by `cache` (built from the same `PartitionMeta` this - // function would otherwise re-derive) so the writer and every reader - // agree; see `PartitionCache::fast_mode`'s docs. - let fast_mode = cache.fast_mode(); - - // ── The annex file itself is the reconciliation state, indexed by - // this layer's k-mer iteration order (the physical layout of - // `unitigs.bin`), never by MPHF slot — this layer's own k-mers are - // known members by construction, so no evidence check, no MPHF - // slot, and no slot -> k-mer reconstruction is needed or legitimate - // here (see `DevDocMD/architecture/siblings.md`: the MPHF is not - // invertible, and evidence answers membership, not identity). No - // separate in-memory accumulator: every concurrent write goes - // straight through `SiblingAnnexBuilder::atomic_slot` into the - // mmap — a layer can hold billions of k-mers, so a `Vec` shadowing - // the whole file in RAM just to copy it out again at the end (the - // previous design) doubles memory for no benefit once the builder - // itself can be written concurrently. - // - // `Arc`, not a bare `SiblingAnnexBuilder` — - // `Pipe::apply` requires its source iterator to be `Send + 'static` - // (its items are dispatched to worker threads that outlive this - // call), so the batch-generating closure below needs an owned - // handle it can move in, not a borrow of a local. `atomic_slot`, - // not `set`, for every concurrent write below: the gather phase - // parallelises across destination partitions (independent - // `cache.find` calls, safe to run concurrently) and their hits can - // land on arbitrary, possibly-shared source entries — a lock-free - // `fetch_or` avoids needing any synchronisation beyond that. - // Reclaimed as a plain owned value (`Arc::try_unwrap`) once every - // concurrent phase below is done, for the sequential finalisation - // pass. ───────────────────────────────────────────────────────── - let annex_path = layer_dir.join(ANNEX_FILE_NAME); - let builder = Arc::new(SiblingAnnexBuilder::new(n, &annex_path)?); - - // ── obipipeline: a *batch* transform, not a per-k-mer `Flat` one — - // the actual cross-partition lookup reuses - // `KmerPartition::query_partition_with` (the same batching mechanism - // `obikmer query` already uses: open a partition's files once, - // answer a whole batch of queries against it) instead of one lookup - // per pipeline item. A per-item lookup (tried first) reopened/ - // re-mmap'd every target partition's files on every single variant - // — fine at toy scale, but ~90% system time against a real index, - // observed in practice. A *later* attempt still pushed one pipeline - // message per generated variant (a `Flat` stage, `SourceItem` => - // `VariantQuery`, one k-mer in => up to 3 variants out as separate - // messages) — cheaper than reopening files, but sampling a real run - // showed most wall-clock time going into per-message channel - // send/notify syscalls instead of the lookup itself: the pipeline's - // whole point is amortising synchronisation over a batch, and a - // single k-mer's ≤3 variants is far too fine a granularity for - // that. Batching `BATCH_SIZE` source k-mers into one pipeline item - // — a plain 1-to-1 (`|`, not `||`) transform, batch in, batch of - // variants out, one message either way — keeps the per-message - // synchronisation cost amortised over thousands of lookups instead - // of one to three. `BATCH_SIZE` was originally tuned back when the - // per-k-mer cost inside this stage (minimiser via `RollingStat`) - // was ~3x higher than it is now (see `CanonicalKmerOf::minimizer`, - // a direct O(k) bit-arithmetic replacement) — at the old per-k-mer - // cost, dispatch overhead (the scheduler's single-threaded `Select` - // loop: one channel round-trip per batch) was negligible next to - // the work each batch represented; now that the work itself is - // cheaper, that fixed per-batch overhead is proportionally larger, - // so a bigger batch amortises it over more k-mers again. ───────── - const BATCH_SIZE: usize = 32768; - let n_workers = obisys::effective_parallelism(); - let capacity = 256; - - // Throttling limits how many *batches* are in flight at once — the - // permit is acquired per batch (not per k-mer) in the source - // thread, and released once its `VariantBatch` has been read out of - // the pipeline by the accumulation loop below. See - // `obipipeline::throttle`'s docs for why this is required, not - // optional, once a `Flat`-style stage sits in the pipeline. - // - // Batches stream straight from `unitigs.bin` via - // `enumerate_kmers_batch` — never a full-layer `Vec` collect (a - // layer can hold billions of k-mers; see the "no full collect" - // rule). Each k-mer's own base is seeded straight into the annex in - // this same pass, since this is exactly the iteration order the - // annex is keyed on — no separate seeding pass needed. - let seed_builder = Arc::clone(&builder); - let batches = mphf - .enumerate_kmers_batch(BATCH_SIZE) - .map(move |(start, kmers)| { - kmers - .into_iter() - .enumerate() - .map(|(i, kmer)| { - let base = central_base(kmer, k); - let bits = if fast_mode { - FamilyMask::layer_bits(base, l) - } else { - FamilyMask::presence_bits(base) - }; - seed_builder - .atomic_slot(start + i) - .fetch_or(bits, Ordering::Relaxed); - (start + i, kmer) - }) - .collect::>() - }); - - // Diagnostic: count still-empty annex slots after seeding + cross-partition - // resolution. A non-zero count here means some k-mer's own base was never - // written (should be impossible after the batch_offset fix above). - let check_empty = |label: &str| { - let empty = (0..n).filter(|&slot| builder.get(slot).is_none()).count(); - if empty > 0 { - tracing::warn!( - "{label}: {empty} mask slots still empty after resolution (layer {layer_dir:?})" - ); - } - }; - let throttled = obipipeline::throttle(batches, n_workers).map(|t| SourceBatch { - items: t.item, - _permit: t.guard, - }); - - let pipe = obipipeline::make_pipe! { - SibData : SourceBatch => VariantBatch, - | { - move |batch: SourceBatch| -> VariantBatch { - let mut items = Vec::with_capacity(batch.items.len() * 3); - for (order, kmer) in batch.items { - for variant in kmer.central_canonical_neighbors() { - if variant == kmer { - continue; - } - items.push(( - variant.partition(n_parts), - variant, - order, - central_base(variant, k), - )); - } - } - VariantBatch { items, _permit: batch._permit } - } - } : Batch => Variants, - }; - - // ── Group generated variants by destination partition. `cache` - // holds every partition already mmap'd (no more `open()` cost), but - // `mmap` pages are still loaded on demand and can be evicted — a - // lookup is not free just because the file isn't reopened. Grouping - // keeps one partition's pages hot while its whole batch is resolved, - // instead of faulting pages in and out as lookups jump between - // partitions in whatever order the pipeline happens to produce - // them. Each batch's throttle permit drops here, once accumulated. - let mut outgoing: Vec> = - (0..n_parts).map(|_| Vec::new()).collect(); - for vb in pipe.apply(throttled, n_workers, capacity) { - for (dest_partition, variant, source_order, base) in vb.items { - outgoing[dest_partition].push((variant, source_order, base)); - } - } - - check_empty("after_seeding"); - - // ── Resolve each partition's batch against the cache, parallelised - // over roughly equal-sized *chunks*, not over partitions — a plain - // `outgoing.par_iter()` over `n_parts` buckets gives one thread the - // whole of one partition's bucket, however large, and a lot of - // buckets are far from equal: a central-base substitution changes - // the minimiser (and thus routes to a different partition) only - // when the winning minimiser window overlaps the central position. - // For k=31/m=11 that is ~11 of the 21 possible windows — so *most* - // of the remaining ~10/21 send the variant right back to the - // partition already being built. That self-partition bucket ends up - // far larger than any other, so the naive per-partition split - // leaves one thread grinding through it alone long after every - // other partition's (tiny) bucket is done — confirmed by sampling a - // real run: one thread solely in `MphfLayer::find` while the rest - // of the pool sits idle. Splitting each non-empty bucket into - // `chunk_size`-sized pieces first keeps this pass's per-partition - // locality (each chunk is still contiguous within one partition, - // still resolved via `cache.find` grouped by that partition) while - // letting Rayon spread a single oversized bucket across several - // threads instead of pinning it to one. ────────────────────────── - let chunk_size = ((outgoing.iter().map(Vec::len).sum::() / n_workers).max(1)).min(4096); - let work: Vec<(usize, &[(CanonicalKmer, usize, u8)])> = outgoing - .iter() - .enumerate() - .filter(|(_, q)| !q.is_empty()) - .flat_map(|(dest, q)| q.chunks(chunk_size).map(move |c| (dest, c))) - .collect(); - work.par_iter().for_each(|&(dest, chunk)| { - for &(variant, source_order, base) in chunk { - if let Some(layer) = cache.find(dest, variant) { - let bits = if fast_mode { - FamilyMask::layer_bits(base, layer) - } else { - FamilyMask::presence_bits(base) - }; - builder - .atomic_slot(source_order) - .fetch_or(bits, Ordering::Relaxed); - } - } - }); - - check_empty("after_resolution"); - - // ── Write the layer's annex file, indexed by iteration order — a - // second streamed pass over `unitigs.bin` (via `enumerate_kmers`), - // now that every entry's mask is final; never a `Vec` hold of the - // whole layer. The minorant flag is computed here, not by every - // later reader: this is the one place the whole family's *final* - // mask and this entry's own k-mer (already in hand, no extra - // lookup) are both available together. Every consumer that only - // needs to know "is this k-mer the family's minorant" — the common - // case, since a family is tallied once, at its minorant — reads - // the bit straight back instead of re-deriving it (re-scanning - // `unitigs.bin` and re-hashing through the MPHF, the cost - // `is_minorant` was cheap to *compute* but expensive to *get the - // inputs for* every time). - // Every concurrent phase above is done and its `Arc` clone (the - // seeding pass's `seed_builder`) has already been dropped along - // with the now-fully-drained `batches` iterator — this is the only - // remaining strong reference, so reclaiming plain ownership for the - // sequential `set` calls below is guaranteed to succeed. - let mut builder = Arc::try_unwrap(builder).unwrap_or_else(|_| { - panic!("sibling-annex builder still shared after every concurrent phase completed") - }); - for (order, kmer) in mphf.enumerate_kmers() { - let final_mask = builder.get(order).expect("seeded by construction"); - let minorant = is_minorant(kmer, final_mask, k); - builder.set(order, final_mask.with_minorant(minorant)); - } - builder.close()?; - - Ok(n as u64) -} diff --git a/src/obikphylo/src/siblings_old/cache.rs b/src/obikphylo/src/siblings_old/cache.rs deleted file mode 100644 index b0422cbe..00000000 --- a/src/obikphylo/src/siblings_old/cache.rs +++ /dev/null @@ -1,235 +0,0 @@ -use rayon::prelude::*; - -use obikindex::layer::KmerLayer; -use obikseq::CanonicalKmer; -use obisys::progress_bar; - -use obikindex::{KmerIndex, OKIResult}; - -/// Every partition's already-open layers, built **once** for the whole -/// `build_sibling_annex` run and shared (read-only) across every lookup, in -/// every source layer, for the rest of the run — not reopened/re-mmap'd per -/// query, nor per source layer. -/// -/// Confirmed necessary by sampling a real run: routing lookups through -/// `KmerPartition::query_partition_with` (the same batching `obikmer query` -/// uses) still reopens+re-mmaps every target partition's files on every -/// call, and it is called once per destination partition **per source -/// layer** — for an index with many layers this repeats the same -/// `MphfLayer::open`/`Evidence::open`/`PersistentBitMatrix::open` work over -/// and over. Parallelising those calls (see the gather step below) spread -/// the redundant work across more cores but did not reduce it: sampling -/// showed Rayon workers spending their time inside repeated `open()` -/// syscalls, not computation. This cache amortises that cost to once per -/// partition for the entire run, regardless of how many source layers or -/// lookups follow. -/// -/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the -/// format-erased `Count`/`Presence` handle, not a sibling-specific type: -/// this module used to bundle its own `Mat` enum here, duplicating exactly -/// what `Layer` now does one crate down (see -/// `DevDocMD/implementation/partition_layer_cache.md`). Its -/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`. -pub(super) struct PartitionCache { - mats: Vec>, - /// Whether every `FamilyMask` field in this index's sibling annexes can - /// be trusted as a real layer index (`n_layers <= 7`, the same decision - /// `build_layer_sibling_annex` makes when writing them) — computed once - /// here, from `PartitionMeta::n_layers` (not `mats[p].len()`, which can - /// undercount if a layer failed to open), reused by both the annex - /// writer (`build.rs`) and reader (`family_scan.rs`) so the two never - /// disagree. `n_layers` is guaranteed identical across every partition - /// (a merge adds one layer to all of them at once), so the first - /// non-empty partition's count speaks for the whole index. - fast_mode: bool, -} - -impl PartitionCache { - pub(super) fn build(index: &KmerIndex, n_parts: usize, with_counts: bool) -> OKIResult { - let pb = progress_bar("open_partitions", n_parts as u64, "partitions"); - let built: Vec<(Vec, usize)> = (0..n_parts) - .into_par_iter() - .map(|part| -> OKIResult<(Vec, usize)> { - let index_dir = index.index_dir(part); - if !index_dir.exists() { - pb.inc(1); - return Ok((Vec::new(), 0)); - } - let meta = index.partition_meta(part)?; - let mut mats = Vec::with_capacity(meta.n_layers); - for l in 0..meta.n_layers { - let Ok(mat) = - KmerLayer::open(&index.layer_dir(part, l), &meta.mode, with_counts) - else { - continue; - }; - mats.push(mat); - } - pb.inc(1); - Ok((mats, meta.n_layers)) - }) - .collect::>>()?; - pb.finish_and_clear(); - - let n_layers = built.iter().map(|&(_, n)| n).find(|&n| n > 0).unwrap_or(0); - let fast_mode = n_layers <= 7; - if !fast_mode { - tracing::warn!( - "index has {n_layers} layers (>7) — sibling-annex fast layer lookup disabled; \ - consider compacting this index" - ); - } - - let mats = built.into_iter().map(|(m, _)| m).collect(); - Ok(Self { mats, fast_mode }) - } - - /// See the [`fast_mode`](Self::fast_mode) field docs. - pub(super) fn fast_mode(&self) -> bool { - self.fast_mode - } - - /// Existence lookup of `variant` in partition `dest_partition`: tries - /// each of the partition's already-open layers in turn, stopping at the - /// first hit and reporting which layer it was — `find_slot`, not - /// `sub_matrix`/`carries`, no data read needed for a plain existence - /// check. - pub(super) fn find(&self, dest_partition: usize, variant: CanonicalKmer) -> Option { - self.mats - .get(dest_partition)? - .iter() - .enumerate() - .find_map(|(li, mat)| mat.find_slot(variant).map(|_| li)) - } - - /// Resolve many `(variant, family_idx, base)` queries against one - /// partition's matrices at once, calling `on_hit(family_idx, base, g)` - /// for every genome `g` that carries the resolved variant. - /// - /// Genome-major, not query-major: `PersistentBitMatrix`/ - /// `PersistentCompactIntMatrix` are stored one contiguous block per - /// genome (column), slot as the offset within it (see - /// `obicompactvec::bitmatrix::packed::PackedBitMatrix` — each column's - /// own `mmap` region). Resolving query-by-query (`for query { for genome - /// { mat.carries(genome, slot) } }`, this function's predecessor) visits - /// every genome's block once *per query* — for a batch of thousands of - /// queries against ~90 genomes, that is thousands of jumps into each of - /// ~90 widely separated multi-MB regions, in query order, not genome - /// order: the access pattern a column-major layout is least suited to. - /// Grouping first (by layer, since each layer's matrix is a separate - /// column set) and letting `fill_sub_matrix_carries` sort each group by - /// slot internally, then visiting genome by genome, turns that into ~90 - /// mostly-sequential sweeps through one column's own bytes — the - /// layout's fast axis — confirmed by sampling a real run: - /// `PersistentBitMatrix::get` dominated wall-clock time, mostly blocked - /// on page faults, even after every partition/batch locality fix above - /// this in the traversal. - /// `queries`' 4th element (the annex-recorded layer) is ignored here — - /// every layer is probed with evidence regardless, since this is the - /// `!fast_mode` fallback where that field isn't trustworthy (see - /// [`find_presence_batch_fast`](Self::find_presence_batch_fast)). Same - /// tuple shape as the fast path purely so callers don't need two - /// query representations. - pub(super) fn find_presence_batch( - &self, - dest_partition: usize, - queries: &[(CanonicalKmer, usize, u8, u8)], - n_genomes: usize, - mut on_hit: impl FnMut(usize, u8, usize), - ) { - let Some(mats) = self.mats.get(dest_partition) else { - return; - }; - - // First hit wins, same semantics as the old per-query loop (a - // variant present in an earlier layer shadows later ones). - let mut by_layer: Vec> = vec![Vec::new(); mats.len()]; - for &(variant, family_idx, base, _layer) in queries { - for (li, mat) in mats.iter().enumerate() { - if let Some(slot) = mat.find_slot(variant) { - by_layer[li].push((slot, family_idx, base)); - break; - } - } - } - - for (li, hits) in by_layer.into_iter().enumerate() { - if hits.is_empty() { - continue; - } - resolve_layer_hits(&mats[li], &hits, n_genomes, &mut on_hit); - } - } - - /// Same contract as [`find_presence_batch`](Self::find_presence_batch), - /// but only valid when [`fast_mode`](Self::fast_mode) is true for this - /// whole index: every query's 4th element is trusted outright as its - /// real destination layer (recorded once, evidence-checked, at - /// annex-build time by [`find`](Self::find)) — no per-layer probing, a - /// direct `index_batch` (no evidence) straight to that layer instead. - /// Safe for the same reason `build_layer_sibling_annex`'s own - /// iteration-pipeline lookups are: this exact variant's membership in - /// this exact layer was already positively established once: replaying - /// it via `index`/`index_batch` here isn't a fresh, unverified - /// assumption, it's reusing a fact already paid for. - pub(super) fn find_presence_batch_fast( - &self, - dest_partition: usize, - queries: &[(CanonicalKmer, usize, u8, u8)], - n_genomes: usize, - mut on_hit: impl FnMut(usize, u8, usize), - ) { - let Some(mats) = self.mats.get(dest_partition) else { - return; - }; - - let mut by_layer: Vec> = vec![Vec::new(); mats.len()]; - for &(variant, family_idx, base, layer) in queries { - if let Some(bucket) = by_layer.get_mut(layer as usize) { - bucket.push((variant, family_idx, base)); - } - } - - for (li, entries) in by_layer.into_iter().enumerate() { - if entries.is_empty() { - continue; - } - let mat = &mats[li]; - let variants: Vec = - entries.iter().map(|&(variant, _, _)| variant).collect(); - let slots = mat.hash_batch(&variants); - let hits: Vec<(usize, usize, u8)> = slots - .into_iter() - .zip(entries.iter()) - .map(|(slot, &(_, family_idx, base))| (slot, family_idx, base)) - .collect(); - resolve_layer_hits(mat, &hits, n_genomes, &mut on_hit); - } - } -} - -/// Shared tail of both [`PartitionCache::find_presence_batch`] and -/// [`PartitionCache::find_presence_batch_fast`]: given one layer's already- -/// resolved `(slot, family_idx, base)` hits, sweep genome-major (via -/// `fill_sub_matrix_carries`, buffer-reusing, no per-call allocation) and -/// call `on_hit` for every carry. The two callers differ only in *how* -/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this -/// is everything after that. -fn resolve_layer_hits( - mat: &KmerLayer, - hits: &[(usize, usize, u8)], - n_genomes: usize, - on_hit: &mut impl FnMut(usize, u8, usize), -) { - let n_cols = mat.n_cols().min(n_genomes); - let slots: Vec = hits.iter().map(|&(slot, _, _)| slot).collect(); - let mut carries: Vec> = (0..n_cols).map(|_| Vec::new()).collect(); - mat.fill_sub_matrix_carries(&slots, &mut carries); - for (g, col) in carries.iter().enumerate() { - for (&(_, family_idx, base), &carries_it) in hits.iter().zip(col.iter()) { - if carries_it { - on_hit(family_idx, base, g); - } - } - } -} diff --git a/src/obikphylo/src/siblings_old/cardcomp.rs b/src/obikphylo/src/siblings_old/cardcomp.rs deleted file mode 100644 index dbc87e3d..00000000 --- a/src/obikphylo/src/siblings_old/cardcomp.rs +++ /dev/null @@ -1,336 +0,0 @@ -//! Cardinality × composition decomposition of the 16-state transition cost -//! matrix — replaces `sankoff::build_cost_matrix`'s elementary-edge graph -//! and its Floyd-Warshall shortest-path closure, which double-counts -//! multi-hop transitions once IQ-TREE's own matrix exponential composes -//! them again. See `DevDocMD/theory/evolutionary_distances.md`, "`R` via -//! `exp(-cost)` is wrong for a CTMC; cardinality/composition decomposition". -//! -//! Two small, directly-estimated first-order Markov models (diagonals -//! included — "stay the same" is a real, calibrated outcome, not implicit): -//! a 5-state cardinality model (`0..=4` members of a family) and a 4-state -//! base-composition model (`A,C,G,T`, unambiguous/cardinality-1 loci only — -//! composition bias is a substitution phenomenon, only meaningful when -//! cardinality is conserved). Composed per pair of 16 states via a -//! parsimony-style pairing of the non-shared elements (see -//! `pairwise_transition_matrix`'s doc), then symmetrised — Sankoff -//! parsimony's score is only independent of root placement (as required, -//! since TNT/PhyG search *unrooted* trees) if the cost matrix is -//! symmetric, the discrete-parsimony analogue of CTMC reversibility. - -use super::{BasePairTally, CardinalityTally}; - -/// Row-stochastic 5×5 cardinality transition probabilities (`0..=4`), -/// diagonal included ("stay at the same cardinality"), from -/// [`CardinalityTally`]'s pooled, symmetric co-occurrence counts. -pub fn cardinality_transition_probs(tally: &CardinalityTally) -> [[f64; 5]; 5] { - let mut p = [[0.0f64; 5]; 5]; - for a in 0..5 { - let row_sum: u64 = tally.counts[a].iter().sum(); - if row_sum == 0 { - continue; - } - for b in 0..5 { - p[a][b] = tally.counts[a][b] as f64 / row_sum as f64; - } - } - p -} - -/// Row-stochastic 4×4 base-composition transition probabilities -/// (`A,C,G,T`), diagonal included ("stay the same base"), from -/// [`BasePairTally`]'s pooled substitution (off-diagonal) and agreement -/// (`same`, diagonal) counts — unambiguous, cardinality-1 loci only. -pub fn composition_transition_probs(tally: &BasePairTally) -> [[f64; 4]; 4] { - let mut p = [[0.0f64; 4]; 4]; - for a in 0..4 { - let row_sum = tally.same[a] + (0..4).map(|b| tally.counts[a][b]).sum::(); - if row_sum == 0 { - continue; - } - p[a][a] = tally.same[a] as f64 / row_sum as f64; - for b in 0..4 { - if a != b { - p[a][b] = tally.counts[a][b] as f64 / row_sum as f64; - } - } - } - p -} - -/// Enumerate every injective mapping from `small` (indices `0..small.len()`) -/// into `large` (indices `0..large.len()`, `large.len() >= small.len()`), -/// calling `visit` with each full permutation of chosen `large` indices. -/// `small.len() <= 4` always here (at most 4 bases can be lost or gained), -/// so a plain recursive enumeration (at most `4! = 24` calls) is simplest — -/// not worth a dependency for. -fn for_each_injection(small_len: usize, large_len: usize, visit: &mut dyn FnMut(&[usize])) { - let mut chosen = vec![0usize; small_len]; - let mut used = vec![false; large_len]; - fn go(pos: usize, chosen: &mut [usize], used: &mut [bool], visit: &mut dyn FnMut(&[usize])) { - if pos == chosen.len() { - visit(chosen); - return; - } - for l in 0..used.len() { - if used[l] { - continue; - } - used[l] = true; - chosen[pos] = l; - go(pos + 1, chosen, used, visit); - used[l] = false; - } - } - if small_len == 0 { - visit(&[]); - return; - } - go(0, &mut chosen, &mut used, visit); -} - -/// Parsimony-style pairing cost for the non-shared elements of a -/// transition: pair every element of the smaller of `lost`/`gained` with -/// some element of the larger one, choosing the pairing that minimises -/// total `-ln(P_composition)` (maximises likelihood) — the discrete -/// analogue of "prefer a substitution over an independent loss+gain". -/// Leftover elements of the larger side (`|large|-|small|` of them) are -/// *not* charged here — they're pure cardinality change, already priced by -/// `P_cardinality(|A|→|B|)` in the caller. -fn best_pairing_cost(lost: &[u8], gained: &[u8], p_comp: &[[f64; 4]; 4]) -> f64 { - let (small, large) = if lost.len() <= gained.len() { (lost, gained) } else { (gained, lost) }; - if small.is_empty() { - return 0.0; - } - let mut best = f64::INFINITY; - for_each_injection(small.len(), large.len(), &mut |perm| { - let mut cost = 0.0; - for (i, &li) in perm.iter().enumerate() { - let (x, y) = (small[i], large[li]); - let p = p_comp[x as usize][y as usize]; - cost += if p > 0.0 { -p.ln() } else { f64::INFINITY }; - } - if cost < best { - best = cost; - } - }); - best -} - -/// The full 16-state transition cost matrix, replacing -/// `sankoff::build_cost_matrix`. For every pair of states `A`, `B` -/// (bitmasks, bit `0..4` = `A,C,G,T`): `shared = A∩B` contributes -/// `P_composition(x→x)` per shared base (diagonal — "stayed"); the -/// remaining `lost = A\B`, `gained = B\A` are parsimony-paired into -/// substitutions via [`best_pairing_cost`], and whatever's left over after -/// pairing (only possible on one side, since one side is fully consumed) -/// is *pure* cardinality change, priced once via `P_cardinality(|A|→|B|)` -/// — never chained through intermediate states. Row-normalised (`P(A→·)` -/// sums to `1` over all `B`), converted to a cost via `-ln`, then -/// symmetrised (`(cost(A,B)+cost(B,A))/2` — the row-normalised `P` is not -/// symmetric in general, but a Sankoff parsimony cost must be, so the -/// score is independent of where an unrooted tree gets rooted). -/// -/// `free_loss`: drop the `P_cardinality(|A|→|B|)` factor entirely (never -/// added to `log_p`) — the same low/incomplete-coverage argument that -/// justifies recoding whole-family non-detection as `?` (see -/// `DevDocMD/theory/evolutionary_distances.md`, "Locus dropout under -/// incomplete coverage") applies one level down: whether a genome shows 1 -/// vs 2 (etc.) detected members of a *present* family is exactly as -/// vulnerable to sampling failure as whether the family was detected at -/// all. Without this, `∅`-involving transitions are neutralised (via the -/// `?` recoding, bypassing this matrix's row/column 0 entirely) but -/// cardinality changes *between two otherwise-detected, non-empty* states -/// (e.g. `{A} -> {A,C}`) still carried the same calibrated -/// `P_cardinality` penalty as any other gain/loss — inconsistent with -/// `--free-loss`'s own rationale. With the factor dropped, cost is driven -/// only by composition matching (shared-base retention and paired -/// substitutions), never by a state pair's cardinality difference alone. -pub fn pairwise_cost_matrix(p_card: &[[f64; 5]; 5], p_comp: &[[f64; 4]; 4], free_loss: bool) -> [[f64; 16]; 16] { - let mut log_p = [[0.0f64; 16]; 16]; // ln(P), *before* row-normalisation - for a in 0u8..16 { - for b in 0u8..16 { - let shared = a & b; - let lost: Vec = (0..4).filter(|&i| a & (1 << i) != 0 && b & (1 << i) == 0).collect(); - let gained: Vec = (0..4).filter(|&i| b & (1 << i) != 0 && a & (1 << i) == 0).collect(); - - let mut lp = 0.0; // accumulate ln(P), so 0.0 = probability 1 - if !free_loss { - let card_a = a.count_ones() as usize; - let card_b = b.count_ones() as usize; - let p_c = p_card[card_a][card_b]; - lp += if p_c > 0.0 { p_c.ln() } else { f64::NEG_INFINITY }; - } - - for i in 0..4u8 { - if shared & (1 << i) != 0 { - let p = p_comp[i as usize][i as usize]; - lp += if p > 0.0 { p.ln() } else { f64::NEG_INFINITY }; - } - } - lp -= best_pairing_cost(&lost, &gained, p_comp); - - log_p[a as usize][b as usize] = lp; - } - } - - // Row-normalise and convert to cost entirely in log-space - // (log-sum-exp), never exponentiating a raw `log_p` value directly — - // for a transition reachable only through several low-probability - // steps, `log_p.exp()` can underflow to exactly `0.0` (anything below - // roughly `-709` does, in `f64`), silently turning a real, if small, - // probability into a hard `+∞` cost. Observed in practice: a - // calibrated matrix with many such exact-zero entries is numerically - // unstable for IQ-TREE's own likelihood-derivative computation - // ("Numerical underflow for lh-derivative"). `log_sum_exp` never - // exponentiates anything above `0` (every term is shifted by the - // row's own max first), so it stays accurate across the full range - // `f64` can represent, not just what survives a direct `exp()`. - let mut cost = [[0.0f64; 16]; 16]; - for a in 0..16 { - let row_max = log_p[a].iter().cloned().fold(f64::NEG_INFINITY, f64::max); - if row_max == f64::NEG_INFINITY { - // Every transition out of this state has probability 0 in the - // calibration data — genuinely unreachable, not underflow. - cost[a] = [f64::INFINITY; 16]; - continue; - } - let log_sum = row_max + log_p[a].iter().map(|&lp| (lp - row_max).exp()).sum::().ln(); - for b in 0..16 { - // `log_sum - log_p[a][b]` is `+∞` automatically when - // `log_p[a][b] == -∞` (IEEE 754: finite − (−∞) = +∞) — no - // separate branch needed for a genuinely zero-probability - // transition. - cost[a][b] = log_sum - log_p[a][b]; - } - } - - // Symmetrise (see doc comment: parsimony on an unrooted tree requires - // a symmetric cost matrix). - let mut sym = [[0.0f64; 16]; 16]; - for a in 0..16 { - for b in 0..16 { - sym[a][b] = if a == b { 0.0 } else { (cost[a][b] + cost[b][a]) / 2.0 }; - } - } - sym -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cardinality_probs_row_normalised() { - let mut counts = [[0u64; 5]; 5]; - counts[0][0] = 10; - counts[0][1] = 5; - counts[1][0] = 5; - counts[1][1] = 2; - let tally = CardinalityTally { counts }; - let p = cardinality_transition_probs(&tally); - assert!((p[0].iter().sum::() - 1.0).abs() < 1e-12); - assert!((p[1].iter().sum::() - 1.0).abs() < 1e-12); - assert!((p[0][0] - 10.0 / 15.0).abs() < 1e-12); - } - - #[test] - fn composition_probs_include_diagonal() { - let mut tally = BasePairTally { counts: [[0u64; 4]; 4], same: [0u64; 4] }; - tally.same[0] = 90; // A stays A 90 times - tally.counts[0][1] = 10; // A -> C 10 times - tally.counts[1][0] = 10; - let p = composition_transition_probs(&tally); - assert!((p[0][0] - 0.9).abs() < 1e-12); - assert!((p[0][1] - 0.1).abs() < 1e-12); - assert!((p[0].iter().sum::() - 1.0).abs() < 1e-12); - } - - #[test] - fn pairwise_cost_matrix_is_symmetric_and_zero_diagonal() { - // A plausible-looking pair of models (not calibrated from real - // data — just needs every state reachable with nonzero probability - // for this structural test). - let p_card = [ - [0.5, 0.3, 0.15, 0.04, 0.01], - [0.3, 0.4, 0.2, 0.08, 0.02], - [0.15, 0.2, 0.4, 0.2, 0.05], - [0.04, 0.08, 0.2, 0.5, 0.18], - [0.01, 0.02, 0.05, 0.18, 0.74], - ]; - let p_comp = [ - [0.7, 0.1, 0.15, 0.05], - [0.1, 0.7, 0.05, 0.15], - [0.15, 0.05, 0.7, 0.1], - [0.05, 0.15, 0.1, 0.7], - ]; - let cost = pairwise_cost_matrix(&p_card, &p_comp, false); - for a in 0..16 { - assert_eq!(cost[a][a], 0.0); - for b in 0..16 { - assert!((cost[a][b] - cost[b][a]).abs() < 1e-9, "cost[{a}][{b}]={} cost[{b}][{a}]={}", cost[a][b], cost[b][a]); - } - } - } - - #[test] - fn free_loss_ignores_cardinality_transition_probs() { - // A skewed cardinality model (cardinality change made artificially - // expensive) must have zero effect on the cost matrix once - // `free_loss` is set — the whole point of the flag. - let p_card_uniform = [[0.2f64; 5]; 5]; - let p_card_skewed = [ - [0.96, 0.01, 0.01, 0.01, 0.01], - [0.01, 0.96, 0.01, 0.01, 0.01], - [0.01, 0.01, 0.96, 0.01, 0.01], - [0.01, 0.01, 0.01, 0.96, 0.01], - [0.01, 0.01, 0.01, 0.01, 0.96], - ]; - let p_comp = [ - [0.7, 0.1, 0.15, 0.05], - [0.1, 0.7, 0.05, 0.15], - [0.15, 0.05, 0.7, 0.1], - [0.05, 0.15, 0.1, 0.7], - ]; - let cost_uniform = pairwise_cost_matrix(&p_card_uniform, &p_comp, true); - let cost_skewed = pairwise_cost_matrix(&p_card_skewed, &p_comp, true); - for a in 0..16 { - for b in 0..16 { - assert!( - (cost_uniform[a][b] - cost_skewed[a][b]).abs() < 1e-9, - "cost[{a}][{b}] differs between cardinality models under free_loss: {} vs {}", - cost_uniform[a][b], cost_skewed[a][b], - ); - } - } - } - - #[test] - fn underflow_prone_transition_gets_finite_cost_not_infinite() { - // Every off-diagonal composition probability is tiny (1e-200) but - // not exactly zero — small enough that the naive `log_p.exp()` - // (exponentiate before row-normalising) hard-underflows to `0.0` - // in `f64` well before the probability is truly zero, silently - // turning a real, if minuscule, probability into `+∞` cost. The - // log-sum-exp normalisation in `pairwise_cost_matrix` must keep - // this finite instead. - let p_card = [[0.2f64; 5]; 5]; - let tiny = 1e-200; - let mut p_comp = [[tiny; 4]; 4]; - for i in 0..4 { - p_comp[i][i] = 1.0 - 3.0 * tiny; - } - - let cost = pairwise_cost_matrix(&p_card, &p_comp, false); - - // {A,C} -> {G,T}: no shared bases, both members substituted — the - // shape most prone to underflow (several tiny-probability factors - // multiplied together). - let a = 0b0011u8; // A, C - let b = 0b1100u8; // G, T - assert!( - cost[a as usize][b as usize].is_finite(), - "cost must stay finite for a merely tiny (not exactly zero) probability, got {}", - cost[a as usize][b as usize] - ); - } -} diff --git a/src/obikphylo/src/siblings_old/cardinality.rs b/src/obikphylo/src/siblings_old/cardinality.rs deleted file mode 100644 index bc16372a..00000000 --- a/src/obikphylo/src/siblings_old/cardinality.rs +++ /dev/null @@ -1,132 +0,0 @@ -use std::sync::Arc; - -use ndarray::Array2; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::distance::RawSnpDistanceOutput; -use super::family_scan::{Selection, scan_layer_families}; - -/// See [`KmerIndex::cardinality_tally`]. -pub struct CardinalityTally { - /// `counts[a][b] == counts[b][a]` = number of family sites, pooled over - /// included genome pairs, where one genome's family cardinality - /// (popcount of its presence mask, `0..=4`) is `a` and the other's is - /// `b`. Diagonal is real data here (both genomes at the same - /// cardinality), unlike [`super::distance::BasePairTally::counts`]. - pub counts: [[u64; 5]; 5], -} - -/// Adds [`cardinality_tally`](Self::cardinality_tally) to `KmerIndex`. -pub trait CardinalityExt { - /// Cardinality co-occurrence, pooled only over genome pairs whose - /// overall SNP ratio in `raw` is at or below `ratio_ceiling` — same - /// saturation/no-data exclusion discipline as - /// [`base_pair_tally`](Self::base_pair_tally). Unlike - /// `scan_family_pairs` (which resolves each genome to a single form and - /// silently drops any genome carrying more than one member of the - /// family), this needs the *full* per-genome presence mask — a family - /// member count of 2, 3 or 4 is exactly the signal being tallied, not - /// noise to discard — so it re-implements the traversal rather than - /// reusing that helper. - /// - /// Restricted to variable families (`family_size() >= 2`), matching - /// `snp_pseudo_alignment`'s own scope — briefly removed, then - /// reinstated: without it, the diagonal is dominated by genome-wide - /// invariant background (family_size()<2 loci vastly outnumber the - /// ones that ever vary anywhere), which is inconsistent with the - /// `+ASC`-corrected alignment this matrix is ultimately used with — - /// `+ASC` exists specifically because the likelihood only ever sees - /// variable sites, so a rate model calibrated mostly from invariant - /// background sites doesn't describe the population it's applied to. - /// Verified empirically: removing the filter measurably worsened a - /// real IQ-TREE run (log-likelihood dropped, `NNI search needs - /// unusual large number of steps to converge` warnings appeared) — see - /// `DevDocMD/theory/evolutionary_distances.md` for the full account. - /// [`base_pair_tally`](Self::base_pair_tally)'s own diagonal (`same`) - /// gets the matching restriction via `scan_family_pairs`'s new - /// `variable` flag, rather than a `family_size()` check of its own (it - /// doesn't have direct access to the family's mask). - fn cardinality_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult; -} - -impl CardinalityExt for KmerIndex { - fn cardinality_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| { - if i == j { - return false; - } - let snp = raw.snp[[i, j]]; - let total = snp + raw.shared[[i, j]]; - total > 0 && (snp as f64 / total as f64) <= ratio_ceiling - }); - - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - - let pb = progress_bar("cardinality_tally", layer_dirs.len() as u64, "layers"); - // One layer at a time — see `snp_pseudo_alignment`'s comment for why - // `par_iter()` over layers would defeat `scan_layer_families`'s - // partition-grouped locality. Tallied straight into `total` as each - // family comes back — no per-layer buffer. - let mut total = [[0u64; 5]; 5]; - for layer_dir in &layer_dirs { - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::All, - |_family_idx, mask, genome_mask| { - if mask.family_size() < 2 { - // Fully invariant family (never varies anywhere in - // the index) — genome-wide background, not - // SNP-adjacent signal; would otherwise swamp the - // diagonal (`c=1/c=1` etc.), which needs to reflect - // the same variable-families-only population the - // `+ASC`-corrected alignment/likelihood actually - // models. See `base_pair_tally`'s `variable` gate - // on its own `same` diagonal for the matching fix. - return; - } - - for i in 0..n_genomes { - let card_i = genome_mask[i].count_ones() as usize; - for j in (i + 1)..n_genomes { - if !included[[i, j]] { - continue; - } - let card_j = genome_mask[j].count_ones() as usize; - total[card_i][card_j] += 1; - if card_i != card_j { - total[card_j][card_i] += 1; - } - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(CardinalityTally { counts: total }) - } -} diff --git a/src/obikphylo/src/siblings_old/distance.rs b/src/obikphylo/src/siblings_old/distance.rs deleted file mode 100644 index abd92012..00000000 --- a/src/obikphylo/src/siblings_old/distance.rs +++ /dev/null @@ -1,244 +0,0 @@ -use std::sync::Arc; - -use ndarray::Array2; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::family_scan::{Selection, scan_layer_families}; - -/// Raw p-distance restricted to loci that are single-copy in **both** -/// genomes of a pair — the "stringent / paralogy-aware" locus eligibility -/// rule (`DevDocMD/theory/evolutionary_distances.md`, "Locus eligibility"), -/// without the JC/K2P/LogDet finalisation math: `snp[i,j]` + `shared[i,j]` -/// is the number of eligible loci, `snp[i,j] / (snp[i,j] + shared[i,j])` is -/// `p_hat`. A quick, self-contained way to sanity-check the estimator -/// against a real index before the full `SnpTally` design is built. -/// -/// A locus (family, tallied once at its minorant) is eligible for pair -/// `(i, j)` iff genome `i` carries exactly one of the family's observed -/// forms **and** genome `j` carries exactly one (possibly a different one) -/// — presence-only: a genome carrying the same form twice (a same-allele -/// duplicate) is indistinguishable from carrying it once when only a -/// presence matrix is available, so such cases are not excluded here even -/// when a count index exists. See "Locus eligibility", stringent rule, for -/// why this matters and how a count index would close the gap — left as a -/// follow-up, not applied here. -pub struct RawSnpDistanceOutput { - /// n×n count of eligible loci where the two genomes' single forms differ. - pub snp: Array2, - /// n×n count of eligible loci where the two genomes' single forms agree. - pub shared: Array2, -} - -/// Shared traversal behind [`DistanceExt::raw_snp_distance`] and -/// [`DistanceExt::base_pair_tally`]: for every family (tallied once, at its -/// minorant) of every layer of the already-built sibling annex, resolves -/// each genome's single observed form (`None` if absent or -/// ambiguous/multi-copy), then calls `on_pair(acc, i, j, bi, bj, variable)` -/// for every genome pair `(i, j)` where both are unambiguous and -/// single-copy (`bi == bj` means shared at that locus, `bi != bj` means a -/// SNP). `variable` is the family's own `family_size() >= 2` (true if more -/// than one member is observed *anywhere* in the family, i.e. it isn't -/// fully invariant across the whole index) — `raw_snp_distance` ignores it -/// (a fully-invariant family is still legitimately "shared"), but callers -/// whose diagonal should only reflect genuine SNP-adjacent agreement, not -/// the genome-wide invariant background, need it (see -/// [`DistanceExt::base_pair_tally`]'s `same` field). Layers are processed -/// one at a time, not in parallel — see `snp_pseudo_alignment`'s comment -/// for why `par_iter()` over layers would defeat `scan_layer_families`'s -/// partition-grouped locality; each layer gets its own accumulator from -/// `zero()`, combined pairwise via `combine`. A free function, not a -/// `KmerIndex` method — called only from this file's `DistanceExt` impl. -fn scan_family_pairs( - index: &KmerIndex, - label: &str, - zero: impl Fn() -> Acc + Sync, - on_pair: F, - combine: C, -) -> OKIResult -where - Acc: Send, - F: Fn(&mut Acc, usize, usize, u8, u8, bool) + Sync, - C: Fn(Acc, Acc) -> Acc, -{ - let n_parts = index.n_partitions(); - let n_genomes = index.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = index.meta().config.with_counts; - let k = index.kmer_size(); - let cache = Arc::new(PartitionCache::build(index, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(index)?; - - let pb = progress_bar(label, layer_dirs.len() as u64, "layers"); - // One layer at a time — see `snp_pseudo_alignment`'s comment for why - // `par_iter()` over layers would defeat `scan_layer_families`'s - // partition-grouped locality. - let mut total = zero(); - for layer_dir in &layer_dirs { - let mut acc = zero(); - - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::All, - |_family_idx, mask, genome_mask| { - let variable = mask.family_size() >= 2; - - // Per genome: which single form (if exactly one) it - // carries — `None` (a `popcount != 1` mask) once a - // second form is seen, ambiguous/not single-copy, - // ineligible for either side of a pair. - let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - - for i in 0..n_genomes { - let Some(bi) = single_form(i) else { continue }; - for j in (i + 1)..n_genomes { - let Some(bj) = single_form(j) else { continue }; - on_pair(&mut acc, i, j, bi, bj, variable); - } - } - }, - )?; - - total = combine(total, acc); - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(total) -} - -/// Adds [`raw_snp_distance`](Self::raw_snp_distance) and -/// [`base_pair_tally`](Self::base_pair_tally) to `KmerIndex`. -pub trait DistanceExt { - /// Compute [`RawSnpDistanceOutput`] from an already-built sibling annex - /// (run [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) first). - fn raw_snp_distance(&self) -> OKIResult; - - /// Symmetric 6-category base-pair substitution tally (AC, AG, AT, CG, - /// CT, GT — indexed `0=A,1=C,2=G,3=T`), pooled only over genome pairs - /// whose overall SNP ratio in `raw` is at or below `ratio_ceiling` — - /// same saturation-exclusion discipline as - /// [`cardinality_tally`](super::cardinality::CardinalityExt::cardinality_tally), - /// for the same reason: a saturated pair's observed base-pair mix - /// trends toward neutral base composition, not the true point-mutation - /// spectrum. - /// - /// A second full pass over the annex, sharing - /// [`raw_snp_distance`](Self::raw_snp_distance)'s traversal (guided by - /// it, not a blind re-scan) — needed because `raw_snp_distance` only - /// keeps aggregate SNP/shared counts per genome pair, not which bases - /// were actually involved at each locus, and the ratio-ceiling filter - /// can only be evaluated once the aggregate counts are known. - fn base_pair_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult; -} - -impl DistanceExt for KmerIndex { - fn raw_snp_distance(&self) -> OKIResult { - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let (snp, shared) = scan_family_pairs( - self, - "raw_snp_distance", - || { - ( - Array2::::zeros((n_genomes, n_genomes)), - Array2::::zeros((n_genomes, n_genomes)), - ) - }, - |(snp, shared), i, j, bi, bj, _variable| { - if bi == bj { - shared[[i, j]] += 1; - shared[[j, i]] += 1; - } else { - snp[[i, j]] += 1; - snp[[j, i]] += 1; - } - }, - |(mut snp, mut shared), (s, sh)| { - snp += &s; - shared += &sh; - (snp, shared) - }, - )?; - Ok(RawSnpDistanceOutput { snp, shared }) - } - - fn base_pair_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult { - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| { - if i == j { - return false; - } - let snp = raw.snp[[i, j]]; - let total = snp + raw.shared[[i, j]]; - total > 0 && (snp as f64 / total as f64) <= ratio_ceiling - }); - - let (counts, same) = scan_family_pairs( - self, - "base_pair_tally", - || ([[0u64; 4]; 4], [0u64; 4]), - |(counts, same), i, j, bi, bj, variable| { - if !included[[i, j]] { - return; - } - if bi != bj { - counts[bi as usize][bj as usize] += 1; - counts[bj as usize][bi as usize] += 1; - } else if variable { - // Only count "stayed the same" from families that vary - // *somewhere* in the index — a fully invariant family - // (never varies anywhere) isn't a SNP-adjacent - // agreement, it's genome-wide background, and would - // otherwise swamp the diagonal (see - // `DevDocMD/theory/evolutionary_distances.md`, the - // ascertainment-bias regression this was reverting). - same[bi as usize] += 1; - } - }, - |(mut counts, mut same), (partial_counts, partial_same)| { - for a in 0..4 { - same[a] += partial_same[a]; - for b in 0..4 { - counts[a][b] += partial_counts[a][b]; - } - } - (counts, same) - }, - )?; - Ok(BasePairTally { counts, same }) - } -} - -/// See [`DistanceExt::base_pair_tally`]. -pub struct BasePairTally { - /// `counts[a][b] == counts[b][a]` = number of eligible loci, pooled - /// over included genome pairs, where the two genomes' single forms are - /// `a` and `b` (0=A, 1=C, 2=G, 3=T). Diagonal always `0` — an `a == b` - /// locus is counted in `same`, not here. - pub counts: [[u64; 4]; 4], - /// `same[a]` = number of eligible loci, pooled over included genome - /// pairs, where both genomes' single forms are `a` — the diagonal - /// `counts` omits, needed to build a proper row-stochastic composition - /// probability matrix (the "stay the same base" entries), not just the - /// substitution-cost off-diagonal. - pub same: [u64; 4], -} diff --git a/src/obikphylo/src/siblings_old/entropy.rs b/src/obikphylo/src/siblings_old/entropy.rs deleted file mode 100644 index 5bead80c..00000000 --- a/src/obikphylo/src/siblings_old/entropy.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! `--shannon` — per-family Shannon entropy over the 15 non-empty subsets -//! of `{A,C,G,T}`. See `DevDocMD/architecture/siblings.md`, "Entropy -//! definition — settled 2026-08-15" and "`--subsample`/`--shannon`", for -//! the design discussion this implements: the project's calibrated 16-state -//! Sankoff cost matrix (`cardcomp::pairwise_cost_matrix`) treats every -//! observed combination of bases as its own first-class state, so entropy -//! over that same 15-symbol space (state `0`/`∅` excluded — a family is -//! only informative among the genomes where it is actually observed) is the -//! consistent informativeness proxy for this project, not a 4-symbol -//! reduction. - -use std::io::{BufWriter, Write}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::entropy_annex::{ENTROPY_ANNEX_FILE_NAME, EntropyAnnexBuilder}; -use super::family_scan::{Selection, scan_layer_families}; -use super::subsample::EntropyBias; - -/// Shannon entropy (bits) of one family's per-genome states, over the 15 -/// non-empty subsets of `{A,C,G,T}` actually observed among the genomes -/// carrying it (`genome_mask[g] != 0`) — genomes where the family is absent -/// are excluded from the denominator, not scored as a 16th state. `None` -/// only if the family is absent from every genome (shouldn't happen for a -/// real minorant, guarded against rather than dividing by zero). Second -/// value: the denominator (`n_genomes_present`) used, for traceability. -pub(super) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> { - let mut freq = [0u32; 16]; // index 0 (∅) never incremented, kept for direct `m as usize` indexing - let mut present = 0u32; - for &m in genome_mask { - if m != 0 { - freq[m as usize] += 1; - present += 1; - } - } - if present == 0 { - return None; - } - let mut h = 0.0f64; - for &c in &freq[1..] { - if c == 0 { - continue; - } - let p = f64::from(c) / f64::from(present); - h -= p * p.log2(); - } - Some((h, present as usize)) -} - -/// Shannon entropy (bits) of one family's per-genome states, reduced to the -/// 4 plain nucleotide symbols instead of the 15-state space above — kept -/// alongside it (not instead of it) purely to measure how much the two -/// diverge on real data, per the discussion in -/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`". A genome -/// carrying several bases at once (`genome_mask[g]` with more than one bit -/// set) contributes to *each* of those bases' counts — counted once per -/// base present, not split fractionally, and not folded into a combined -/// state the way [`family_entropy`] does: a genome polymorphic for the -/// family is, by construction, present at more than one base, so it simply -/// counts more than once here. The denominator is therefore the total -/// number of base-occurrences summed over every present genome, not the -/// genome count — the two coincide only when no genome carries more than -/// one base. `None` under the same condition as [`family_entropy`]. -pub(super) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> { - let mut freq = [0u32; 4]; - let mut total = 0u32; - for &m in genome_mask { - for (base, count) in freq.iter_mut().enumerate() { - if m & (1 << base) != 0 { - *count += 1; - total += 1; - } - } - } - if total == 0 { - return None; - } - let mut h = 0.0f64; - for &c in &freq { - if c == 0 { - continue; - } - let p = f64::from(c) / f64::from(total); - h -= p * p.log2(); - } - Some((h, total as usize)) -} - -/// Adds [`shannon_entropy_csv`](Self::shannon_entropy_csv) to `KmerIndex`. -pub trait ShannonEntropyExt { - /// Writes a CSV - /// (`layer,family_idx,entropy15,entropy4,family_size,n_genomes_present`) - /// of per-family Shannon entropy — both [`family_entropy`] (15 non-empty - /// states, the project's settled definition) and [`family_entropy_4`] - /// (plain 4-symbol reduction, kept alongside for comparison) — one row - /// per non-monomorphic minorant visited, from an already-built sibling - /// annex (run - /// [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) - /// first). `layer` is this layer's ordinal among every layer carrying an - /// annex (not a partition/layer pair — an opaque but stable id, unique - /// together with `family_idx`); `family_idx` is the family's - /// iteration-order index within that layer, the same numbering - /// `--subsample`'s selection is drawn from. - /// - /// `subsample`/`entropy_bias`: same meaning as - /// [`SnpAlignmentExt::snp_pseudo_alignment`](super::SnpAlignmentExt::snp_pseudo_alignment)'s - /// — `subsample = None`, `entropy_bias = None` streams every - /// non-monomorphic minorant of the whole index (a time cost, not a - /// memory one: entropy is computed and written per-family as soon as it - /// resolves); `subsample = Some(n)` bounds the index's cross-partition - /// resolution work to ~`n` sampled families; `entropy_bias = Some(_)` - /// weights that sample by its own entropy (see - /// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection") — - /// useful here specifically to see the biased sample's own entropy - /// distribution, not just to speed up a distance computation. - fn shannon_entropy_csv( - &self, - path: &Path, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult<()>; -} - -impl ShannonEntropyExt for KmerIndex { - fn shannon_entropy_csv( - &self, - path: &Path, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult<()> { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - let selections = - super::subsample::compute_selections(self, &layer_dirs, subsample, entropy_bias)?; - - let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?); - writeln!( - f, - "layer,family_idx,entropy15,entropy4,family_size,n_genomes_present" - ) - .map_err(OKIError::Io)?; - - let pb = progress_bar("shannon_entropy", layer_dirs.len() as u64, "layers"); - for (layer_ord, (layer_dir, layer_selection)) in - layer_dirs.iter().zip(selections.iter()).enumerate() - { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - let mut write_err = None; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |family_idx, mask, genome_mask| { - if write_err.is_some() { - return; - } - if mask.family_size() < 2 { - return; // monomorphic family — entropy trivially 0, no signal, skip - } - let Some((h15, n_present)) = family_entropy(genome_mask) else { - return; - }; - let h4 = family_entropy_4(genome_mask).map_or(0.0, |(h, _)| h); - if let Err(e) = writeln!( - f, - "{layer_ord},{family_idx},{h15:.6},{h4:.6},{},{n_present}", - mask.family_size() - ) { - write_err = Some(e); - } - }, - )?; - if let Some(e) = write_err { - return Err(OKIError::Io(e)); - } - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(()) - } -} - -/// Builds the entropy annex (`entropy_annex.rs`) for every layer in -/// `layer_dirs` that doesn't already have one — a no-op, cheap existence -/// check per layer, once every layer's annex file exists. Called from -/// [`super::subsample::compute_selections`] whenever entropy-biased -/// selection is requested; see `DevDocMD/architecture/siblings.md`, -/// "Entropy-biased selection", for why this is the one place that pays a -/// scan (every non-monomorphic minorant's `genome_mask` must be resolved -/// to know its entropy) — every later read of the resulting file is a -/// plain positional mmap lookup, not a re-scan. -/// -/// Bounded to non-monomorphic minorants only -/// ([`super::subsample::non_monomorphic_selection_layer`], a cheap -/// annex-only pass — no cross-partition resolution), not -/// `Selection::All`: monomorphism (`family_size() < 2`) is already known -/// from the annex bits alone, so there is no reason to pay -/// `scan_layer_families`'s expensive per-genome resolution for the ~98% -/// of minorants that are monomorphic only to discard the result. -pub(super) fn ensure_entropy_annexes(index: &KmerIndex, layer_dirs: &[PathBuf]) -> OKIResult<()> { - let missing: Vec = layer_dirs - .iter() - .enumerate() - .filter(|(_, dir)| !dir.join(ENTROPY_ANNEX_FILE_NAME).exists()) - .map(|(i, _)| i) - .collect(); - if missing.is_empty() { - return Ok(()); - } - - let n_parts = index.n_partitions(); - let n_genomes = index.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = index.meta().config.with_counts; - let k = index.kmer_size(); - let cache = Arc::new(PartitionCache::build(index, n_parts, with_counts)?); - let minorant_counts = super::subsample::minorant_counts(layer_dirs)?; - - let pb = progress_bar("entropy_annex_build", missing.len() as u64, "layers"); - for &li in &missing { - let layer_dir = &layer_dirs[li]; - let mut builder = EntropyAnnexBuilder::new( - minorant_counts[li] as usize, - &layer_dir.join(ENTROPY_ANNEX_FILE_NAME), - ) - .map_err(OKIError::Io)?; - let eligible = super::subsample::non_monomorphic_selection_layer(layer_dir)?; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::Some(&eligible), - |family_idx, mask, genome_mask| { - debug_assert!( - mask.family_size() >= 2, - "Selection::Some(eligible) must only visit non-monomorphic minorants" - ); - if let Some((h15, _)) = family_entropy(genome_mask) { - builder.set(family_idx, h15 as f32); - } - }, - )?; - builder.close().map_err(OKIError::Io)?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(()) -} diff --git a/src/obikphylo/src/siblings_old/entropy_annex.rs b/src/obikphylo/src/siblings_old/entropy_annex.rs deleted file mode 100644 index 1ac2457a..00000000 --- a/src/obikphylo/src/siblings_old/entropy_annex.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! Persisted, per-layer Shannon entropy (`entropy15`, see `entropy.rs`), -//! one value per **minorant family**, indexed by `family_idx` — the same -//! numbering `scan_layer_families`/`Selection`/`subsample::reservoir_sample_layer` -//! already use (every minorant of the layer, monomorphic ones included, -//! counted in iteration order; see `subsample.rs`'s module docs for why -//! `family_idx` and the raw annex slot are kept as two different counters). -//! -//! Exists so entropy-biased selection (`--entropy`/`--entropy-sd`) can -//! weight candidates without re-resolving each one's `genome_mask` (the -//! expensive cross-partition step) on every run — see -//! `DevDocMD/architecture/siblings.md`, "Entropy-biased selection": entropy is -//! only knowable once presence is resolved, so *building* this file still -//! costs a full, unsampled scan the first time (`ensure_entropy_annexes` in -//! `entropy.rs`), but every subsequent read is a plain positional mmap -//! lookup, restoring `Selection::Some`'s usual "skip resolving excluded -//! families entirely" speedup for every run after the first. -//! -//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat -//! per-entry data, read-only after build) but is not slot-indexed the way -//! `SiblingAnnex` is: `SiblingAnnex` is indexed by every k-mer of the layer -//! (needed because any slot might turn out to be a minorant), while this -//! file is built *from* an already-open `SiblingAnnex`, so it only ever -//! needs to be as long as the layer's minorant count — monomorphic -//! minorants still get an entry (a `-1.0` sentinel, entropy not meaningful -//! for them, never read), keeping the same family_idx numbering every -//! other consumer uses, but non-minorant k-mers (the vast majority of -//! slots) are never represented here at all. - -use std::fs::{File, OpenOptions}; -use std::io; -use std::path::Path; - -use memmap2::{Mmap, MmapMut}; - -const MAGIC: [u8; 4] = *b"PENT"; - -// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows. -const HEADER_SIZE: usize = 16; - -/// Sentinel: monomorphic minorant (no signal, never sampled) or otherwise -/// not applicable. Entropy is always `>= 0.0`, so any negative value is -/// unambiguous — chosen over `NaN` to avoid `NaN`-comparison footguns at -/// every read site. -const SENTINEL: f32 = -1.0; - -pub(super) const ENTROPY_ANNEX_FILE_NAME: &str = "entropy.pent"; - -pub(super) struct EntropyAnnex { - mmap: Mmap, - n: usize, -} - -impl EntropyAnnex { - pub(super) fn open(path: &Path) -> io::Result { - let mmap = unsafe { Mmap::map(&File::open(path)?)? }; - if mmap.len() < HEADER_SIZE { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PENT file too short")); - } - if mmap[0..4] != MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PENT magic")); - } - let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; - if mmap.len() < HEADER_SIZE + n * 4 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PENT file truncated")); - } - Ok(Self { mmap, n }) - } - - pub(super) fn len(&self) -> usize { self.n } - - /// `None` for the sentinel (monomorphic minorant, or not yet computed). - pub(super) fn get(&self, family_idx: usize) -> Option { - let off = HEADER_SIZE + family_idx * 4; - let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap()); - (v >= 0.0).then_some(v) - } -} - -pub(super) struct EntropyAnnexBuilder { - mmap: MmapMut, -} - -impl EntropyAnnexBuilder { - /// Create a new annex of `n` entries (a layer's total minorant count) at - /// `path`, pre-initialised to the sentinel — entries never written - /// (monomorphic minorants) stay the sentinel, not an accidental `0.0` - /// (a real, achievable entropy value). - pub(super) fn new(n: usize, path: &Path) -> io::Result { - let file_size = HEADER_SIZE + n * 4; - let file = OpenOptions::new() - .read(true).write(true).create(true).truncate(true) - .open(path)?; - file.set_len(file_size as u64)?; - let mut mmap = unsafe { MmapMut::map_mut(&file)? }; - mmap[0..4].copy_from_slice(&MAGIC); - mmap[4..8].copy_from_slice(&[0u8; 4]); - mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); - for i in 0..n { - let off = HEADER_SIZE + i * 4; - mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes()); - } - Ok(Self { mmap }) - } - - pub(super) fn set(&mut self, family_idx: usize, value: f32) { - debug_assert!(value >= 0.0, "entropy must be non-negative, got {value}"); - let off = HEADER_SIZE + family_idx * 4; - self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes()); - } - - pub(super) fn close(self) -> io::Result<()> { - self.mmap.flush() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn sentinel_is_negative_and_unset_entries_read_as_none() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let builder = EntropyAnnexBuilder::new(4, &path).unwrap(); - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - for i in 0..4 { - assert_eq!(annex.get(i), None); - } - } - - #[test] - fn roundtrip_including_zero_entropy() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let mut builder = EntropyAnnexBuilder::new(4, &path).unwrap(); - let values = [0.0f32, 1.5, 3.9, 0.0001]; - for (i, &v) in values.iter().enumerate() { - builder.set(i, v); - } - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - for (i, &v) in values.iter().enumerate() { - assert_eq!(annex.get(i), Some(v), "entry {i}"); - } - } - - #[test] - fn mixed_sentinel_and_real_values() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let mut builder = EntropyAnnexBuilder::new(3, &path).unwrap(); - builder.set(1, 2.0); - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - assert_eq!(annex.get(0), None); - assert_eq!(annex.get(1), Some(2.0)); - assert_eq!(annex.get(2), None); - assert_eq!(annex.len(), 3); - } -} diff --git a/src/obikphylo/src/siblings_old/family_scan.rs b/src/obikphylo/src/siblings_old/family_scan.rs deleted file mode 100644 index 59d4d368..00000000 --- a/src/obikphylo/src/siblings_old/family_scan.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! Shared per-layer family traversal, used by every sibling-annex consumer -//! (`snp_pseudo_alignment`, `cardinality_tally`, `scan_family_pairs`, -//! `sibling_annex_stats`) — resolves each minorant family's per-genome -//! base-presence, with the same locality discipline as -//! `build_sibling_annex` (see `cache.rs`/`build.rs`'s docs): cross-partition -//! lookups are grouped by destination partition and resolved in one -//! contiguous sweep per partition, instead of one lookup jumping between -//! partitions in family order. -//! -//! Two concerns, kept on two different mechanisms because they need -//! opposite things: -//! -//! - **Generating** each batch's cross-partition queries is pure CPU (bit -//! tests, one hash per variant) — cheap, and safe to run for several -//! batches at once. It runs on an `obipipeline::throttle` + `make_pipe!` -//! pipeline, the same mechanism `build_layer_sibling_annex` uses, for the -//! same reason: it overlaps this CPU work with the *previous* batch's -//! resolution instead of leaving cores idle between batches. -//! - **Resolving** those queries against `PartitionCache` is I/O (mmap page -//! faults on a real index far larger than RAM) — one batch at a time, -//! `rayon`-parallel *across partitions* (like `build_sibling_annex`'s own -//! `outgoing.par_iter()`), never several batches concurrently. An earlier -//! version resolved each batch fully inside its own pipeline worker, so -//! `n_workers` batches were resolved concurrently — each one spreading its -//! queries thin across every partition of the layer at once. Sampling a -//! real run showed the fix hadn't helped at all: 16 threads "busy" in -//! `find_presence`, but mostly blocked on page faults (~1.7 GB/s pagein), -//! because 16 concurrent sweeps across the same partition space is exactly -//! the scattering the grouping was meant to prevent — just spread over -//! threads instead of over layers this time. Resolving one batch's worth -//! at a time, with each of `rayon`'s threads owning one partition -//! contiguously until that batch is done, keeps only one partition set -//! "hot" at once, restoring the locality the batching is for. -//! -//! `obipipeline` does not guarantee output order, so generated batches carry -//! their own starting family index and are replayed through a small reorder -//! buffer before resolution — bounded by the throttle's own concurrency -//! (`n_workers` batches), not by the layer's size. -//! -//! [`FAMILY_BATCH`] bounds a single batch's memory to `FAMILY_BATCH * -//! genome_count` bytes, and is chosen large enough that a batch still gives -//! each partition a decent number of queries to resolve in one go — too -//! small a batch starves that density regardless of how the resolution step -//! is parallelised. An earlier version materialised the whole layer's -//! families up front before resolving anything: O(layer's family count × -//! genome count) memory, measured pushing a real run into heavy VM -//! compression/swap. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU8, Ordering}; - -use rayon::prelude::*; - -use obikindex::layer::meta::PartitionMeta; -use obikseq::CanonicalKmer; -use obipipeline::{ThrottleGuard, throttle}; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use obikindex::layer::KmerLayer; - -use super::cache::PartitionCache; -use super::helpers::central_base; -use super::iter::{SiblingEntry, SiblingLayerExt}; -use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex}; - -/// Families per batch — see the module docs for the memory-vs-per-partition- -/// density trade-off this picks a point on. At ~90 genomes and a few -/// hundred partitions, this keeps a batch's resolution memory in the low -/// tens of MB while still giving each partition on the order of a thousand -/// queries per batch to amortise against. -const FAMILY_BATCH: usize = 65536; - -/// Restricts [`scan_layer_families`] to a subset of a layer's families, -/// identified by their iteration-order index — see -/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`", for why: -/// bounding which families actually pay the expensive cross-partition -/// resolution cost is what makes a sampled run cheap on an index far larger -/// than the sample itself. `All` (the default for every pre-existing -/// caller) keeps today's behaviour exactly. -pub(super) enum Selection<'a> { - All, - Some(&'a std::collections::HashSet), -} - -/// Shared by the pipeline worker (via the `Arc`-cloned `Option`, -/// see below — a `Selection` itself doesn't cross the worker thread -/// boundary) and the final replay loop, so both sides agree on membership -/// without duplicating the match. -fn is_selected(selected: &Option>, family_idx: usize) -> bool { - match selected { - None => true, - Some(set) => set.contains(&family_idx), - } -} - -/// Every (partition, layer) directory carrying a sibling annex, checked -/// up front so a missing one is reported before any real work starts. -/// Shared by every sibling-annex consumer's extension-trait impl (`alignment`, -/// `build`, `cardinality`, `distance`, `stats`) — a free function, not a -/// `KmerIndex` method, since it is crate-internal only and `KmerIndex` lives -/// in `obikindex`, a foreign crate from here (orphan rule). -pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult> { - let n_parts = index.n_partitions(); - let mut layer_dirs = Vec::new(); - for part in 0..n_parts { - let index_dir = index.index_dir(part); - if !index_dir.exists() { - continue; - } - let n_layers = index.n_layers(part)?; - for l in 0..n_layers { - let this_layer_dir = index.layer_dir(part, l); - let annex_path = this_layer_dir.join(ANNEX_FILE_NAME); - if !annex_path.exists() { - return Err(OKIError::InvalidInput(format!( - "no sibling annex at {} — run build_sibling_annex first", - annex_path.display() - ))); - } - layer_dirs.push(this_layer_dir); - } - } - Ok(layer_dirs) -} - -/// Read-only state shared (via `Arc`) across every pipeline worker -/// generating this layer's batches — opened once, not per batch. No `cache` -/// here: generation never touches the cross-partition cache, only this -/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either — -/// `mat` (a `TypedLayer`) already bundles the MPHF, and each `SiblingEntry` -/// arrives with its kmer and mask already in hand from `iter_minorants_batch`. -struct LayerCtx { - mat: KmerLayer, - n_parts: usize, - n_genomes: usize, - n_cols: usize, - k: usize, -} - -struct SourceBatch { - start_family_idx: usize, - /// One entry per minorant family in this batch, straight from - /// `iter_minorants_batch` — `order` (iteration-order index, not an MPHF - /// slot; see `DevDocMD/architecture/siblings.md`), `kmer`, and `mask` - /// already carried through, no second annex read. - entries: Vec, - _permit: ThrottleGuard, -} - -/// One batch's generated work: local presence already resolved (straight -/// from this layer's own matrix, no lookup needed), cross-partition queries -/// collected but not yet resolved against the cache. -struct GeneratedBatch { - start_family_idx: usize, - masks: Vec, - /// Flat `slots.len() * n_genomes` — `genome_mask[i * n_genomes + g]`. - genome_mask: Vec, - /// `outgoing[dest_partition]` = `(variant, family_idx_in_batch, base, - /// layer)` — `layer` is the annex-recorded destination layer - /// (`FamilyMask::family_members`'s `layer_value`, `0` if absent), - /// trustworthy only when `PartitionCache::fast_mode()` is true. - outgoing: Vec>, - _permit: ThrottleGuard, -} - -enum FamData { - Batch(SourceBatch), - Generated(GeneratedBatch), -} - -/// Visits every minorant family of one layer, in iteration order, batched -/// [`FAMILY_BATCH`] at a time — see the module docs for why generation and -/// resolution use different concurrency. `on_family` is called once per -/// family, in iteration order, with that family's iteration-order index -/// (the same numbering [`Selection`] indices are drawn from), its own annex -/// mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set -/// iff genome `g` carries the member whose own canonical central base is -/// `b`), backed by a scratch buffer reused across every call — callers that -/// need to keep data past the call must copy it themselves. -pub(super) fn scan_layer_families( - layer_dir: &Path, - n_parts: usize, - n_genomes: usize, - with_counts: bool, - k: usize, - cache: &Arc, - selection: &Selection, - mut on_family: impl FnMut(usize, FamilyMask, &[u8]), -) -> OKIResult<()> { - let index_dir = layer_dir - .parent() - .expect("layer_dir has a parent index dir"); - let meta = PartitionMeta::load(index_dir)?; - let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?); - - let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts)?; - let n_cols = mat.n_cols().min(n_genomes); - - let ctx = Arc::new(LayerCtx { - mat, - n_parts, - n_genomes, - n_cols, - k, - }); - - // Streamed straight from `iter_minorants_batch` (zips this layer's own - // `iter_kmers()` with the annex, both in iteration order — never an - // MPHF slot; see `DevDocMD/architecture/siblings.md`) — never collected - // into a `Vec` first: a layer can hold billions of k-mers, so - // materialising every minorant family up front is exactly the memory - // blowup an earlier version of this traversal was rewritten to avoid - // (see the module docs). `.scan()` computes each batch's starting - // family index lazily, mirroring what the eager `chunks()`+running - // `offset` used to do. - let batches = - ctx.mat - .iter_minorants_batch(annex, FAMILY_BATCH) - .scan(0usize, |offset, batch| { - let start = *offset; - *offset += batch.len(); - Some((start, batch)) - }); - - let n_workers = obisys::effective_parallelism(); - let capacity = 4; - let throttled = throttle(batches, n_workers).map(|t| SourceBatch { - start_family_idx: t.item.0, - entries: t.item.1, - _permit: t.guard, - }); - - // `Selection::Some` borrows a `HashSet` whose lifetime doesn't span the - // pipeline's worker threads — cloned once into an `Arc` so every worker - // can share it cheaply instead of requiring `selection` itself to be - // `'static`. `None` means [`Selection::All`], checked with a plain - // `map_or` at each use instead of allocating an always-true set. - let selected: Arc>> = Arc::new(match selection { - Selection::All => None, - Selection::Some(set) => Some((*set).clone()), - }); - - let worker_ctx = Arc::clone(&ctx); - let worker_selected = Arc::clone(&selected); - let pipe = obipipeline::make_pipe! { - FamData : SourceBatch => GeneratedBatch, - | { - move |batch: SourceBatch| -> GeneratedBatch { - let ctx = &worker_ctx; - let n = batch.entries.len(); - let mut masks = Vec::with_capacity(n); - let mut bases = Vec::with_capacity(n); - let mut genome_mask = vec![0u8; n * ctx.n_genomes]; - let mut outgoing: Vec> = (0..ctx.n_parts).map(|_| Vec::new()).collect(); - - // Pass 1: cheap, no matrix access — own base and this - // batch's cross-partition queries. Kmer and mask already in - // hand from `iter_minorants_batch`, no second annex read, - // no slot lookup needed for this pass. `family_members` - // (not a hand-rolled `central_canonical_neighbors` + - // `mask.has` loop) already filters to present members and - // hands back each one's annex-recorded layer alongside. - // Families outside `selected` (sampling only) still get a - // mask/base entry — pass 2 indexes uniformly by `i` — but - // never an outgoing query: that's the expensive part a - // sampled run exists to avoid paying for every family. - for (i, entry) in batch.entries.iter().enumerate() { - let (kmer, mask) = (entry.kmer, entry.mask); - masks.push(mask); - let base = central_base(kmer, ctx.k); - bases.push(base); - let family_idx = batch.start_family_idx + i; - if !is_selected(&worker_selected, family_idx) { - continue; - } - for (member, layer) in mask.family_members(kmer, ctx.k) { - if member == kmer { - continue; // local — resolved below straight from `mat`, no lookup - } - let b = central_base(member, ctx.k); - let dest = member.partition(ctx.n_parts); - outgoing[dest].push((member, i, b, layer.unwrap_or(0))); - } - } - - // Pass 2: genome-major, not family-major — `mat` is stored - // one contiguous block per genome (column), slot as the - // offset within it (see `PartitionCache::find_presence_batch`'s - // docs for the full rationale). `fill_sub_matrix_carries` - // sorts the slots internally for a sequential mmap sweep per - // column, then restores this batch's order — no hand-rolled - // sort/genome-major loop needed here. The presence/count - // matrix is still MPHF-slot-indexed (unlike the annex), so - // each entry's kmer is mapped to its slot via `index_batch` - // — a pure MPHF lookup, no evidence check, since these are - // this layer's own kmers, known members by construction. - let kmers: Vec = batch.entries.iter().map(|e| e.kmer).collect(); - let slots = ctx.mat.hash_batch(&kmers); - let mut carries: Vec> = (0..ctx.n_cols).map(|_| Vec::new()).collect(); - ctx.mat.fill_sub_matrix_carries(&slots, &mut carries); - for (g, col) in carries.iter().enumerate() { - for (i, &carries_it) in col.iter().enumerate() { - if carries_it { - genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i]; - } - } - } - - GeneratedBatch { start_family_idx: batch.start_family_idx, masks, genome_mask, outgoing, _permit: batch._permit } - } - } : Batch => Generated, - }; - - // Batches finish generation in whatever order their worker completes - // them, not submission order — replay in order through a buffer bounded - // by the throttle's own concurrency (`n_workers` batches can be in - // flight at once, so at most that many can be waiting here). - let mut pending: HashMap = HashMap::new(); - let mut next_expected = 0usize; - let mut scratch = vec![0u8; n_genomes]; - for generated in pipe.apply(throttled, n_workers, capacity) { - pending.insert(generated.start_family_idx, generated); - while let Some(batch) = pending.remove(&next_expected) { - let n = batch.masks.len(); - - // ── Resolve this one batch's cross-partition queries — the - // only place that touches `cache`. Parallel across partitions - // (one batch at a time, never several concurrently — see the - // module docs), each thread owning one partition's queries - // contiguously until this batch is done. - let genome_mask: Vec = - batch.genome_mask.into_iter().map(AtomicU8::new).collect(); - let fast_mode = cache.fast_mode(); - batch - .outgoing - .par_iter() - .enumerate() - .filter(|(_, q)| !q.is_empty()) - .for_each(|(dest, queries)| { - let on_hit = |i: usize, base: u8, g: usize| { - genome_mask[i * n_genomes + g].fetch_or(1 << base, Ordering::Relaxed); - }; - if fast_mode { - cache.find_presence_batch_fast(dest, queries, n_genomes, on_hit); - } else { - cache.find_presence_batch(dest, queries, n_genomes, on_hit); - } - }); - - for i in 0..n { - let family_idx = batch.start_family_idx + i; - if !is_selected(&selected, family_idx) { - continue; // not in the sample — never resolved above, nothing to report - } - for (g, dst) in scratch.iter_mut().enumerate() { - *dst = genome_mask[i * n_genomes + g].load(Ordering::Relaxed); - } - on_family(family_idx, batch.masks[i], &scratch); - } - next_expected += n; - } - } - debug_assert!( - pending.is_empty(), - "every generated batch must have been replayed" - ); - - Ok(()) -} diff --git a/src/obikphylo/src/siblings_old/helpers.rs b/src/obikphylo/src/siblings_old/helpers.rs deleted file mode 100644 index b11c7c52..00000000 --- a/src/obikphylo/src/siblings_old/helpers.rs +++ /dev/null @@ -1,23 +0,0 @@ -use obikseq::CanonicalKmer; - -use super::FamilyMask; - -/// Central-position base of a canonical k-mer, in the fixed 0=A/1=C/2=G/3=T -/// encoding — the mask's bit index. `k` must be odd (project invariant). -#[inline] -pub(super) fn central_base(kmer: CanonicalKmer, k: usize) -> u8 { - kmer.nucleotide((k - 1) / 2) -} - -/// Is `kmer` the minorant of its family, given the family's presence mask? -/// Regenerates the family's 4 canonical forms from `kmer` itself (cheap, no -/// lookup — see the design doc's "Definitions" section for why this is -/// always safe: the set of 4 forms is invariant regardless of which member -/// you start from), and compares the raw encodings of whichever are marked -/// present in `mask`. -pub(super) fn is_minorant(kmer: CanonicalKmer, mask: FamilyMask, k: usize) -> bool { - kmer.central_canonical_neighbors().into_iter().all(|other| { - other == kmer || !mask.has(central_base(other, k)) || kmer.raw() <= other.raw() - }) -} - diff --git a/src/obikphylo/src/siblings_old/iter.rs b/src/obikphylo/src/siblings_old/iter.rs deleted file mode 100644 index 8d134639..00000000 --- a/src/obikphylo/src/siblings_old/iter.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! Phylo/sibling-domain iteration over a layer — an extension trait, not a -//! new field on `MphfLayer`/`TypedLayer`: "family"/"minorant" are phylo -//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see -//! `DevDocMD/architecture/siblings.md`). -//! -//! The sibling annex is persisted in the same order as `iter_kmers()` -//! (`build_sibling_annex`, see `build.rs`), so pairing them is a plain zip — -//! no MPHF, no slot, no `kmer_at`. Both sides are already `Send + 'static` -//! (`KmerIter` owns an `Arc` clone; `SiblingAnnex` is -//! mmap-backed and handed in as an `Arc` by the caller), so `SiblingIter` -//! streams straight from disk and can be fed to `obipipeline` batch by -//! batch — never collected whole into memory (see the project's "no full -//! collect" rule). -//! -//! Four iterator types, deliberately mirroring `obikindex::layer`'s own -//! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the -//! minorant-filtered variant of each, since "all siblings" and "one row per -//! family" are both common cases: -//! -//! | | all entries | minorants only | -//! |------------|-------------------|---------------------| -//! | single | [`SiblingIter`] | [`MinorantIter`] | -//! | batch | [`SiblingBatchIter`] | [`MinorantBatchIter`] | -//! -//! No separate "enumerate" variant (unlike `KmerIter`/`enumerate_kmers`): -//! [`SiblingEntry`] already carries `order` for free, since pairing with -//! the annex requires it anyway. - -use std::sync::Arc; - -use obikindex::layer::{KmerIter, LayerData, TypedLayer}; -use obikseq::CanonicalKmer; - -use super::{FamilyMask, SiblingAnnex}; - -/// One layer entry: a k-mer's position in the layer's iteration order (the -/// same index the sibling annex is keyed on — not an MPHF slot), the k-mer -/// itself, and its family mask. -#[derive(Debug, Clone, Copy)] -pub struct SiblingEntry { - pub order: usize, - pub kmer: CanonicalKmer, - pub mask: FamilyMask, -} - -/// Streams `(order, kmer, mask)` triples for one layer, in iteration order. -/// Produced by [`SiblingLayerExt::iter_siblings`]. -pub struct SiblingIter { - kmers: KmerIter, - annex: Arc, - order: usize, -} - -impl Iterator for SiblingIter { - type Item = SiblingEntry; - - fn next(&mut self) -> Option { - loop { - let kmer = self.kmers.next()?; - let order = self.order; - self.order += 1; - // `None` means "not yet computed" (see `SiblingAnnex` module - // docs) — shouldn't happen against a fully-built annex, but - // skip rather than misalign the two streams if it does. - if let Some(mask) = self.annex.get(order) { - return Some(SiblingEntry { order, kmer, mask }); - } - } - } -} - -/// Batches of [`SiblingIter`]'s entries, `batch_size` at a time — the last -/// batch may be shorter. Produced by [`SiblingLayerExt::iter_siblings_batch`]. -pub struct SiblingBatchIter { - inner: SiblingIter, - batch_size: usize, -} - -impl Iterator for SiblingBatchIter { - type Item = Vec; - - fn next(&mut self) -> Option { - collect_batch(&mut self.inner, self.batch_size) - } -} - -/// Like [`SiblingIter`], filtered to the minorant of each family — the -/// common case, since a family is tallied once, at its minorant. Produced -/// by [`SiblingLayerExt::iter_minorants`]. -pub struct MinorantIter { - inner: SiblingIter, -} - -impl Iterator for MinorantIter { - type Item = SiblingEntry; - - fn next(&mut self) -> Option { - self.inner.by_ref().find(|e| e.mask.is_minorant()) - } -} - -/// Batches of [`MinorantIter`]'s entries, `batch_size` at a time — the last -/// batch may be shorter. Produced by -/// [`SiblingLayerExt::iter_minorants_batch`]. -pub struct MinorantBatchIter { - inner: MinorantIter, - batch_size: usize, -} - -impl Iterator for MinorantBatchIter { - type Item = Vec; - - fn next(&mut self) -> Option { - collect_batch(&mut self.inner, self.batch_size) - } -} - -/// Shared by [`SiblingBatchIter`] and [`MinorantBatchIter`] — pull up to -/// `batch_size` items, `None` once the source is exhausted with nothing left. -fn collect_batch(inner: &mut I, batch_size: usize) -> Option> { - let mut batch = Vec::with_capacity(batch_size); - for _ in 0..batch_size { - match inner.next() { - Some(item) => batch.push(item), - None => break, - } - } - if batch.is_empty() { None } else { Some(batch) } -} - -/// Adds phylo/sibling iteration to any `TypedLayer` — the extension that -/// turns a plain layer into a "sibling layer". Generic over `D` -/// (`LayerData`) rather than implemented once per matrix kind: kmer -/// iteration doesn't depend on the data payload, and `TypedLayer` already -/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every -/// `D` — reusing that instead of going back through a separate `MphfLayer`. -pub trait SiblingLayerExt { - /// Zip this layer's k-mers with their sibling-annex entry, in iteration - /// order. `annex` must have been built from this same layer (its length - /// must match the layer's k-mer count). - fn iter_siblings(&self, annex: Arc) -> SiblingIter; - - /// Like [`iter_siblings`](Self::iter_siblings), yielding `batch_size` - /// entries at a time. - fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter; - - /// Like [`iter_siblings`](Self::iter_siblings), filtered to the - /// minorant of each family — the common case, since a family is - /// tallied once, at its minorant. - fn iter_minorants(&self, annex: Arc) -> MinorantIter; - - /// Like [`iter_minorants`](Self::iter_minorants), yielding `batch_size` - /// minorants at a time. - fn iter_minorants_batch( - &self, - annex: Arc, - batch_size: usize, - ) -> MinorantBatchIter; -} - -impl SiblingLayerExt for TypedLayer { - fn iter_siblings(&self, annex: Arc) -> SiblingIter { - SiblingIter { - kmers: self.iter_kmers(), - annex, - order: 0, - } - } - - fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter { - SiblingBatchIter { - inner: self.iter_siblings(annex), - batch_size, - } - } - - fn iter_minorants(&self, annex: Arc) -> MinorantIter { - MinorantIter { - inner: self.iter_siblings(annex), - } - } - - fn iter_minorants_batch( - &self, - annex: Arc, - batch_size: usize, - ) -> MinorantBatchIter { - MinorantBatchIter { - inner: self.iter_minorants(annex), - batch_size, - } - } -} - -/// Same extension, over `obikindex::layer::Layer` (the format-erased -/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch -/// only, both arms return the same concrete iterator types (they don't -/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer` -/// itself can't implement this: "family"/"minorant" are phylo concepts, it -/// stays kmer/slot-mapping only (see the module docs above). -impl SiblingLayerExt for obikindex::layer::KmerLayer { - fn iter_siblings(&self, annex: Arc) -> SiblingIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_siblings(annex), - obikindex::layer::KmerLayer::Presence(l) => l.iter_siblings(annex), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_siblings() called on an Empty layer") - } - } - } - - fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_siblings_batch(annex, batch_size), - obikindex::layer::KmerLayer::Presence(l) => l.iter_siblings_batch(annex, batch_size), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_siblings_batch() called on an Empty layer") - } - } - } - - fn iter_minorants(&self, annex: Arc) -> MinorantIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_minorants(annex), - obikindex::layer::KmerLayer::Presence(l) => l.iter_minorants(annex), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_minorants() called on an Empty layer") - } - } - } - - fn iter_minorants_batch( - &self, - annex: Arc, - batch_size: usize, - ) -> MinorantBatchIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_minorants_batch(annex, batch_size), - obikindex::layer::KmerLayer::Presence(l) => l.iter_minorants_batch(annex, batch_size), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_minorants_batch() called on an Empty layer") - } - } - } -} diff --git a/src/obikphylo/src/siblings_old/mod.rs b/src/obikphylo/src/siblings_old/mod.rs deleted file mode 100644 index 2a59b795..00000000 --- a/src/obikphylo/src/siblings_old/mod.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Family presence-mask annex construction. -//! -//! See `DevDocMD/theory/evolutionary_distances.md`, "Definitions: family, and -//! the canonical form of a family" and "Step 2b", for the full design -//! discussion this implements. -//! -//! For each distinct k-mer of each layer of the (already built/merged) -//! index, computes a 4-bit presence mask for its "family" (the up to 4 -//! k-mers sharing its flanks, differing only at the central base — -//! well-defined for odd k): bit `b` set iff the family member whose own -//! canonical central base is `b` (0=A, 1=C, 2=G, 3=T) is observed anywhere -//! in the current multi-genome index — a property of the whole index, not -//! of any one genome. Sibling count and minorant are *derived* from the -//! mask by callers, not stored (see `FamilyMask` and -//! [`sibling_annex_stats`](stats::SiblingStatsExt::sibling_annex_stats) -//! below). -//! -//! Per layer, an `obipipeline` batch transform (throttled — see -//! `obipipeline::throttle`) generates a whole batch's central variants at -//! once (`BATCH_SIZE` source k-mers in, that batch's variants out as one -//! pipeline message), interleaved across many in-flight batches by the -//! scheduler's shared worker pool rather than processed on a single -//! thread. The actual cross-partition lookup reuses a `PartitionCache` of -//! every partition's already-open MPHF layers, built once for the whole -//! `build_sibling_annex` run, rather than reopening files per lookup or -//! per source layer. Two earlier, coarser-grained designs were tried and -//! measured (not guessed) to be worse, in order: (1) reopening/re-mmap'ing -//! every target partition's files on every single lookup — fine at toy -//! scale, ~90% system time against a real index; (2) a `Flat` pipeline -//! stage pushing one message per generated *variant* (up to 3 per source -//! k-mer) — cheaper than reopening files, but sampling a real run showed -//! most wall-clock time going into per-message channel send/notify -//! syscalls rather than the lookup itself, because a single k-mer's ≤3 -//! variants is far too fine a granularity to amortise a pipeline's -//! synchronisation cost over. See `DevDocMD/theory/evolutionary_distances.md`, -//! Step 2b, "Mechanism". -//! -//! Submodules, in the order data flows through them: [`cache`] (shared -//! whole-run partition cache), [`helpers`] (small pure functions used -//! throughout), [`build`] (annex construction), [`stats`] (family-size -//! diagnostics), [`distance`] (raw SNP distance + base-pair tally), -//! [`cardinality`] (cardinality co-occurrence), [`alignment`] (SNP-only -//! pseudo-alignment). - -mod alignment; -mod build; -mod cache; -mod cardcomp; -mod cardinality; -mod distance; -mod entropy; -mod entropy_annex; -mod family_scan; -mod helpers; -mod iter; -mod sankoff_bundle; -mod siblingannex; -mod stats; -mod subsample; - -#[cfg(test)] -mod tests; - -pub use alignment::{SnpAlignment, SnpAlignmentExt}; -pub use build::SiblingAnnexBuildExt; -pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix}; -pub use cardinality::{CardinalityExt, CardinalityTally}; -pub use distance::{BasePairTally, DistanceExt, RawSnpDistanceOutput}; -pub use entropy::ShannonEntropyExt; -pub use iter::{MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt}; -pub use sankoff_bundle::{SankoffBundle, SankoffBundleExt}; -pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; -pub use stats::{SiblingAnnexStats, SiblingStatsExt}; -pub use subsample::EntropyBias; - -pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib"; diff --git a/src/obikphylo/src/siblings_old/sankoff_bundle.rs b/src/obikphylo/src/siblings_old/sankoff_bundle.rs deleted file mode 100644 index 9e969f88..00000000 --- a/src/obikphylo/src/siblings_old/sankoff_bundle.rs +++ /dev/null @@ -1,250 +0,0 @@ -//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` -//! pipeline — see `DevDocMD/architecture/siblings.md`, "`--free-loss`/`--tnt` -//! pipeline: four independent scans, three of them unsampled" and "Wired -//! into `pack` and the sibling-annex build path" for the problem this -//! replaces: `raw_snp_distance`, `base_pair_tally`, `cardinality_tally` and -//! `snp_pseudo_alignment` used to each open their own `PartitionCache` and -//! independently re-scan the whole annex, three of the four ignoring -//! `--subsample` entirely. -//! -//! `base_pair_tally`/`cardinality_tally` both need `raw_snp_distance`'s -//! *complete* aggregate SNP/shared counts before they can derive -//! `included[i,j]` (the `ratio_ceiling` filter) — a genuine sequential -//! dependency, not an accidental one, so this is two passes, not one: -//! -//! - **Pass A**: aggregate SNP/shared counts per genome pair, over the -//! shared selection (computed once, [`super::subsample::compute_selections`] -//! — the *same* selection every other pass below uses, per the user's -//! own requirement: "il faut qu'on sélectionne tout le monde avec la -//! même sélection"). Excluded genomes (`--exclude-genome`) are zeroed -//! here, before `included` is derived from it — the ratio filter must -//! see the post-exclusion counts, not the raw ones. -//! - **Pass B**: base-pair tally, cardinality tally, and the pseudo- -//! alignment, fused into *one* scan over the same selection — the three -//! are mutually independent once `included` is known, so they are -//! accumulated together family-by-family instead of three more separate -//! scans. - -use std::sync::Arc; - -use ndarray::Array2; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::alignment::{SnpAlignment, iupac_code}; -use super::cache::PartitionCache; -use super::cardinality::CardinalityTally; -use super::distance::{BasePairTally, RawSnpDistanceOutput}; -use super::family_scan::{Selection, scan_layer_families, sibling_layer_dirs}; -use super::subsample::{EntropyBias, compute_selections}; - -/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs, -/// computed together from one shared, possibly-subsampled/entropy-biased -/// selection — see the module docs. -pub struct SankoffBundle { - pub raw: RawSnpDistanceOutput, - pub base_pair_tally: BasePairTally, - pub cardinality_tally: CardinalityTally, - pub alignment: SnpAlignment, -} - -/// Adds [`sankoff_bundle`](Self::sankoff_bundle) to `KmerIndex`. -pub trait SankoffBundleExt { - /// `subsample`/`entropy_bias`: see - /// [`SnpAlignmentExt::snp_pseudo_alignment`](super::SnpAlignmentExt::snp_pseudo_alignment). - /// `ratio_ceiling`: see - /// [`DistanceExt::base_pair_tally`](super::DistanceExt::base_pair_tally). - /// `exclude_mask[g]`: zero genome `g`'s row/column out of the aggregate - /// SNP/shared counts *before* `included[i,j]` is derived from them — - /// same semantics as `--exclude-genome` had on the old, separate - /// `raw_snp_distance` call. - fn sankoff_bundle( - &self, - subsample: Option, - entropy_bias: Option, - ratio_ceiling: f64, - exclude_mask: &[bool], - ) -> OKIResult; -} - -impl SankoffBundleExt for KmerIndex { - fn sankoff_bundle( - &self, - subsample: Option, - entropy_bias: Option, - ratio_ceiling: f64, - exclude_mask: &[bool], - ) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = sibling_layer_dirs(self)?; - // Computed once — every pass below iterates the exact same - // families, in the exact same layers, per the shared-selection - // requirement this module exists to satisfy. - let selections = compute_selections(self, &layer_dirs, subsample, entropy_bias)?; - - // ── Pass A: aggregate SNP/shared counts ──────────────────────────── - let pb = progress_bar("raw_snp_distance", layer_dirs.len() as u64, "layers"); - let mut snp = Array2::::zeros((n_genomes, n_genomes)); - let mut shared = Array2::::zeros((n_genomes, n_genomes)); - for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |_family_idx, _mask, genome_mask| { - let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - for i in 0..n_genomes { - let Some(bi) = single_form(i) else { continue }; - for j in (i + 1)..n_genomes { - let Some(bj) = single_form(j) else { continue }; - if bi == bj { - shared[[i, j]] += 1; - shared[[j, i]] += 1; - } else { - snp[[i, j]] += 1; - snp[[j, i]] += 1; - } - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - // `--exclude-genome`: zero excluded rows/columns before `included` - // is derived below — same effect the old standalone - // `zero_excluded_pairs` had, applied here since this is now the - // only place `included` gets computed for this pipeline. - for (i, &excluded) in exclude_mask.iter().enumerate() { - if !excluded { - continue; - } - for j in 0..n_genomes { - snp[[i, j]] = 0; - snp[[j, i]] = 0; - shared[[i, j]] = 0; - shared[[j, i]] = 0; - } - } - let raw = RawSnpDistanceOutput { snp, shared }; - - let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| { - if i == j { - return false; - } - let s = raw.snp[[i, j]]; - let total = s + raw.shared[[i, j]]; - total > 0 && (s as f64 / total as f64) <= ratio_ceiling - }); - - // ── Pass B: base-pair tally + cardinality tally + alignment, fused ── - let pb = progress_bar("sankoff_pass_b", layer_dirs.len() as u64, "layers"); - let mut bp_counts = [[0u64; 4]; 4]; - let mut bp_same = [0u64; 4]; - let mut card_counts = [[0u64; 5]; 5]; - let mut sequences: Vec> = vec![Vec::new(); n_genomes]; - for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |_family_idx, mask, genome_mask| { - let variable = mask.family_size() >= 2; - - // base-pair tally — same pairwise single-form resolution as pass A. - let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - for i in 0..n_genomes { - let Some(bi) = single_form(i) else { continue }; - for j in (i + 1)..n_genomes { - let Some(bj) = single_form(j) else { continue }; - if !included[[i, j]] { - continue; - } - if bi != bj { - bp_counts[bi as usize][bj as usize] += 1; - bp_counts[bj as usize][bi as usize] += 1; - } else if variable { - bp_same[bi as usize] += 1; - } - } - } - - // cardinality tally — restricted to variable families, see - // `CardinalityExt::cardinality_tally`'s docs for why. - if variable { - for i in 0..n_genomes { - let card_i = genome_mask[i].count_ones() as usize; - for j in (i + 1)..n_genomes { - if !included[[i, j]] { - continue; - } - let card_j = genome_mask[j].count_ones() as usize; - card_counts[card_i][card_j] += 1; - if card_i != card_j { - card_counts[card_j][card_i] += 1; - } - } - } - } - - // pseudo-alignment — same variable-family gate as - // `snp_pseudo_alignment`. Every family the shared selection - // actually visits already satisfies this when the - // selection is `Selection::Some` (only non-monomorphic - // minorants are ever selected), but the explicit check - // still matters under `Selection::All` (no `--subsample`), - // which visits monomorphic minorants too. - if variable { - for (g, &m) in genome_mask.iter().enumerate() { - sequences[g].push(iupac_code(m)); - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(SankoffBundle { - raw, - base_pair_tally: BasePairTally { - counts: bp_counts, - same: bp_same, - }, - cardinality_tally: CardinalityTally { - counts: card_counts, - }, - alignment: SnpAlignment { sequences }, - }) - } -} diff --git a/src/obikphylo/src/siblings_old/siblingannex.rs b/src/obikphylo/src/siblings_old/siblingannex.rs deleted file mode 100644 index 0ec81d5e..00000000 --- a/src/obikphylo/src/siblings_old/siblingannex.rs +++ /dev/null @@ -1,433 +0,0 @@ -//! Family presence-mask annex: a compact, read-only-after-build, per-slot -//! derived value used by the central-position SNP distance estimator (see -//! `DevDocMD/theory/evolutionary_distances.md`, "Step 2b" and "Definitions: -//! family, and the canonical form of a family"). -//! -//! Two bytes are stored per MPHF slot of a partition/layer, packed as four -//! 3-bit fields (one per central base, in the fixed A/C/G/T = 0/1/2/3 -//! encoding) plus a minorant flag: -//! -//! - Each 3-bit field: `0` = the family member whose *own* central base — -//! in its own canonical orientation — is that base is absent from the -//! whole multi-genome index (a property of the whole index, not of any -//! one genome); a non-zero value `v` (`1..=7`) means it's present, and -//! *may* additionally encode `layer = v - 1` (the destination layer -//! within its partition) once a caller populates it via -//! [`with_layer`](FamilyMask::with_layer) — see that method's docs. -//! `FamilyMask` itself is policy-free about whether a given non-zero -//! value is a trustworthy layer index or just "present, no layer -//! recorded": that decision belongs to callers, who already have -//! `PartitionMeta::n_layers` in scope. -//! - The minorant bit: whether *this slot's own k-mer* is its family's -//! minorant (the smallest raw encoding among the family's observed -//! members) — computed once, when the whole family's mask is already -//! final (see `build::build_layer_sibling_annex`), and read back by every -//! consumer that would otherwise have to reconstruct this slot's k-mer -//! from `unitigs.bin` and hash it through the MPHF again just to ask the -//! same question. -//! -//! Both facts an even earlier design stored explicitly are still derived, -//! not stored: -//! - sibling count = `family_size() - 1`; -//! - which of the (up to 4) family members are present, and their canonical -//! form — [`FamilyMask::family_members`], regenerated from the slot's own -//! k-mer (`CanonicalKmer::central_canonical_neighbors`, cheap, no -//! lookup), not stored. -//! -//! Mask value 0 is logically unreachable as a real result (a slot's own -//! base is always present in its own family) and is reused as the "not yet -//! computed" sentinel: annex files are pre-initialised to all-zero, and a -//! real value is only ever written once, by the computation pass. -//! -//! Widened from the original 1-byte/slot design (4 presence bits + 1 -//! minorant bit) to carry a per-member layer number without an extra -//! lookup pass — see the project discussion this implements. An old -//! (1-byte/slot) `.psib` file is *not* silently misread by the new reader: -//! its length no longer matches `HEADER_SIZE + n * 2`, so `SiblingAnnex::open` -//! fails loudly ("PSIB file truncated") rather than producing garbage. -//! Annexes built before this change must be rebuilt (`--sibling-annex`). - -use std::fs::{File, OpenOptions}; -use std::io; -use std::path::{Path, PathBuf}; -use std::sync::atomic::AtomicU16; - -use memmap2::{Mmap, MmapMut}; - -use obikseq::CanonicalKmer; - -use super::helpers::central_base; - -const MAGIC: [u8; 4] = *b"PSIB"; - -// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (2 bytes/slot) follows. -const HEADER_SIZE: usize = 16; - -/// Width, in bits, of one base's field. -const FIELD_BITS: u32 = 3; -/// Mask for one base's field once shifted into position. -const FIELD_MASK: u16 = 0b111; -/// Highest raw field value a base can carry (`1..=MAX_FIELD_VALUE`, `0` = absent). -const MAX_FIELD_VALUE: u8 = 7; - -const MINORANT_BIT: u16 = 1 << 12; - -/// A family presence mask — see the module docs for the full bit layout. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct FamilyMask(u16); - -impl FamilyMask { - /// The empty mask — never a valid *computed* result (a slot's own base - /// is always present in its own family) — used only to build up a mask - /// via repeated [`with`](Self::with)/[`with_layer`](Self::with_layer) - /// calls before storing it. - pub const EMPTY: FamilyMask = FamilyMask(0); - - #[inline] - fn field_shift(base: u8) -> u32 { - debug_assert!(base < 4, "base out of range: {base}"); - base as u32 * FIELD_BITS - } - - #[inline] - fn field(self, base: u8) -> u8 { - ((self.0 >> Self::field_shift(base)) & FIELD_MASK) as u8 - } - - #[inline] - fn with_field(self, base: u8, value: u8) -> Self { - debug_assert!(value <= MAX_FIELD_VALUE, "field value out of range: {value}"); - let shift = Self::field_shift(base); - let cleared = self.0 & !(FIELD_MASK << shift); - FamilyMask(cleared | ((value as u16) << shift)) - } - - /// Raw bits to `fetch_or` into a shared `AtomicU16` accumulator to mark - /// `base` present at `layer`, without allocating a whole `FamilyMask` - /// per write — for `build::build_layer_sibling_annex`'s concurrent - /// construction, where each `(word, base)` field is written by exactly - /// one thread (see that module's docs), so a lock-free OR is safe. - #[inline] - pub(crate) fn layer_bits(base: u8, layer: usize) -> u16 { - debug_assert!(layer < MAX_FIELD_VALUE as usize, "layer out of range: {layer}"); - (layer as u16 + 1) << Self::field_shift(base) - } - - /// Same as [`layer_bits`](Self::layer_bits), presence-only (no layer - /// recorded) — the accumulator-side equivalent of [`with`](Self::with). - #[inline] - pub(crate) fn presence_bits(base: u8) -> u16 { - 1u16 << Self::field_shift(base) - } - - /// Mark the member with central base `base` (0=A, 1=C, 2=G, 3=T) as - /// present, without recording a layer (compat path: same observable - /// effect as the original 1-byte design's `with`). Use - /// [`with_layer`](Self::with_layer) instead when the destination layer - /// is already known. - #[inline] - pub fn with(self, base: u8) -> Self { - self.with_field(base, 1) - } - - /// Mark the member with central base `base` as present *and* record its - /// destination layer. `layer` must be `< 7` (`debug_assert`ed) — a - /// caller facing more layers than that has no compact field to record - /// them in and must fall back to [`with`](Self::with) instead (still - /// correct, just without the fast-path payoff); see the module docs on - /// why `FamilyMask` doesn't decide that threshold itself. - #[inline] - pub fn with_layer(self, base: u8, layer: usize) -> Self { - debug_assert!(layer < MAX_FIELD_VALUE as usize, "layer out of range: {layer}"); - self.with_field(base, layer as u8 + 1) - } - - /// Is the member with central base `base` (0..3) present? - #[inline] - pub fn has(self, base: u8) -> bool { - self.field(base) != 0 - } - - /// Raw stored value for `base`'s field, if present: `Some(v - 1)` where - /// `v` is the non-zero field value. This is the *raw* stored value, not - /// a validated layer index — a caller must cross-check it against - /// `PartitionMeta::n_layers` (`<= 7`) before trusting it as a real - /// layer, since a mask written via [`with`](Self::with) (no layer - /// known) also reads back as `Some(0)` here. - #[inline] - pub fn layer_value(self, base: u8) -> Option { - let v = self.field(base); - if v == 0 { None } else { Some(v - 1) } - } - - /// Number of family members observed anywhere in the index (1..=4). - #[inline] - pub fn family_size(self) -> u32 { - (0..4).filter(|&b| self.has(b)).count() as u32 - } - - /// Number of *other* members observed (0..=3) — `family_size() - 1`. - #[inline] - pub fn siblings(self) -> u32 { - self.family_size() - 1 - } - - /// Set or clear the minorant flag (see the module docs). - #[inline] - pub fn with_minorant(self, is_minorant: bool) -> Self { - if is_minorant { - FamilyMask(self.0 | MINORANT_BIT) - } else { - FamilyMask(self.0 & !MINORANT_BIT) - } - } - - /// Is this slot's own k-mer its family's minorant? Only meaningful once - /// `with_minorant` has been called with the family's *final* mask (i.e. - /// after construction) — see the module docs. - #[inline] - pub fn is_minorant(self) -> bool { - self.0 & MINORANT_BIT != 0 - } - - /// Raw presence bitmask (bit `b` = base `b` present, low 4 bits only — - /// never includes the minorant flag or any layer information) — for - /// callers that only need the old 1-bit-per-base view, e.g. to compare - /// against a previously-computed value built via their own bit - /// operations. - #[inline] - pub fn bits(self) -> u8 { - (0..4).fold(0u8, |acc, b| if self.has(b) { acc | (1 << b) } else { acc }) - } - - /// Construct from a raw presence bitmask (only the low 4 bits are kept - /// — the minorant flag is not part of this, use - /// [`with_minorant`](Self::with_minorant) separately). Each set bit is - /// recorded as "present, no layer known" — see [`with`](Self::with). - #[inline] - pub fn from_bits(bits: u8) -> Self { - (0..4).fold(FamilyMask::EMPTY, |mask, b| { - if bits & (1 << b) != 0 { mask.with(b) } else { mask } - }) - } - - /// This family's present members, in canonical form, paired with their - /// raw stored field value (see [`layer_value`](Self::layer_value) for - /// why it's not directly a validated layer index). `owner` is this - /// slot's own k-mer (any member works — the 4 canonical forms are - /// invariant regardless of which member you start from). - pub fn family_members(self, owner: CanonicalKmer, k: usize) -> impl Iterator)> { - owner - .central_canonical_neighbors() - .into_iter() - .filter_map(move |member| { - let base = central_base(member, k); - self.has(base).then(|| (member, self.layer_value(base))) - }) - } - - #[inline] - fn encode(self) -> u16 { - self.0 - } - - #[inline] - fn decode(word: u16) -> Option { - if word == 0 { - // Unreachable for a real result — reserved as the "not yet - // computed" sentinel. A real entry always has at least one - // non-zero base field, so word == 0 always means "nothing - // written yet", never a genuine minorant-only value. - return None; - } - Some(FamilyMask(word)) - } -} - -// ── SiblingAnnex (reader) ─────────────────────────────────────────────────── - -pub struct SiblingAnnex { - mmap: Mmap, - n: usize, - path: PathBuf, -} - -impl SiblingAnnex { - pub fn open(path: &Path) -> io::Result { - let mmap = unsafe { Mmap::map(&File::open(path)?)? }; - if mmap.len() < HEADER_SIZE { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short")); - } - if mmap[0..4] != MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic")); - } - let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; - if mmap.len() < HEADER_SIZE + n * 2 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated")); - } - Ok(Self { mmap, n, path: path.to_path_buf() }) - } - - pub fn path(&self) -> &Path { &self.path } - pub fn len(&self) -> usize { self.n } - pub fn is_empty(&self) -> bool { self.n == 0 } - - /// `None` means the slot has not (yet) been computed — see module docs. - pub fn get(&self, slot: usize) -> Option { - let off = HEADER_SIZE + slot * 2; - FamilyMask::decode(u16::from_le_bytes(self.mmap[off..off + 2].try_into().unwrap())) - } -} - -// ── SiblingAnnexBuilder (writer) ──────────────────────────────────────────── - -pub struct SiblingAnnexBuilder { - mmap: MmapMut, -} - -impl SiblingAnnexBuilder { - /// Create a new annex of `n` slots at `path`, pre-initialised to the - /// "not yet computed" sentinel (all-zero). - pub fn new(n: usize, path: &Path) -> io::Result { - let file_size = HEADER_SIZE + n * 2; - let file = OpenOptions::new() - .read(true).write(true).create(true).truncate(true) - .open(path)?; - file.set_len(file_size as u64)?; - let mut mmap = unsafe { MmapMut::map_mut(&file)? }; - mmap[0..4].copy_from_slice(&MAGIC); - mmap[4..8].copy_from_slice(&[0u8; 4]); - mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); - // Data region left at 0 by `set_len`/mmap — the sentinel value. - Ok(Self { mmap }) - } - - /// `None` means the slot has not (yet) been computed — see module - /// docs. Used to read back a slot's concurrently-accumulated value - /// (via [`atomic_slot`](Self::atomic_slot)) before finalising it with - /// [`set`](Self::set). - pub fn get(&self, slot: usize) -> Option { - let off = HEADER_SIZE + slot * 2; - FamilyMask::decode(u16::from_le_bytes(self.mmap[off..off + 2].try_into().unwrap())) - } - - /// Atomic handle to `slot`'s raw storage, for lock-free concurrent - /// construction straight into the mmap — no separate in-memory - /// accumulator needed (see `build::build_layer_sibling_annex`, the - /// only caller). Safe: every concurrent write goes through - /// `fetch_or`/atomic RMW, and by that caller's own invariant, a given - /// 3-bit field within a slot's word is written by exactly one thread — - /// only *different* fields of the same word can race, which `fetch_or` - /// already serialises correctly. Alignment holds: `HEADER_SIZE` is - /// even and every slot is 2 bytes, so `HEADER_SIZE + slot * 2` is - /// always even, and the mmap's own base address is page-aligned. - pub(crate) fn atomic_slot(&self, slot: usize) -> &AtomicU16 { - let off = HEADER_SIZE + slot * 2; - debug_assert!(off + 2 <= self.mmap.len()); - let ptr = unsafe { self.mmap.as_ptr().add(off) as *mut u16 }; - unsafe { AtomicU16::from_ptr(ptr) } - } - - /// Plain (non-atomic) store — for the sequential finalisation pass - /// only, once every concurrent write (via - /// [`atomic_slot`](Self::atomic_slot)) is done. - pub fn set(&mut self, slot: usize, mask: FamilyMask) { - let off = HEADER_SIZE + slot * 2; - self.mmap[off..off + 2].copy_from_slice(&mask.encode().to_le_bytes()); - } - - pub fn close(self) -> io::Result<()> { self.mmap.flush() } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.psib"); - let builder = SiblingAnnexBuilder::new(4, &path).unwrap(); - builder.close().unwrap(); - let annex = SiblingAnnex::open(&path).unwrap(); - for slot in 0..4 { - assert_eq!(annex.get(slot), None); - } - } - - #[test] - fn roundtrip_all_valid_masks() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.psib"); - let mut builder = SiblingAnnexBuilder::new(4, &path).unwrap(); - - let masks = [ - FamilyMask::EMPTY.with(0), // just A: family size 1 - FamilyMask::EMPTY.with(0).with(3), // A + T: size 2 - FamilyMask::EMPTY.with(1).with(2).with(3), // C+G+T: size 3 - FamilyMask::EMPTY.with(0).with(1).with(2).with(3), // all 4 - ]; - for (slot, mask) in masks.iter().enumerate() { - builder.set(slot, *mask); - } - builder.close().unwrap(); - let annex = SiblingAnnex::open(&path).unwrap(); - for (slot, mask) in masks.iter().enumerate() { - assert_eq!(annex.get(slot), Some(*mask)); - } - assert_eq!(annex.get(0).unwrap().siblings(), 0); - assert_eq!(annex.get(1).unwrap().siblings(), 1); - assert_eq!(annex.get(2).unwrap().siblings(), 2); - assert_eq!(annex.get(3).unwrap().siblings(), 3); - assert_eq!(annex.get(3).unwrap().family_size(), 4); - } - - #[test] - fn has_reflects_individual_bits() { - let mask = FamilyMask::EMPTY.with(0).with(2); - assert!(mask.has(0)); - assert!(!mask.has(1)); - assert!(mask.has(2)); - assert!(!mask.has(3)); - } - - #[test] - fn with_layer_roundtrips_and_leaves_absent_bases_none() { - let mask = FamilyMask::EMPTY.with_layer(0, 0).with_layer(2, 5); - assert_eq!(mask.layer_value(0), Some(0)); - assert_eq!(mask.layer_value(2), Some(5)); - assert_eq!(mask.layer_value(1), None); - assert_eq!(mask.layer_value(3), None); - assert!(mask.has(0) && mask.has(2)); - assert!(!mask.has(1) && !mask.has(3)); - assert_eq!(mask.family_size(), 2); - } - - #[test] - fn with_marks_present_without_a_layer() { - // Compat path: `with` still means "present", now reading back as - // `layer_value == Some(0)` (raw field value 1, i.e. "no layer - // recorded") — distinguishable from a real layer 0 only by a - // caller that already knows whether layers were ever wired in. - let mask = FamilyMask::EMPTY.with(1); - assert!(mask.has(1)); - assert_eq!(mask.layer_value(1), Some(0)); - } - - #[test] - fn family_members_reconstructs_present_canonical_forms() { - use obikseq::{Kmer, Sequence}; - - const K: usize = 11; - obikseq::params::set_k(K); - // Centre (index 5) is 'C' (base 1) in this k-mer's own orientation. - let owner = Kmer::from_ascii(b"AACCGCTTAAG").unwrap().canonical(); - let mask = FamilyMask::EMPTY.with_layer(1, 0).with_layer(2, 3); // C (own) + G present - - let members: Vec<_> = mask.family_members(owner, K).collect(); - assert_eq!(members.len(), 2, "only the 2 present members should be yielded"); - assert!(members.iter().any(|(k, layer)| *k == owner && *layer == Some(0))); - assert!(members.iter().any(|(k, layer)| *k != owner && *layer == Some(3))); - } -} diff --git a/src/obikphylo/src/siblings_old/stats.rs b/src/obikphylo/src/siblings_old/stats.rs deleted file mode 100644 index dbf24379..00000000 --- a/src/obikphylo/src/siblings_old/stats.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::sync::Arc; - -use rayon::prelude::*; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::ANNEX_FILE_NAME; -use super::SiblingAnnex; -use super::cache::PartitionCache; -use super::family_scan::{Selection, scan_layer_families}; - -/// Distribution of family sizes (1-4), read back from an already-built -/// annex (see [`super::build::SiblingAnnexBuildExt::build_sibling_annex`]) -/// plus the index's presence/count data — a separate, occasional diagnostic -/// pass, not fused into construction. -/// -/// Every count here is **per family, not per slot**: a family with `F` -/// members occupies `F` annex slots (one per observed member), all sharing -/// the same mask. Counting every slot would count each family up to 4 -/// times over; only the minorant's slot is tallied (minorant is derived on -/// the fly — see `is_minorant` — not stored, but cheap: no lookup, pure -/// bit arithmetic on already-in-hand data). -#[derive(Debug, Clone, Default)] -pub struct SiblingAnnexStats { - /// `counts[s]` = number of *families* of size `s + 1` (index 0 = size 1, - /// i.e. 0 siblings, ... index 3 = size 4, i.e. 3 siblings). - pub counts: [u64; 4], - /// `per_genome[g][s]` = number of families of size `s + 1` for which - /// genome `g` (index into `KmerIndex::meta().genomes`) carries at least - /// one member. - pub per_genome: Vec<[u64; 4]>, -} - -/// Adds [`sibling_family_size_histogram`](Self::sibling_family_size_histogram) -/// and [`sibling_annex_stats`](Self::sibling_annex_stats) to `KmerIndex`. -pub trait SiblingStatsExt { - /// The global family-size histogram alone (`SiblingAnnexStats::counts`, - /// no `per_genome`) — reads only the already-built annex (`mask.siblings()` - /// + `mask.is_minorant()`, 1 byte/slot, mmap'd), nothing else: no - /// `unitigs.bin` scan, no MPHF lookup, no `PartitionCache`. Cost is a - /// linear scan of one small file per layer, independent of the rest of - /// the index's size or how much of it is paged in. An earlier version - /// re-derived minorant-ness per slot (re-reading `unitigs.bin`, hashing - /// every k-mer through the MPHF again to reconstruct what - /// `build_sibling_annex` already knew) — sampling a real run showed - /// `MphfLayer::find` alone at 71% of wall-clock time for what should be - /// a near-instant four-bucket count. `mask.is_minorant()` is that same - /// fact, computed once at construction (see `build_layer_sibling_annex`) - /// and stored in the annex's own spare bits — free to read back. - /// [`sibling_annex_stats`](Self::sibling_annex_stats) computes the same - /// `counts` but pays the full per-genome cross-partition resolution - /// cost to get there too; use this when only the global histogram is - /// needed. Requires an annex built after the minorant flag was added — - /// re-run `build_sibling_annex` if this reads as all-zero on an older one. - fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]>; - - /// Tally the family-size distribution of an already-built annex - /// (globally, and per genome), counting each family once (at its - /// minorant slot). Errors if - /// [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) - /// has not been run on this index first. - fn sibling_annex_stats(&self) -> OKIResult; -} - -impl SiblingStatsExt for KmerIndex { - fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> { - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - - // Each layer's annex file is independent — no shared `PartitionCache` - // and no `scan_layer_families` partition-grouped locality to protect - // (unlike `sibling_annex_stats`/`distance`/`cardinality_tally`), so - // layers can be scanned concurrently. - layer_dirs - .par_iter() - .try_fold( - || [0u64; 4], - |mut counts, layer_dir| -> OKIResult<[u64; 4]> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { - continue; - }; - if !mask.is_minorant() { - continue; // family tallied once, at its minorant - } - counts[mask.siblings() as usize] += 1; - } - Ok(counts) - }, - ) - .try_reduce( - || [0u64; 4], - |a, b| { - let mut sum = a; - for i in 0..4 { - sum[i] += b[i]; - } - Ok(sum) - }, - ) - } - - fn sibling_annex_stats(&self) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - // Same whole-run cache as `build_sibling_annex` — see its docs for - // why re-opening per lookup (or per call to a batching helper) is - // not good enough on a real index. - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - - // One layer at a time, not parallelised across layers — see - // `snp_pseudo_alignment`'s comment for why `par_iter()` over layers - // would defeat `scan_layer_families`'s partition-grouped locality. - let pb = progress_bar("sibling_annex_stats", layer_dirs.len() as u64, "layers"); - let mut stats = SiblingAnnexStats { - per_genome: vec![[0u64; 4]; n_genomes], - ..Default::default() - }; - for layer_dir in &layer_dirs { - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::All, - |_family_idx, mask, genome_mask| { - // "Genome g represents this family" means g carries - // *any* of its members, not just the minorant's own — - // `genome_mask[g] != 0` is exactly that. - let s = mask.siblings() as usize; - stats.counts[s] += 1; - for (g, &m) in genome_mask.iter().enumerate() { - if m != 0 { - stats.per_genome[g][s] += 1; - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(stats) - } -} diff --git a/src/obikphylo/src/siblings_old/subsample.rs b/src/obikphylo/src/siblings_old/subsample.rs deleted file mode 100644 index fc6b0a32..00000000 --- a/src/obikphylo/src/siblings_old/subsample.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! Proportional per-layer sampling of non-monomorphic minorants -//! (`family_size() >= 2`) — see `DevDocMD/architecture/siblings.md`, -//! "`--subsample`/`--shannon`", for the full design discussion. -//! -//! Three steps, in order, each cheaper than the one before because it -//! narrows what the next one has to touch: -//! -//! 1. [`non_monomorphic_counts`] — one cheap structural pass per layer -//! (annex bits only, no cross-partition resolution), giving `count_layer` -//! for every layer. -//! 2. [`sample_layers`] — given a target `n` and those counts, computes -//! `n_layer = round(n * count_layer / total_count)` per layer (never -//! exceeds `count_layer` as long as `n <= total_count`; if -//! `total_count <= n`, sampling is skipped entirely — every -//! non-monomorphic minorant of every layer is kept) and reservoir-samples -//! (Algorithm R, one more cheap structural pass, `O(n_layer)` memory) the -//! selected iteration-order indices. -//! 3. The result feeds [`super::family_scan::scan_layer_families`]'s -//! `Selection` parameter, which bounds the expensive cross-partition -//! resolution to just the selected families. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; - -use rand::RngExt; -use rayon::prelude::*; - -use obikindex::{KmerIndex, OKIError, OKIResult}; - -use super::ANNEX_FILE_NAME; -use super::SiblingAnnex; -use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME}; - -/// Gaussian-kernel entropy bias parameters — see -/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". `mu`/ -/// `sigma` are the `--entropy`/`--entropy-sd` CLI values (defaulted by the -/// caller, not here — this type carries only the resolved numbers). `pub` -/// (not `pub(super)`): part of the public signature of -/// `SnpAlignmentExt::snp_pseudo_alignment`, -/// `ShannonEntropyExt::shannon_entropy_csv`, and -/// `SankoffBundleExt::sankoff_bundle`, all consumed from `obikmer`. -#[derive(Clone, Copy)] -pub struct EntropyBias { - pub mu: f64, - pub sigma: f64, -} - -/// A family is non-monomorphic (informative-eligible) iff at least one -/// sibling is registered alongside it — `family_size() >= 2`, i.e. -/// `siblings() >= 1`. Mirrors the filter already applied post-hoc in -/// `snp_pseudo_alignment`, but checked here *before* any resolution work. -fn is_non_monomorphic_minorant(mask: super::FamilyMask) -> bool { - mask.is_minorant() && mask.siblings() >= 1 -} - -/// Step 1: per-layer count of non-monomorphic minorants — cheap, parallel -/// across layers, annex bits only (same shape as -/// `SiblingStatsExt::sibling_family_size_histogram`, but keeping the -/// per-layer breakdown instead of collapsing it into one index-wide sum, -/// since proportional sampling needs each layer's own share). -pub(super) fn non_monomorphic_counts(layer_dirs: &[PathBuf]) -> OKIResult> { - layer_dirs - .par_iter() - .map(|layer_dir| -> OKIResult { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut count = 0u64; - for slot in 0..annex.len() { - if annex.get(slot).is_some_and(is_non_monomorphic_minorant) { - count += 1; - } - } - Ok(count) - }) - .collect() -} - -/// Step 2, per layer: Algorithm-R reservoir sampling of `n_layer` -/// non-monomorphic minorant iteration-indices — single pass, no -/// cross-partition resolution, `O(n_layer)` memory regardless of the -/// layer's true size (never materialises the full non-monomorphic index -/// list, only the reservoir itself). -/// -/// The stored values are **`family_idx`, `scan_layer_families`'s own -/// numbering** — the index among *every* minorant of the layer (monomorphic -/// ones included, since `iter_minorants_batch` only filters on -/// `is_minorant()`), not the raw annex slot (`for slot in 0..annex.len()` -/// spans every k-mer, minorant or not) and not a counter over -/// non-monomorphic minorants alone. Conflating any of these with -/// `family_idx` silently drifts apart as soon as the layer contains a -/// monomorphic minorant — i.e. almost immediately, since ~98% of minorants -/// are monomorphic — so the two counters below are deliberately kept -/// separate: `family_idx` tracks every minorant (matching -/// `scan_layer_families`), `seen` tracks only the non-monomorphic ones -/// (what Algorithm R actually samples over). -fn reservoir_sample_layer(layer_dir: &Path, n_layer: usize) -> OKIResult> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut reservoir: Vec = Vec::with_capacity(n_layer); - let mut seen: u64 = 0; - let mut family_idx: usize = 0; - let mut rng = rand::rng(); - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { continue }; - if !mask.is_minorant() { - continue; // not a minorant at all — doesn't advance `family_idx` either - } - let this_family_idx = family_idx; - family_idx += 1; - if mask.siblings() < 1 { - continue; // monomorphic minorant — consumed a family_idx above, but not sample-eligible - } - if reservoir.len() < n_layer { - reservoir.push(this_family_idx); - } else { - let j = rng.random_range(0..=seen); - if j < n_layer as u64 { - reservoir[j as usize] = this_family_idx; - } - } - seen += 1; - } - Ok(reservoir.into_iter().collect()) -} - -/// Every non-monomorphic minorant's `family_idx`, for one layer — not a -/// sample, the *complete* set. Structural, annex-only, no cross-partition -/// resolution (same discipline as [`reservoir_sample_layer`], just keeping -/// everything instead of a bounded reservoir). Used to bound -/// [`super::entropy::ensure_entropy_annexes`]'s `scan_layer_families` call -/// to a `Selection::Some` of exactly the families that need their entropy -/// computed, instead of `Selection::All` — monomorphic minorants (~98% of -/// them, measured) are known to be ineligible straight from the annex bits -/// already, with no need to pay `scan_layer_families`'s expensive -/// per-genome resolution (`fill_sub_matrix_carries`) only to discard the -/// result once `family_size() < 2` is checked *after* resolving. -pub(super) fn non_monomorphic_selection_layer(layer_dir: &Path) -> OKIResult> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut selected = HashSet::new(); - let mut family_idx: usize = 0; - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { continue }; - if !mask.is_minorant() { - continue; // doesn't advance `family_idx` either - } - let this_family_idx = family_idx; - family_idx += 1; - if is_non_monomorphic_minorant(mask) { - selected.insert(this_family_idx); - } - } - Ok(selected) -} - -/// Total minorant count per layer (`is_minorant()`, monomorphic included) -/// — sizes the entropy annex (`entropy_annex.rs`), which needs exactly one -/// entry per minorant to keep its `family_idx` numbering aligned with -/// every other consumer (`scan_layer_families`, `Selection`, -/// `reservoir_sample_layer`'s own `family_idx` counter above). -pub(super) fn minorant_counts(layer_dirs: &[PathBuf]) -> OKIResult> { - layer_dirs - .par_iter() - .map(|layer_dir| -> OKIResult { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut count = 0u64; - for slot in 0..annex.len() { - if annex.get(slot).is_some_and(|m| m.is_minorant()) { - count += 1; - } - } - Ok(count) - }) - .collect() -} - -/// Entropy-biased alternative to [`reservoir_sample_layer`]: an -/// independent accept/reject draw per non-monomorphic minorant, probability -/// `p0 * w(entropy)` where `w(entropy) = exp(-(entropy-mu)^2 / (2*sigma^2))` -/// (the unnormalised Gaussian kernel — `1` exactly at `entropy == mu`, -/// smoothly decaying away from it, no hard cutoff) and `p0` is the same -/// uniform rate the plain reservoir targets in expectation (`n / total_count` -/// index-wide, or `1.0` when there is no size target at all — `--entropy` -/// given without `--subsample`, a pure soft entropy filter over the whole -/// index). See `DevDocMD/architecture/siblings.md`, "Entropy-biased -/// selection", for why this is a single streaming pass with no -/// cross-partition resolution: entropy is read from the already-built -/// entropy annex (`ensure_entropy_annexes`), not recomputed here. -fn entropy_biased_sample_layer(layer_dir: &Path, p0: f64, bias: EntropyBias) -> OKIResult> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let entropy_annex = EntropyAnnex::open(&layer_dir.join(ENTROPY_ANNEX_FILE_NAME)).map_err(OKIError::Io)?; - let mut selected = HashSet::new(); - let mut family_idx: usize = 0; - let mut rng = rand::rng(); - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { continue }; - if !mask.is_minorant() { - continue; // not a minorant at all — doesn't advance `family_idx` either - } - let this_family_idx = family_idx; - family_idx += 1; - debug_assert!( - this_family_idx < entropy_annex.len(), - "entropy annex ({}) shorter than this layer's minorant count — built against a different annex?", - entropy_annex.len(), - ); - if mask.siblings() < 1 { - continue; // monomorphic minorant — no entropy, never eligible - } - let Some(entropy) = entropy_annex.get(this_family_idx) else { continue }; - let d = (entropy as f64 - bias.mu) / bias.sigma; - let w = (-0.5 * d * d).exp(); - if rng.random::() < p0 * w { - selected.insert(this_family_idx); - } - } - Ok(selected) -} - -/// Per-layer sampling outcome — `None` at a given layer means "keep every -/// non-monomorphic minorant of this layer" (either the `total_count <= n` -/// case, or a layer whose entire non-monomorphic set was already smaller -/// than its computed share and got fully absorbed by rounding). -pub(super) type LayerSelection = Option>; - -/// Full entry point for every caller (`snp_pseudo_alignment`, the Shannon -/// entropy report, `sankoff_bundle`): `n = None` and `entropy_bias = None` -/// means "no `--subsample`, no `--entropy`/`--entropy-sd`", i.e. keep every -/// non-monomorphic minorant of every layer — skips steps 1 and 2 entirely, -/// since there is nothing to proportion. -/// -/// `entropy_bias = Some(_)` switches selection from the plain uniform -/// reservoir to [`entropy_biased_sample_layer`] — see -/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". `index` is -/// only used on that path, to build any missing per-layer entropy annex -/// first ([`super::entropy::ensure_entropy_annexes`]) — a one-time cost, -/// not paid again once every layer's annex file exists. The target size -/// `n` still proportions the *expected* rate across the whole index (a -/// single `p0 = n / total_count` applied uniformly at every layer already -/// yields each layer its proportional share, with no need to compute a -/// per-layer target the way [`sample_layers`] does for the uniform -/// reservoir) — but unlike the uniform reservoir, the result is -/// approximate, not exact (see [`entropy_biased_sample_layer`]'s docs). -pub(super) fn compute_selections( - index: &KmerIndex, - layer_dirs: &[PathBuf], - n: Option, - entropy_bias: Option, -) -> OKIResult> { - let Some(bias) = entropy_bias else { - return match n { - None => Ok(vec![None; layer_dirs.len()]), - Some(n) => { - let counts = non_monomorphic_counts(layer_dirs)?; - sample_layers(layer_dirs, &counts, n) - } - }; - }; - - super::entropy::ensure_entropy_annexes(index, layer_dirs)?; - let p0 = match n { - None => 1.0, - Some(n) => { - let counts = non_monomorphic_counts(layer_dirs)?; - let total_count: u64 = counts.iter().sum(); - if total_count == 0 { 0.0 } else { (n as f64 / total_count as f64).min(1.0) } - } - }; - layer_dirs - .par_iter() - .map(|layer_dir| -> OKIResult { - Ok(Some(entropy_biased_sample_layer(layer_dir, p0, bias)?)) - }) - .collect() -} - -/// Step 2 entry point: `n` is the total number of families requested across -/// the whole index (`--subsample N`); `layer_dirs` and `counts` must be the -/// same length and in the same order as returned by -/// [`super::family_scan::sibling_layer_dirs`] / -/// [`non_monomorphic_counts`]. -fn sample_layers(layer_dirs: &[PathBuf], counts: &[u64], n: usize) -> OKIResult> { - let total_count: u64 = counts.iter().sum(); - - if total_count <= n as u64 { - // Fewer non-monomorphic minorants in the whole index than requested - // — no sampling needed, keep everything everywhere. - return Ok(vec![None; layer_dirs.len()]); - } - - layer_dirs - .par_iter() - .zip(counts.par_iter()) - .map(|(layer_dir, &count_layer)| -> OKIResult { - if count_layer == 0 { - return Ok(Some(HashSet::new())); - } - let n_layer = ((n as u128 * count_layer as u128) / total_count as u128) as usize; - if n_layer == 0 { - return Ok(Some(HashSet::new())); - } - let selected = reservoir_sample_layer(layer_dir, n_layer)?; - Ok(Some(selected)) - }) - .collect() -} diff --git a/src/obikphylo/src/siblings_old/tests.rs b/src/obikphylo/src/siblings_old/tests.rs deleted file mode 100644 index 880e67af..00000000 --- a/src/obikphylo/src/siblings_old/tests.rs +++ /dev/null @@ -1,557 +0,0 @@ -use std::io::Write; -use std::path::Path; - -use obikseq::{CanonicalKmer, Kmer, Sequence}; -use obikindex::layer::MphfLayer; -use obisys::Reporter; -use tempfile::tempdir; - -use obikalgorithm::Algorithm; -use obikindexer::algorithms::counter::Counter; -use obikindexer::algorithms::dereplicator::Dereplicator; -use obikindexer::algorithms::layer_builder::LayerBuilder; -use obikindexer::algorithms::partitionner::PartitionRouter; -use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode}; - -use super::alignment::SnpAlignmentExt; -use super::build::SiblingAnnexBuildExt; -use super::cardinality::CardinalityExt; -use super::distance::DistanceExt; -use super::entropy::ShannonEntropyExt; -use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME}; -use super::helpers::is_minorant; -use super::sankoff_bundle::SankoffBundleExt; -use super::stats::SiblingStatsExt; -use super::subsample::EntropyBias; -use super::{FamilyMask, SiblingAnnex, ANNEX_FILE_NAME}; - -// k must be >= 11 (project constraint, "k ∈ [11,31]"); k=11, level_max=1, -// theta=0.0 mirror `obiskbuilder`'s own tests (smaller k/level_max -// combinations trip an unrelated pre-existing bug in `obikentropy`'s -// sliding-window ring buffer — not this feature's concern). -const K: usize = 11; -const M: usize = 5; - -/// Build a single-genome index from one in-memory FASTA sequence, driving -/// the same primitives `obikmer`'s `scatter` step uses (minus the -/// multi-file `obipipeline` wrapper — a single sequence needs none of -/// that): normalise -> build superkmers -> route -> write. -/// `cargo test` doesn't install a `tracing` subscriber the way `obikmer`'s -/// CLI does, so `debug!`/etc. are silent no-ops by default — including the -/// `PartitionRunner` instrumentation that would matter most for -/// re-diagnosing a hang here. `try_init` is idempotent across concurrently -/// running tests (later calls just find a subscriber already installed). -fn init_tracing() { - let _ = tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .with_writer(std::io::stderr) - .try_init(); -} - -fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex { - init_tracing(); - let fasta_path = dir.join(format!("{label}.fasta")); - let mut f = std::fs::File::create(&fasta_path).unwrap(); - writeln!(f, ">{label}").unwrap(); - f.write_all(seq).unwrap(); - writeln!(f).unwrap(); - drop(f); - - let index_path = dir.join(format!("{label}.idx")); - let config = IndexConfig { - kmer_size: K, - minimizer_size: M, - n_bits: 0, // 1 partition — keeps the test deterministic and simple - with_counts: false, - evidence: obikindex::layer::IndexMode::Exact, - block_bits: 0, - }; - let idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label))) - .expect("create"); - - let stream = obiread::open_nuc_stream(fasta_path.to_str().unwrap(), K).expect("open fasta"); - let mut router = PartitionRouter::new(&idx); - for page in stream { - let batch = obiskbuilder::build_superkmers_page(page, K, /* level_max */ 1, /* theta */ 0.0); - router.write_batch(batch).expect("write_batch"); - } - router.close().expect("close partition writers"); // also marks scatter done - drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope - - Dereplicator::new(&idx).run().expect("dereplicate"); - Counter::new(&idx).run().expect("count_kmer"); // also writes the spectrum + marks counted - LayerBuilder::new(&idx) - .run() - .expect("build_layers"); // also marks indexed - idx -} - -fn canonical(ascii: &[u8]) -> CanonicalKmer { - Kmer::from_ascii(ascii).unwrap().canonical() -} - -/// Read back the annex entry for a given canonical k-mer from the merged -/// index's (single) partition/layer, asserting it was found at all. -fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask { - let meta = idx.partition_meta(0).unwrap(); - for l in 0..meta.n_layers { - let layer_dir = idx.layer_dir(0, l); - let mphf = MphfLayer::open(&layer_dir).unwrap(); - if let Some(slot) = mphf.find(kmer) { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).unwrap(); - return annex.get(slot).expect("slot must have a computed annex entry"); - } - } - panic!("kmer not found in any layer of partition 0"); -} - -fn merge_two(dir: &Path, g1: &KmerIndex, g2: &KmerIndex) -> KmerIndex { - let mut rep = Reporter::new(); - KmerIndex::merge( - &dir.join("merged.idx"), - &[g1, g2], - MergeMode::Presence, - false, - false, - 1.0, - &mut rep, - ) - .expect("merge") -} - -#[test] -fn sibling_annex_one_sibling_each() { - // k=11, centre = index 5 (0-based). Two genomes, each exactly one - // k-mer, sharing every base except the centre: - // g1 = "AACCGCTTAAG" (centre 'C', base index 1) - // g2 = "AACCGGTTAAG" (centre 'G', base index 2) - // Hand-verified: both stay forward-oriented under canonicalisation - // (each is lexicographically smaller than its own reverse - // complement, since both start with "AA"), and raw(g1) < raw(g2) - // (only differing base: C=0b01 < G=0b10 at the centre) — so g1 is - // the minorant, g2 is not. The mask is a family-wide value: both - // slots must read back the *same* presence bits (1 and 2 set) — but - // only g1's slot should carry the stored minorant flag. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let g1_kmer = canonical(b"AACCGCTTAAG"); - let g2_kmer = canonical(b"AACCGGTTAAG"); - let expected_mask = FamilyMask::EMPTY.with(1).with(2); - - let a = annex_info_for(&merged, g1_kmer); - assert_eq!(a.bits(), expected_mask.bits(), "AACCGCTTAAG"); - assert_eq!(a.siblings(), 1); - assert!(is_minorant(g1_kmer, a, K), "g1 should be the minorant"); - assert!(a.is_minorant(), "g1's stored minorant flag should be set at build time"); - - let b = annex_info_for(&merged, g2_kmer); - assert_eq!(b.bits(), expected_mask.bits(), "AACCGGTTAAG"); - assert_eq!(b.siblings(), 1); - assert!(!is_minorant(g2_kmer, b, K), "g2 should not be the minorant"); - assert!(!b.is_minorant(), "g2's stored minorant flag should not be set"); -} - -#[test] -fn sibling_annex_works_after_pack_sparse() { - // Same fixture as `sibling_annex_one_sibling_each`, but with - // `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged - // index before building the sibling annex — proves `PartitionCache`'s - // sparse-detection (`Mat::SparsePresence`, gated on `presence/ - // is_multi.prsb`) and the generic `TypedLayer` methods it relies on - // actually round-trip through the real build pipeline, not just the - // unit-level `TypedLayer` tests in - // `obikindex::layer`. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.pack_matrices(true).expect("pack_matrices(sparse)"); - - let index_dir = merged.index_dir(0); - assert!( - index_dir.join("layer_0").join("presence").join("is_multi.prsb").exists(), - "pack_matrices(true) must leave the sparse marker file behind" - ); - - merged.build_sibling_annex().expect("build_sibling_annex"); - - let g1_kmer = canonical(b"AACCGCTTAAG"); - let g2_kmer = canonical(b"AACCGGTTAAG"); - let expected_mask = FamilyMask::EMPTY.with(1).with(2); - - let a = annex_info_for(&merged, g1_kmer); - assert_eq!(a.bits(), expected_mask.bits(), "AACCGCTTAAG"); - assert_eq!(a.siblings(), 1); - assert!(is_minorant(g1_kmer, a, K), "g1 should be the minorant"); - assert!(a.is_minorant(), "g1's stored minorant flag should be set at build time"); - - let b = annex_info_for(&merged, g2_kmer); - assert_eq!(b.bits(), expected_mask.bits(), "AACCGGTTAAG"); - assert_eq!(b.siblings(), 1); - assert!(!is_minorant(g2_kmer, b, K), "g2 should not be the minorant"); - assert!(!b.is_minorant(), "g2's stored minorant flag should not be set"); -} - -#[test] -fn sibling_annex_zero_siblings_when_identical_and_no_variant_exists() { - // Same k-mer in both genomes, no other genome around to carry a - // variant -> 0 siblings, trivially its own minorant. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"GATTACAGATC"); - let g2 = build_single_genome_index(dir.path(), "g2", b"GATTACAGATC"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let kmer = canonical(b"GATTACAGATC"); - let mask = annex_info_for(&merged, kmer); - assert_eq!(mask.siblings(), 0, "GATTACAGATC"); - assert_eq!(mask.family_size(), 1); - assert!(is_minorant(kmer, mask, K)); -} - -#[test] -fn sibling_annex_stats_counts_each_family_once_and_per_genome() { - // Reuses the one-sibling-each fixture: a single family of size 2 - // (g1's centre-C form + g2's centre-G form), each genome carrying - // exactly one of the two members. Stats must report exactly one - // family of size 2 (`counts[1] == 1`, since index 1 = size 2), not - // two (which naively summing both slots would give), and both - // genomes represented at size 2, neither at any other size. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let stats = merged.sibling_annex_stats().expect("sibling_annex_stats"); - - assert_eq!(stats.counts, [0, 1, 0, 0], "one family of size 2, counted once"); - assert_eq!(stats.per_genome.len(), 2); - for g in 0..2 { - assert_eq!( - stats.per_genome[g], [0, 1, 0, 0], - "genome {g} should represent exactly one size-2 family" - ); - } -} - -#[test] -fn sibling_family_size_histogram_matches_full_stats_global_counts() { - // Same fixture as `sibling_annex_stats_counts_each_family_once_and_per_genome` - // — the cheap, annex-only histogram must agree with the `counts` half of - // the full (cross-partition) stats pass, without needing a `PartitionCache` - // at all. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let histogram = merged.sibling_family_size_histogram().expect("sibling_family_size_histogram"); - assert_eq!(histogram, [0, 1, 0, 0], "one family of size 2, counted once"); - - let stats = merged.sibling_annex_stats().expect("sibling_annex_stats"); - assert_eq!(histogram, stats.counts, "must agree with the full stats pass's global counts"); -} - -/// Exercises the four sibling-annex consumers that were rewritten to share -/// `family_scan::scan_layer_families` (partition-grouped lookups instead of -/// one lookup per family) — same one-sibling-each fixture as the tests -/// above (g1 carries the family's centre-C member, g2 the centre-G one), -/// hand-verified expected output for each. -#[test] -fn family_scan_consumers_agree_on_one_sibling_each() { - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - // Merge doesn't promise to preserve source order, so resolve each - // genome's index by label rather than assuming g1 -> 0, g2 -> 1. - let idx_of = |label: &str| merged.meta().genomes().unwrap().iter().position(|g| g.label == label).unwrap(); - let (i1, i2) = (idx_of("g1"), idx_of("g2")); - - // snp_pseudo_alignment: one variable family, one column — g1's row - // reads 'C' (its own member), g2's reads 'G'. - let alignment = merged.snp_pseudo_alignment(None, None).expect("snp_pseudo_alignment"); - assert_eq!(alignment.sequences[i1], vec![b'C']); - assert_eq!(alignment.sequences[i2], vec![b'G']); - - // raw_snp_distance: g1's single form (C) != g2's (G) at the family's - // one eligible locus -> a SNP, not a shared site. - let raw = merged.raw_snp_distance().expect("raw_snp_distance"); - assert_eq!(raw.snp[[i1, i2]], 1); - assert_eq!(raw.snp[[i2, i1]], 1); - assert_eq!(raw.shared[[i1, i2]], 0); - assert_eq!(raw.shared[[i2, i1]], 0); - - // cardinality_tally: both genomes carry exactly one member of the - // family (cardinality 1 each) -> one co-occurrence at [1][1]. - let cardinality = merged.cardinality_tally(&raw, 1.0).expect("cardinality_tally"); - assert_eq!(cardinality.counts[1][1], 1); - let total: u64 = cardinality.counts.iter().flatten().sum(); - assert_eq!(total, 1, "no other cardinality pair should be tallied"); - - // base_pair_tally: the one eligible, differing locus is C (base 1) vs - // G (base 2). - let base_pairs = merged.base_pair_tally(&raw, 1.0).expect("base_pair_tally"); - assert_eq!(base_pairs.counts[1][2], 1); - assert_eq!(base_pairs.counts[2][1], 1); - let total: u64 = base_pairs.counts.iter().flatten().sum(); - assert_eq!(total, 2, "no other base pair should be tallied"); - assert_eq!(base_pairs.same, [0, 0, 0, 0], "the two genomes never agree at this locus"); -} - -#[test] -fn sibling_annex_no_empty_masks_after_build() { - let dir = tempdir().unwrap(); - let seq = b"ACGTACGTACGT".repeat(500); - let g1 = build_single_genome_index(dir.path(), "g1", &seq); - g1.build_sibling_annex().expect("build_sibling_annex"); - - let n_layers = g1.n_layers(0).expect("partition meta"); - for l in 0..n_layers { - let layer_dir = g1.layer_dir(0, l); - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open"); - for slot in 0..annex.len() { - let mask = annex.get(slot).expect("slot must have an entry"); - assert!( - mask.family_size() >= 1, - "layer {l} slot {slot}: empty family mask (bits={:04b}) after build — \ - indicates batch-offset bug or unseeded mask slot", - mask.bits() - ); - } - } -} - -#[test] -fn sibling_histogram_does_not_panic_on_partial_last_batch() { - // Regression test for the bug where enumerate_kmers_batch().enumerate() - // used the batch index instead of the sequence offset, leaving the - // trailing slots of a non-multiple-of-BATCH_SIZE layer unseeded and - // producing the minorant-only sentinel (0x10) that caused - // sibling_family_size_histogram to panic with index u32::MAX. - let dir = tempdir().unwrap(); - let seq = b"ACGTACGTACGT".repeat(500); - let g1 = build_single_genome_index(dir.path(), "g1", &seq); - g1.build_sibling_annex().expect("build_sibling_annex"); - let hist = g1.sibling_family_size_histogram().expect("sibling_family_size_histogram"); - let total: u64 = hist.iter().sum(); - assert!(total > 0, "histogram should contain at least one family"); -} - -/// `sibling_annex_one_sibling_each`'s fixture happens to put g2's k-mer in -/// layer 0 and g1's in layer 1 (a fresh single-genome index, then one -/// merge that adds exactly one new layer for content absent from the -/// first — confirmed empirically, not assumed) — a real two-layer index, -/// not a contrived one, so it exercises both write paths -/// `build_layer_sibling_annex` now has to get right: the seeding pass (a -/// slot's own base gets *this* layer's index directly) and the -/// cross-partition/cross-layer resolution pass (`PartitionCache::find` -/// reporting which layer the other family member actually lives in, not -/// just that it exists). -#[test] -fn sibling_annex_records_the_real_layer_of_each_family_member() { - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let n_layers = merged.n_layers(0).unwrap(); - assert_eq!(n_layers, 2, "fixture assumption: one merge, one new layer"); - - let g1_kmer = canonical(b"AACCGCTTAAG"); // own base C (1), sibling base G (2) — lives in layer 1 - let g2_kmer = canonical(b"AACCGGTTAAG"); // own base G (2), sibling base C (1) — lives in layer 0 - - let a = annex_info_for(&merged, g1_kmer); - assert_eq!(a.layer_value(1), Some(1), "g1's own base: seeded with its own layer (1)"); - assert_eq!(a.layer_value(2), Some(0), "g1's sibling (g2's form): resolved to its real layer (0)"); - - let b = annex_info_for(&merged, g2_kmer); - assert_eq!(b.layer_value(2), Some(0), "g2's own base: seeded with its own layer (0)"); - assert_eq!(b.layer_value(1), Some(1), "g2's sibling (g1's form): resolved to its real layer (1)"); -} - -#[test] -fn subsample_and_shannon_on_one_variable_family() { - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let idx_of = |label: &str| merged.meta().genomes().unwrap().iter().position(|g| g.label == label).unwrap(); - let (i1, i2) = (idx_of("g1"), idx_of("g2")); - - // This fixture has exactly one non-monomorphic family (see - // `family_scan_consumers_agree_on_one_sibling_each`) — total_count = 1. - // Asking for far more than that must fall back to "keep everything", - // identical to the unsampled (`None`) result. - let sampled = merged.snp_pseudo_alignment(Some(1000), None).expect("snp_pseudo_alignment (subsample >> total)"); - assert_eq!(sampled.sequences[i1], vec![b'C']); - assert_eq!(sampled.sequences[i2], vec![b'G']); - - // Asking for 0 must keep nothing — an empty (but still one-row-per-genome) - // alignment, not an error. - let empty = merged.snp_pseudo_alignment(Some(0), None).expect("snp_pseudo_alignment (subsample 0)"); - assert!(empty.sequences[i1].is_empty()); - assert!(empty.sequences[i2].is_empty()); - - // shannon_entropy_csv: one row, both entropy15 and entropy4 = 1 bit - // exactly (2 genomes present, each carrying exactly one distinct base, - // uniform split -> -2*(0.5*log2(0.5)) = 1 — the two definitions - // coincide here since no genome carries more than one base). - let csv_path = dir.path().join("shannon.csv"); - merged.shannon_entropy_csv(&csv_path, Some(1000), None).expect("shannon_entropy_csv"); - let csv = std::fs::read_to_string(&csv_path).unwrap(); - let mut lines = csv.lines(); - assert_eq!(lines.next(), Some("layer,family_idx,entropy15,entropy4,family_size,n_genomes_present")); - let row = lines.next().expect("exactly one data row"); - assert!(lines.next().is_none(), "exactly one non-monomorphic family in this fixture"); - let fields: Vec<&str> = row.split(',').collect(); - let entropy15: f64 = fields[2].parse().unwrap(); - let entropy4: f64 = fields[3].parse().unwrap(); - assert!((entropy15 - 1.0).abs() < 1e-6, "entropy15 should be exactly 1 bit, got {entropy15}"); - assert!((entropy4 - 1.0).abs() < 1e-6, "entropy4 should be exactly 1 bit, got {entropy4}"); - assert_eq!(fields[4], "2", "family_size"); - assert_eq!(fields[5], "2", "n_genomes_present"); -} - -#[test] -fn sankoff_bundle_matches_old_separate_calls() { - // Same fixture as `sibling_annex_one_sibling_each`/ - // `subsample_and_shannon_on_one_variable_family`: one non-monomorphic - // family, g1='C' (minorant), g2='G'. Regression check that fusing - // raw_snp_distance/base_pair_tally/cardinality_tally/snp_pseudo_alignment - // into `sankoff_bundle`'s two passes produces identical results to the - // old four independent calls on `Selection::All` (no `--subsample`, - // no `--entropy`) — see `DevDocMD/architecture/siblings.md`, "Wired into - // `pack`...". - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let n_genomes = merged.meta().genomes().unwrap().len(); - let exclude_mask = vec![false; n_genomes]; - let ratio_ceiling = 0.5; - - let expected_raw = merged.raw_snp_distance().expect("raw_snp_distance"); - let expected_base = merged.base_pair_tally(&expected_raw, ratio_ceiling).expect("base_pair_tally"); - let expected_card = merged.cardinality_tally(&expected_raw, ratio_ceiling).expect("cardinality_tally"); - let expected_alignment = merged.snp_pseudo_alignment(None, None).expect("snp_pseudo_alignment"); - - let bundle = merged.sankoff_bundle(None, None, ratio_ceiling, &exclude_mask).expect("sankoff_bundle"); - - assert_eq!(bundle.raw.snp, expected_raw.snp); - assert_eq!(bundle.raw.shared, expected_raw.shared); - assert_eq!(bundle.base_pair_tally.counts, expected_base.counts); - assert_eq!(bundle.base_pair_tally.same, expected_base.same); - assert_eq!(bundle.cardinality_tally.counts, expected_card.counts); - assert_eq!(bundle.alignment.sequences, expected_alignment.sequences); -} - -#[test] -fn base_pair_tally_accumulates_base_a_diagnostic() { - // Diagnostic for a user-reported observation on real data: - // `composition_transitions`'s row/column for base 'A' (index 0) was - // *entirely* zero, including the diagonal (`same[0]`, "A stays A"), - // despite 'A' appearing at ~6% frequency in the pseudo-alignment - // itself. g1/g2 are identical (both central base 'A'), g3 differs - // (central base 'C') at the same family — makes the family variable - // (family_size=2) while g1/g2 form an eligible (ratio 0), ordinary - // pair both resolving `single_form` to `Some(0)` = 'A'. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AAAAAAAAAAA"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AAAAAAAAAAA"); - let g3 = build_single_genome_index(dir.path(), "g3", b"AAAAACAAAAA"); - let mut rep = Reporter::new(); - let merged = KmerIndex::merge( - &dir.path().join("merged.idx"), - &[&g1, &g2, &g3], - MergeMode::Presence, - false, - false, - 1.0, - &mut rep, - ).expect("merge"); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let n_genomes = merged.meta().genomes().unwrap().len(); - let exclude_mask = vec![false; n_genomes]; - let bundle = merged.sankoff_bundle(None, None, 0.5, &exclude_mask).expect("sankoff_bundle"); - - assert!( - bundle.base_pair_tally.same[0] >= 1, - "base 'A' (index 0) must accumulate in BasePairTally.same for an identical, included g1/g2 \ - pair both resolving to 'A' — got same={:?}, counts={:?}", - bundle.base_pair_tally.same, bundle.base_pair_tally.counts, - ); -} - -#[test] -fn entropy_annex_builds_on_demand_and_biases_selection() { - // Same fixture, same single non-monomorphic family, known entropy15 = - // 1.0 bit exactly (see `subsample_and_shannon_on_one_variable_family`). - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let layer_dirs = super::family_scan::sibling_layer_dirs(&merged).expect("layer dirs"); - for layer_dir in &layer_dirs { - assert!( - !layer_dir.join(ENTROPY_ANNEX_FILE_NAME).exists(), - "entropy annex must not exist before any entropy-biased selection is requested" - ); - } - - // This fixture spans 2 layers (a single-genome bootstrap layer plus the - // merged one, see `merge_two`) — the one non-monomorphic family lives - // in whichever layer actually got a non-empty selection, not - // necessarily `layer_dirs[0]`. - let find_layer = |selections: &[super::subsample::LayerSelection]| { - selections.iter().position(|s| matches!(s, Some(set) if !set.is_empty())) - }; - - // mu == the family's real entropy, tiny sigma -> weight == 1 exactly, - // p0 == 1.0 (no `--subsample`) -> the family is selected with - // probability 1 (up to a measure-zero event on the uniform draw). - let bias_center = EntropyBias { mu: 1.0, sigma: 0.5 }; - let selections = super::subsample::compute_selections(&merged, &layer_dirs, None, Some(bias_center)) - .expect("compute_selections (entropy-biased)"); - for layer_dir in &layer_dirs { - assert!( - layer_dir.join(ENTROPY_ANNEX_FILE_NAME).exists(), - "compute_selections must build the entropy annex for every layer on first use" - ); - } - let li = find_layer(&selections).expect("mu at the true entropy must select the one non-monomorphic family, in some layer"); - assert_eq!(selections[li], Some(std::collections::HashSet::from([0]))); - - // The annex now exists — read it back directly and check the stored - // value matches `--shannon`'s own entropy15 computation exactly. - let entropy_annex = EntropyAnnex::open(&layer_dirs[li].join(ENTROPY_ANNEX_FILE_NAME)).expect("open entropy annex"); - let stored = entropy_annex.get(0).expect("family 0 has a real (non-sentinel) entropy value"); - assert!((stored as f64 - 1.0).abs() < 1e-6, "stored entropy should be 1 bit, got {stored}"); - - // mu far from any achievable entropy, tiny sigma -> weight underflows - // to exactly 0.0 -> never selected, deterministically, in any layer. - let bias_far = EntropyBias { mu: 100.0, sigma: 0.001 }; - let selections_far = super::subsample::compute_selections(&merged, &layer_dirs, None, Some(bias_far)) - .expect("compute_selections (entropy-biased, far)"); - assert!(find_layer(&selections_far).is_none(), "mu far from the true entropy must never select it, in any layer"); -} -