Refactor siblings module and enhance sampling logic
Introduces filtering for scan families, handles genome exclusion arguments, refactors the `siblings` module architecture, and clarifies invariants for statistical calculations.
This commit is contained in:
1 parent
996ebaf977
commit
fdfc1eb3dd
27 files changed
+106
-4383
No files matched your search
@@ -35,8 +35,9 @@ fn parse_gamma_shape(s: &str) -> Result<GammaShapeArg, String> {
|
||||
/// `--distance` value — either one of `obikphylo::DistanceMetric`'s
|
||||
/// whole-index metrics (routed to `IndexCache::distance`) or one of
|
||||
/// `obikphylo::SnpDistanceKind`'s `snp-*` corrections (routed to
|
||||
/// `SiblingExt::snp_distance`, the sibling-annex pipeline) — two genuinely
|
||||
/// different code paths behind one CLI vocabulary, see
|
||||
/// `obikphylo::siblings::distance_matrix`, over a tally built by
|
||||
/// `sampled_scan`/`PairwiseReducer`, the sibling-annex pipeline) — two
|
||||
/// genuinely different code paths behind one CLI vocabulary, see
|
||||
/// `DevDocMD/theory/evolutionary_distances.md`, "`--distance` unification".
|
||||
#[derive(clap::ValueEnum, Clone, Copy, Debug)]
|
||||
pub enum DistanceArg {
|
||||
@@ -121,10 +122,10 @@ pub struct PhyloArgs {
|
||||
/// Exclude a genome (by its exact label) — from `--pseudo-alignment`'s
|
||||
/// sampling (a family whose only polymorphism lived in an excluded
|
||||
/// genome is discarded during sampling, not filtered afterward — see
|
||||
/// `obikphylo::siblings::extensions::SiblingExt::snp_pseudo_alignment`'s
|
||||
/// own docs) and from the distance matrix / shared-kmer matrix CSV
|
||||
/// output (row and column both dropped; the underlying computation
|
||||
/// itself is unaffected). Repeatable.
|
||||
/// `obikphylo::siblings::sampled_scan`'s own docs) and from the
|
||||
/// distance matrix / shared-kmer matrix CSV output (row and column
|
||||
/// both dropped; the underlying computation itself is unaffected).
|
||||
/// Repeatable.
|
||||
#[arg(long = "exclude-genome", value_name = "LABEL")]
|
||||
pub exclude_genome: Vec<String>,
|
||||
|
||||
@@ -194,10 +195,9 @@ pub struct PhyloArgs {
|
||||
/// `--pseudo-alignment`/`--sankoff` (mandatory for both) and for a
|
||||
/// `snp-*` `--distance` value (optional there: omitted means exhaustive
|
||||
/// — every non-monomorphic minorant of the whole index, not an
|
||||
/// approximation, see `obikphylo::siblings::SiblingExt::snp_distance`'s
|
||||
/// own docs) — a target, not a guarantee when given (proportional
|
||||
/// per-layer sampling; see
|
||||
/// `obikphylo::siblings::SiblingExt::snp_pseudo_alignment`'s own docs).
|
||||
/// approximation) — a target, not a guarantee when given (proportional
|
||||
/// per-layer sampling; see `obikphylo::siblings::sampled_scan`'s own
|
||||
/// docs).
|
||||
#[arg(long)]
|
||||
pub subsample: Option<usize>,
|
||||
|
||||
@@ -329,7 +329,8 @@ pub struct PhyloArgs {
|
||||
/// the central-position SNP substitution spectrum (`snp-raw`, `snp-jc`,
|
||||
/// `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv`) —
|
||||
/// the latter route to a different computation entirely
|
||||
/// (`SiblingExt::snp_distance`, requires `--sibling-annex` first; see
|
||||
/// (`obikphylo::siblings::sampled_scan` + `distance_matrix`, requires
|
||||
/// `--sibling-annex` first; see
|
||||
/// `DevDocMD/theory/evolutionary_distances.md`, "`--distance`
|
||||
/// unification" for the full catalog and why LogDet/Tajima-Nei/F84/
|
||||
/// HKY85 aren't offered yet).
|
||||
|
||||
@@ -45,10 +45,10 @@ pub fn run(args: PhyloArgs) {
|
||||
let n = labels.len();
|
||||
|
||||
// ── Genome exclusion (`--exclude-genome`) ───────────────────────────────────
|
||||
// Resolved once, up front: `snp_pseudo_alignment` needs it baked into
|
||||
// sampling itself (see its own docs), and the distance/shared-kmer CSV
|
||||
// writers below just skip these rows/columns at write time — the
|
||||
// underlying `cache.distance(...)` computation is unaffected either way.
|
||||
// Resolved once, up front: `sampled_scan` needs it baked into sampling
|
||||
// itself (see its own docs), and the distance/shared-kmer CSV writers
|
||||
// below just skip these rows/columns at write time — the underlying
|
||||
// `cache.distance(...)` computation is unaffected either way.
|
||||
let exclude_mask: Vec<bool> = {
|
||||
let mut mask = vec![false; n];
|
||||
for label in &args.exclude_genome {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Output writers for `--sankoff` — no calibration logic here, just
|
||||
//! formatting: `obikphylo::siblings::SiblingExt::sankoff_bundle` and the
|
||||
//! formatting: `obikphylo::siblings::SankoffBundle::finish` and the
|
||||
//! `cardinality_transition_probs`/`composition_transition_probs`/
|
||||
//! `pairwise_cost_matrix` calibration functions do all the actual work in
|
||||
//! `mod.rs`, this module only serialises their results.
|
||||
@@ -13,8 +13,8 @@ use tracing::info;
|
||||
|
||||
// ── Sankoff pseudo-alignment → FASTA ────────────────────────────────────────
|
||||
//
|
||||
// Same data as `--pseudo-alignment`'s output (`SnpAlignment`/
|
||||
// `snp_pseudo_alignment`), re-coded so its symbols match the accompanying
|
||||
// Same data as `--pseudo-alignment`'s own output (`SnpAlignment`, built by
|
||||
// `AlignmentReducer`), re-coded so its symbols match the accompanying
|
||||
// `--sankoff` matrix output exactly: `0` for the empty/absent state instead
|
||||
// of `-`, which TNT/PhyG would otherwise read as their own gap character
|
||||
// rather than our "family absent" state. Unless `free_loss` (`--free-loss`)
|
||||
|
||||
@@ -14,12 +14,15 @@
|
||||
//! to a lower-level crate if an indexing-side consumer (e.g. default-`k`
|
||||
//! guidance) appears.
|
||||
//!
|
||||
//! `siblings` is being reconnected piece by piece while `obikindex`'s
|
||||
//! stateless audit is caught up with — see the project memory on this. So
|
||||
//! far it only carries the annex-construction path
|
||||
//! (`SiblingExt::build_sibling_annex`); the rest of the old
|
||||
//! `siblings_old` (SNP distance, cardinality, pseudo-alignment, stats) is
|
||||
//! still disconnected.
|
||||
//! `siblings` carries the full pipeline today: annex construction
|
||||
//! (`SiblingExt::build_sibling_annex`), Shannon entropy/family-overlap
|
||||
//! annexes, and the sampled SNP path (pseudo-alignment, `snp-*`
|
||||
//! `--distance`, Sankoff cost-matrix calibration) driven through a single
|
||||
//! shared scan (`siblings::sampled_scan` + `LayerReducer` impls) — see
|
||||
//! `siblings::algorithms::scan`'s own module docs. `siblings_old` is its
|
||||
//! predecessor, fully superseded and not compiled into this crate (no
|
||||
//! `mod siblings_old;` anywhere) — dead source left on disk, not an
|
||||
//! in-progress migration target.
|
||||
|
||||
mod distance;
|
||||
pub mod karlin;
|
||||
|
||||
@@ -78,9 +78,13 @@ fn family_batch_size(n_genomes: usize) -> usize {
|
||||
}
|
||||
|
||||
/// Restricts [`scan_layer_families`] to a subset of a layer's families,
|
||||
/// identified by their iteration-order index. `All` (the only variant any
|
||||
/// current caller uses) keeps a full, unsampled scan; `Some` is exercised
|
||||
/// once sampling (`--subsample`/`--entropy-bias`) is reconnected.
|
||||
/// identified by their iteration-order index. `All` keeps a full,
|
||||
/// unsampled scan (`SiblingExt::shannon_entropy_csv`,
|
||||
/// `sibling_annex_stats`); `Some` bounds it to a caller-picked set —
|
||||
/// `subsample::sample_layer`'s own accepted-candidates batch, or
|
||||
/// `entropy::ensure_layer_entropy_annex`'s non-monomorphic-only
|
||||
/// restriction — skipping the expensive cross-partition resolution
|
||||
/// entirely for every family outside the set.
|
||||
pub enum Selection<'a> {
|
||||
All,
|
||||
Some(&'a std::collections::HashSet<usize>),
|
||||
|
||||
@@ -43,6 +43,7 @@ pub use subsample::{EntropyBias, SurvivingFamily};
|
||||
pub(crate) use annex::build_layer_sibling_annex;
|
||||
pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4};
|
||||
pub(crate) use family_scan::{Selection, scan_layer_families};
|
||||
pub(crate) use minorant_selection::non_monomorphic_reindex_layer;
|
||||
pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram};
|
||||
|
||||
/// Whether every layer number in `cache` fits in a `FamilyMask` field
|
||||
|
||||
@@ -22,15 +22,19 @@ use rkyv::{Archive, Deserialize, Serialize};
|
||||
|
||||
use super::subsample::SurvivingFamily;
|
||||
|
||||
/// One pair's own tallies, unfiltered — `subst` is populated only by
|
||||
/// variable families (a substitution, `bi != bj`, is only possible when the
|
||||
/// family varies *somewhere* in the index; see the module docs), so it
|
||||
/// needs no separate `variable` gate of its own. `same_all` counts every
|
||||
/// agreement (`bi == bj`, any family); `same_variable` is the subset from
|
||||
/// variable families only — [`RawSnpDistanceOutput`] wants the former,
|
||||
/// [`BasePairTally`] the latter (they are *not* the same population — see
|
||||
/// `BasePairTally::same`'s own docs). `cardinality` is restricted to
|
||||
/// variable families only, matching [`CardinalityTally`]'s own scope.
|
||||
/// One pair's own tallies, unfiltered. Both `subst` and `same_all` are
|
||||
/// populated by every family reaching [`reduce_pairwise`] — no separate
|
||||
/// "variable families only" gate needed for either: [`super::subsample::sample_layer`]
|
||||
/// only ever draws from `minorant_selection::eligible_bitset_layer`, which
|
||||
/// contains exclusively non-monomorphic minorants (`family_size() >= 2`,
|
||||
/// enforced by a `debug_assert!` right where a [`SurvivingFamily`] is
|
||||
/// built), so *every* family folded here already varies somewhere in the
|
||||
/// index. `same_all` is what both [`RawSnpDistanceOutput`] and
|
||||
/// [`BasePairTally`] read (there used to be a second, `same_variable`
|
||||
/// field restricted to variable families for the latter — removed once it
|
||||
/// was confirmed, via the invariant above, to always hold the exact same
|
||||
/// values as `same_all`). `cardinality` is populated for every family too,
|
||||
/// same reasoning.
|
||||
///
|
||||
/// `Archive`/`Serialize`/`Deserialize`: part of [`PairwiseTally`]'s own
|
||||
/// `--session` persistence (see that struct's docs) — `rkyv` needs every
|
||||
@@ -40,7 +44,6 @@ use super::subsample::SurvivingFamily;
|
||||
struct PairStats {
|
||||
subst: [[u64; 4]; 4],
|
||||
same_all: [u64; 4],
|
||||
same_variable: [u64; 4],
|
||||
cardinality: [[u64; 5]; 5],
|
||||
}
|
||||
|
||||
@@ -145,7 +148,7 @@ impl PairwiseTally {
|
||||
}
|
||||
let stats = self.pair(i, j);
|
||||
for a in 0..4 {
|
||||
same[a] += stats.same_variable[a];
|
||||
same[a] += stats.same_all[a];
|
||||
for b in 0..4 {
|
||||
counts[a][b] += stats.subst[a][b];
|
||||
}
|
||||
@@ -267,19 +270,17 @@ pub struct BasePairTally {
|
||||
/// `same`, not here.
|
||||
pub counts: [[u64; 4]; 4],
|
||||
/// `same[a]` = number of eligible loci, pooled over included genome
|
||||
/// pairs, where both genomes' single forms are `a` **and** the family
|
||||
/// varies somewhere in the index — a fully invariant family's
|
||||
/// agreement is genome-wide background, not SNP-adjacent signal (see
|
||||
/// `PairStats::same_all` vs `same_variable`).
|
||||
/// pairs, where both genomes' single forms are `a` — every family
|
||||
/// folded into [`PairwiseTally`] already varies somewhere in the index
|
||||
/// by construction (see `PairStats`'s own docs), so there is no
|
||||
/// further "variable families only" restriction to apply here.
|
||||
pub same: [u64; 4],
|
||||
}
|
||||
|
||||
/// Cardinality (`0..=4` members of a family) co-occurrence, pooled over
|
||||
/// every genome pair except `--exclude-genome` ones (see
|
||||
/// [`PairwiseTally::cardinality_tally`]'s own docs for why this is *not*
|
||||
/// gated by `ratio_ceiling` the way [`BasePairTally`] is), restricted to
|
||||
/// variable families (`family_size() >= 2`) — matching
|
||||
/// `snp_pseudo_alignment`'s own scope. `pub`: part of
|
||||
/// gated by `ratio_ceiling` the way [`BasePairTally`] is). `pub`: part of
|
||||
/// [`super::SankoffBundle`]'s public signature.
|
||||
pub struct CardinalityTally {
|
||||
/// `counts[a][b] == counts[b][a]` = number of family sites, pooled over
|
||||
@@ -418,9 +419,9 @@ impl PartitionDispersion {
|
||||
/// same array [`algorithms::alignment::reduce_alignment`] reads too: this
|
||||
/// is what guarantees `cardinality` agrees with what the alignment
|
||||
/// actually shows for the same family/genome (a masked-away genome
|
||||
/// contributes `0`, matching its `?`). `subst`/`same_all`/`same_variable`
|
||||
/// don't need anything beyond `single_form`'s "exactly one bit" check on
|
||||
/// the already-masked state.
|
||||
/// contributes `0`, matching its `?`). No "variable families only" gate
|
||||
/// on any field: see [`PairStats`]'s own docs for why every family reaching
|
||||
/// this function already varies somewhere in the index, by construction.
|
||||
pub(crate) fn reduce_pairwise(
|
||||
survivors: &[SurvivingFamily],
|
||||
partition: usize,
|
||||
@@ -431,7 +432,6 @@ pub(crate) fn reduce_pairwise(
|
||||
let mut partition_subst: u64 = 0;
|
||||
let mut partition_eligible: u64 = 0;
|
||||
for family in survivors {
|
||||
let variable = family.mask.family_size() >= 2;
|
||||
let single_form = |g: usize| -> Option<u8> {
|
||||
family.masked[g].filter(|m| m.count_ones() == 1).map(|m| m.trailing_zeros() as u8)
|
||||
};
|
||||
@@ -451,18 +451,13 @@ pub(crate) fn reduce_pairwise(
|
||||
stats.subst[bj as usize][bi as usize] += 1;
|
||||
} else {
|
||||
stats.same_all[bi as usize] += 1;
|
||||
if variable {
|
||||
stats.same_variable[bi as usize] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if variable {
|
||||
let card_j = masked_cardinality(j);
|
||||
stats.cardinality[card_i][card_j] += 1;
|
||||
if card_i != card_j {
|
||||
stats.cardinality[card_j][card_i] += 1;
|
||||
}
|
||||
let card_j = masked_cardinality(j);
|
||||
stats.cardinality[card_i][card_j] += 1;
|
||||
if card_i != card_j {
|
||||
stats.cardinality[card_j][card_i] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,7 +93,7 @@ pub(crate) fn sibling_family_size_histogram(cache: &IndexCache) -> OKIResult<[u6
|
||||
/// at a finer grain, across partitions — stacking another parallel layer
|
||||
/// loop on top would fight it for locality instead of helping, exactly the
|
||||
/// same reasoning `SiblingExt::build_sibling_annex`/`shannon_entropy_csv`/
|
||||
/// `snp_pseudo_alignment` already follow).
|
||||
/// `algorithms::scan::sampled_scan` already follow).
|
||||
pub(crate) fn sibling_annex_stats(cache: &IndexCache) -> OKIResult<SiblingAnnexStats> {
|
||||
let n_genomes = cache.meta().genomes().len();
|
||||
let fast_mode = is_fast_mode(cache);
|
||||
|
||||
@@ -89,7 +89,7 @@ pub struct SurvivingFamily {
|
||||
/// Gaussian-kernel entropy bias parameters — `mu`/`sigma` are the
|
||||
/// `--entropy`/`--entropy-sd` CLI values (defaulted by the caller, not
|
||||
/// here). `pub`, not `pub(crate)`: part of the public signature of
|
||||
/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`].
|
||||
/// [`super::scan::sampled_scan`].
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct EntropyBias {
|
||||
pub mu: f64,
|
||||
@@ -138,24 +138,19 @@ fn circular_entropy_values(
|
||||
/// function's own iteration order (`cache.partitions()` then `0..n_layer`,
|
||||
/// both deterministic for an unchanged index), to skip *entirely* —
|
||||
/// neither their (cheap) eligibility-bitset lookup's `sample_layer` call
|
||||
/// nor `on_layer` runs for them. Exists for `--session` resume: quotas are
|
||||
/// computed from `total_eligible` (summed over *every* layer regardless of
|
||||
/// `skip_layers`), so a resumed run reproduces the exact same *per-layer
|
||||
/// quotas* an uninterrupted one would have — i.e. how many sites each
|
||||
/// remaining layer contributes stays correct. It does **not** reproduce
|
||||
/// the exact same *sample*: `sample_layer`'s draws come from `rand::rng()`
|
||||
/// (`Self::sample_layer`'s own docs), a per-process, OS-seeded generator
|
||||
/// with no continuity across a process boundary, so the layers processed
|
||||
/// after a resume draw different (but equally valid) random outcomes than
|
||||
/// an uninterrupted run would have. Confirmed by direct test (kill mid-run,
|
||||
/// resume, compare to an uninterrupted run against the same index/params:
|
||||
/// different distance matrices, both legitimate samples). This is
|
||||
/// deliberate, not a defect: run-to-run sampling variance is wanted (see
|
||||
/// `DevDocMD/theory/evolutionary_distances.md`, "`--session`" section) —
|
||||
/// making resume bit-reproducible would require seeding deterministically,
|
||||
/// which would also remove that variance for ordinary repeated runs
|
||||
/// unless scoped carefully, and was explicitly declined for now. `0` for
|
||||
/// every caller not doing checkpointed resume.
|
||||
/// nor `on_layer` runs for them. Built for checkpointed-resume support
|
||||
/// (quotas are computed from `total_eligible`, summed over *every* layer
|
||||
/// regardless of `skip_layers`, so a resumed run reproduces the exact same
|
||||
/// *per-layer quotas* an uninterrupted one would have — i.e. how many
|
||||
/// sites each remaining layer contributes stays correct — without
|
||||
/// reproducing the exact same *sample*: `sample_layer`'s draws come from
|
||||
/// `rand::rng()`, a per-process, OS-seeded generator with no continuity
|
||||
/// across a process boundary). No current caller passes anything but `0`:
|
||||
/// the `--session`-based resume mechanism this was built for was removed
|
||||
/// (see `algorithms::scan`'s own module docs) pending a rebuild on top of
|
||||
/// [`sampled_scan`](super::scan::sampled_scan) — the parameter stays, as
|
||||
/// the seam a future resume mechanism would hook into, rather than being
|
||||
/// ripped out and re-added later.
|
||||
///
|
||||
/// Returns the actual number of *newly processed* sites kept (not
|
||||
/// counting whatever `skip_layers` layers already contributed on an
|
||||
|
||||
@@ -15,7 +15,8 @@ use obisys::progress_bar;
|
||||
|
||||
use crate::siblings::algorithms::{
|
||||
Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4,
|
||||
is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram,
|
||||
is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, sibling_annex_stats,
|
||||
sibling_family_size_histogram,
|
||||
};
|
||||
use crate::siblings::extensions::SiblingBuilder;
|
||||
use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap};
|
||||
@@ -51,8 +52,12 @@ pub trait SiblingExt {
|
||||
/// every cached layer, from an already-built sibling annex (run
|
||||
/// [`build_sibling_annex`](Self::build_sibling_annex) first).
|
||||
///
|
||||
/// Full, unsampled scan of every cached layer (`Selection::All`) — no
|
||||
/// `--subsample`/`--entropy-bias` yet (see the project memory on this).
|
||||
/// Full, unsampled scan of every non-monomorphic minorant of every
|
||||
/// cached layer (`Selection::Some`, restricted the same way
|
||||
/// `ensure_layer_entropy_annex` is — monomorphic minorants have
|
||||
/// nothing to report here, so never pay `scan_layer_families`'s
|
||||
/// cross-partition resolution for them) — no `--subsample`/
|
||||
/// `--entropy-bias` yet (see the project memory on this).
|
||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>;
|
||||
|
||||
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
|
||||
@@ -152,11 +157,9 @@ impl SiblingExt for IndexCache {
|
||||
|
||||
fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> {
|
||||
// Entropy/family-overlap annexes aren't this method's own output,
|
||||
// but building them is exactly the same full scan this method is
|
||||
// about to run anyway — see
|
||||
// `SiblingBuilder::ensure_entropy_and_overlap_annexes`'s own docs
|
||||
// for why that's a side effect worth taking here rather than a
|
||||
// separate, explicit step.
|
||||
// but ensuring they exist is a cheap prerequisite check, not a
|
||||
// reason to reuse their scan here — see below, this method
|
||||
// deliberately runs its own, differently-scoped one.
|
||||
self.ensure_entropy_and_overlap_annexes()?;
|
||||
|
||||
let n_genomes = self.meta().genomes().len();
|
||||
@@ -170,6 +173,19 @@ impl SiblingExt for IndexCache {
|
||||
for part in self.partitions() {
|
||||
let n_layer = self.n_layer(part).unwrap_or(0);
|
||||
for l in 0..n_layer {
|
||||
let layer = self
|
||||
.get_layer(part, l)
|
||||
.expect("cache-consistent: n_layer(part) already bounds l");
|
||||
// Bounded to non-monomorphic minorants only, same reasoning
|
||||
// as `ensure_layer_entropy_annex`'s own docs: monomorphism
|
||||
// is already known from the annex bits alone (~98% of a
|
||||
// layer's minorants, measured on real data), so there is no
|
||||
// reason to pay `scan_layer_families`'s expensive
|
||||
// cross-partition resolution for families this method
|
||||
// skips right below anyway.
|
||||
let reindex = non_monomorphic_reindex_layer(layer.dir())?;
|
||||
let eligible: std::collections::HashSet<usize> = reindex.keys().copied().collect();
|
||||
|
||||
let mut write_err = None;
|
||||
scan_layer_families(
|
||||
self,
|
||||
@@ -177,14 +193,15 @@ impl SiblingExt for IndexCache {
|
||||
l,
|
||||
n_genomes,
|
||||
fast_mode,
|
||||
&Selection::All,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, mask, genome_mask| {
|
||||
if write_err.is_some() {
|
||||
return;
|
||||
}
|
||||
if mask.family_size() < 2 {
|
||||
return; // monomorphic family — entropy trivially 0, no signal, skip
|
||||
}
|
||||
debug_assert!(
|
||||
mask.family_size() >= 2,
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
let Some((h15, n_present)) = family_entropy(genome_mask) else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::subsample::EntropyBias;
|
||||
|
||||
/// IUPAC ambiguity code for a per-genome family presence mask (bit `b` set
|
||||
/// iff the genome carries the member whose own central base is `b`):
|
||||
/// single bit -> the plain base; 2 or 3 bits -> the matching IUPAC
|
||||
/// ambiguity code (preserves partial information instead of collapsing to
|
||||
/// `N`, the same convention used for diploid heterozygous VCF/FASTA sites);
|
||||
/// all 4 bits -> `N`; no bits (genome carries none of the family's observed
|
||||
/// members) -> `-` (no data at this locus for this genome).
|
||||
pub(super) fn iupac_code(mask: u8) -> u8 {
|
||||
match mask & 0b1111 {
|
||||
0b0000 => b'-',
|
||||
0b0001 => b'A',
|
||||
0b0010 => b'C',
|
||||
0b0100 => b'G',
|
||||
0b1000 => b'T',
|
||||
0b0101 => b'R', // A/G
|
||||
0b1010 => b'Y', // C/T
|
||||
0b0110 => b'S', // C/G
|
||||
0b1001 => b'W', // A/T
|
||||
0b1100 => b'K', // G/T
|
||||
0b0011 => b'M', // A/C
|
||||
0b1110 => b'B', // C/G/T
|
||||
0b1101 => b'D', // A/G/T
|
||||
0b1011 => b'H', // A/C/T
|
||||
0b0111 => b'V', // A/C/G
|
||||
0b1111 => b'N',
|
||||
_ => unreachable!("masked to 4 bits"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A SNP-only pseudo-alignment: one row (byte sequence, IUPAC-coded) per
|
||||
/// genome, one column per variable family (`family_size() >= 2` — monomorphic
|
||||
/// families carry no signal and are skipped, unlike `raw_snp_distance`'s
|
||||
/// tally which does count them as `shared`). Column order is the same,
|
||||
/// deterministic sweep order as the annex build (partition, then layer, then
|
||||
/// slot) — arbitrary but stable and identical across genomes, which is all a
|
||||
/// pseudo-alignment needs (there is no natural genomic coordinate to sort by
|
||||
/// once flanks are dropped). See `DevDocMD/theory/evolutionary_distances.md`,
|
||||
/// "Multi-genome framing: family as pseudo-alignment column".
|
||||
pub struct SnpAlignment {
|
||||
/// `sequences[g]` = genome `g`'s IUPAC-coded row, same length for every
|
||||
/// genome (`sequences.len()` columns).
|
||||
pub sequences: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
/// Adds [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) to `KmerIndex` —
|
||||
/// phylo-domain functionality, kept out of `obikindex` itself (see
|
||||
/// `DevDocMD/architecture/siblings.md`).
|
||||
pub trait SnpAlignmentExt {
|
||||
/// Build the SNP-only pseudo-alignment from an already-built sibling
|
||||
/// annex (run [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) first).
|
||||
///
|
||||
/// `subsample`: `--subsample N` — cap the number of variable families
|
||||
/// retained to (approximately) `N`, sampled proportionally per layer
|
||||
/// among non-monomorphic minorants (see
|
||||
/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`").
|
||||
/// `None` keeps today's behaviour: every non-monomorphic minorant of
|
||||
/// every layer. `entropy_bias`: `--entropy`/`--entropy-sd` — weight
|
||||
/// that sample (or, with `subsample = None`, filter the whole index)
|
||||
/// by a Gaussian kernel on each family's entropy — see
|
||||
/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection".
|
||||
fn snp_pseudo_alignment(
|
||||
&self,
|
||||
subsample: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<SnpAlignment>;
|
||||
}
|
||||
|
||||
impl SnpAlignmentExt for KmerIndex {
|
||||
fn snp_pseudo_alignment(
|
||||
&self,
|
||||
subsample: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<SnpAlignment> {
|
||||
let n_parts = self.n_partitions();
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = self.meta().config.with_counts;
|
||||
let k = self.kmer_size();
|
||||
let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?);
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(self)?;
|
||||
let selections =
|
||||
super::subsample::compute_selections(self, &layer_dirs, subsample, entropy_bias)?;
|
||||
|
||||
let pb = progress_bar("snp_pseudo_alignment", layer_dirs.len() as u64, "layers");
|
||||
// One layer at a time, not `par_iter()` over layers — same
|
||||
// rationale as `build_sibling_annex`: running many layers'
|
||||
// `scan_layer_families` concurrently would each group their own
|
||||
// lookups by partition internally, but interleave those sweeps
|
||||
// across layers at the OS level, scattering page-cache access over
|
||||
// every partition at once again and defeating the whole point of
|
||||
// the grouping. Columns appended straight into `sequences` as each
|
||||
// family comes back from `scan_layer_families` (bounded-batch, not
|
||||
// whole-layer) — no intermediate per-layer buffer, layer order
|
||||
// giving a single deterministic column order across the whole index.
|
||||
let mut sequences: Vec<Vec<u8>> = vec![Vec::new(); n_genomes];
|
||||
for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) {
|
||||
let selection = match layer_selection {
|
||||
None => Selection::All,
|
||||
Some(set) => Selection::Some(set),
|
||||
};
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&selection,
|
||||
|_family_idx, mask, genome_mask| {
|
||||
if mask.family_size() < 2 {
|
||||
return; // monomorphic family — no signal, skip
|
||||
}
|
||||
for (g, &m) in genome_mask.iter().enumerate() {
|
||||
sequences[g].push(iupac_code(m));
|
||||
}
|
||||
},
|
||||
)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(SnpAlignment { sequences })
|
||||
}
|
||||
}
|
||||
@@ -1,368 +0,0 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obikindex::layer::MphfLayer;
|
||||
use obipipeline::ThrottleGuard;
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::OKIResult;
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::helpers::{central_base, is_minorant};
|
||||
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder};
|
||||
|
||||
// ── obipipeline data types ─────────────────────────────────────────────────
|
||||
|
||||
/// A batch of this layer's distinct k-mers (iteration-order index + k-mer),
|
||||
/// the pipeline's source item — batched, not one k-mer per item, so that
|
||||
/// pipeline messages and their synchronisation cost stay amortised over
|
||||
/// thousands of lookups (see `build_layer_sibling_annex`'s comment on
|
||||
/// `BATCH_SIZE`). Carries the throttle permit for the whole batch, moved
|
||||
/// (not cloned) into the corresponding `VariantBatch` — a 1-to-1 transform,
|
||||
/// unlike the fan-out `Flat` stage this replaced, needs no `Arc` sharing.
|
||||
///
|
||||
/// The `usize` is this k-mer's position in `iter_kmers()`'s enumeration
|
||||
/// order, **not** an MPHF slot — see `DevDocMD/architecture/siblings.md`. It
|
||||
/// is the same index the annex file is written under.
|
||||
struct SourceBatch {
|
||||
items: Vec<(usize, CanonicalKmer)>,
|
||||
_permit: ThrottleGuard,
|
||||
}
|
||||
|
||||
/// One batch's worth of central-substitution variants (up to 3 per source
|
||||
/// k-mer), each already routed to its destination partition and carrying
|
||||
/// its own central base (0=A/1=C/2=G/3=T) — the mask bit it will set on a
|
||||
/// hit. `(dest_partition, variant, source_order, base)` per entry, where
|
||||
/// `source_order` is the source k-mer's iteration-order index (see
|
||||
/// `SourceBatch`), not an MPHF slot.
|
||||
struct VariantBatch {
|
||||
items: Vec<(usize, CanonicalKmer, usize, u8)>,
|
||||
_permit: ThrottleGuard,
|
||||
}
|
||||
|
||||
enum SibData {
|
||||
Batch(SourceBatch),
|
||||
Variants(VariantBatch),
|
||||
}
|
||||
|
||||
/// Adds [`build_sibling_annex`](Self::build_sibling_annex) to `KmerIndex`.
|
||||
pub trait SiblingAnnexBuildExt {
|
||||
/// Build the sibling-count/minorant annex for every layer of every
|
||||
/// partition of this (already built) index, writing one annex file per
|
||||
/// layer alongside its existing index files. Safe to call again later
|
||||
/// (e.g. after a fresh `merge`) — each run simply overwrites the annex
|
||||
/// files of the index it is called on.
|
||||
///
|
||||
/// Construction only — no statistics gathered here on purpose: this is
|
||||
/// meant to run routinely (it is the artefact the SNP-family distances
|
||||
/// will consume), while the sibling-count distribution
|
||||
/// ([`sibling_annex_stats`](super::stats::SiblingStatsExt::sibling_annex_stats))
|
||||
/// is a separate, occasional diagnostic pass over the result, not run
|
||||
/// every time.
|
||||
///
|
||||
/// Cross-partition/cross-layer lookups are required (a k-mer's siblings
|
||||
/// can live in any partition), but the layer loop itself — and thus the
|
||||
/// annex file this produces — stays local to one layer at a time.
|
||||
fn build_sibling_annex(&self) -> OKIResult<()>;
|
||||
}
|
||||
|
||||
impl SiblingAnnexBuildExt for KmerIndex {
|
||||
fn build_sibling_annex(&self) -> OKIResult<()> {
|
||||
let n_parts = self.n_partitions();
|
||||
|
||||
tracing::info!("opening {n_parts} partition(s) for the sibling-annex sweep");
|
||||
let cache = Arc::new(PartitionCache::build(
|
||||
self,
|
||||
n_parts,
|
||||
self.meta().config.with_counts,
|
||||
)?);
|
||||
|
||||
let pb = progress_bar("sibling_annex", n_parts as u64, "partitions");
|
||||
let mut total_slots: u64 = 0;
|
||||
for part in 0..n_parts {
|
||||
let index_dir = self.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
pb.inc(1);
|
||||
continue;
|
||||
}
|
||||
let meta = self.partition_meta(part)?;
|
||||
|
||||
let mut part_slots: u64 = 0;
|
||||
for l in 0..meta.n_layers {
|
||||
part_slots += build_layer_sibling_annex(
|
||||
self,
|
||||
&self.layer_dir(part, l),
|
||||
n_parts,
|
||||
l,
|
||||
&cache,
|
||||
)?;
|
||||
}
|
||||
total_slots += part_slots;
|
||||
pb.inc(1);
|
||||
pb.set_message(format!(
|
||||
"partition {part}: {part_slots} kmers ({total_slots} total)"
|
||||
));
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
tracing::info!("sibling annex built — {total_slots} kmers across {n_parts} partitions");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of distinct k-mers (annex entries) processed, for
|
||||
/// progress reporting. A free function, not a `KmerIndex` method — called
|
||||
/// only from `build_sibling_annex` above, in the same file.
|
||||
fn build_layer_sibling_annex(
|
||||
index: &KmerIndex,
|
||||
layer_dir: &Path,
|
||||
n_parts: usize,
|
||||
l: usize,
|
||||
cache: &Arc<PartitionCache>,
|
||||
) -> OKIResult<u64> {
|
||||
let mphf = MphfLayer::open(layer_dir)?;
|
||||
let k = index.kmer_size();
|
||||
let n = mphf.n();
|
||||
|
||||
// Whether this layer's family-member fields can carry a real layer
|
||||
// number (exploited by `family_scan::scan_layer_families`'s fast path)
|
||||
// or must stay presence-only — a single fact about the whole index,
|
||||
// decided once by `cache` (built from the same `PartitionMeta` this
|
||||
// function would otherwise re-derive) so the writer and every reader
|
||||
// agree; see `PartitionCache::fast_mode`'s docs.
|
||||
let fast_mode = cache.fast_mode();
|
||||
|
||||
// ── The annex file itself is the reconciliation state, indexed by
|
||||
// this layer's k-mer iteration order (the physical layout of
|
||||
// `unitigs.bin`), never by MPHF slot — this layer's own k-mers are
|
||||
// known members by construction, so no evidence check, no MPHF
|
||||
// slot, and no slot -> k-mer reconstruction is needed or legitimate
|
||||
// here (see `DevDocMD/architecture/siblings.md`: the MPHF is not
|
||||
// invertible, and evidence answers membership, not identity). No
|
||||
// separate in-memory accumulator: every concurrent write goes
|
||||
// straight through `SiblingAnnexBuilder::atomic_slot` into the
|
||||
// mmap — a layer can hold billions of k-mers, so a `Vec` shadowing
|
||||
// the whole file in RAM just to copy it out again at the end (the
|
||||
// previous design) doubles memory for no benefit once the builder
|
||||
// itself can be written concurrently.
|
||||
//
|
||||
// `Arc<SiblingAnnexBuilder>`, not a bare `SiblingAnnexBuilder` —
|
||||
// `Pipe::apply` requires its source iterator to be `Send + 'static`
|
||||
// (its items are dispatched to worker threads that outlive this
|
||||
// call), so the batch-generating closure below needs an owned
|
||||
// handle it can move in, not a borrow of a local. `atomic_slot`,
|
||||
// not `set`, for every concurrent write below: the gather phase
|
||||
// parallelises across destination partitions (independent
|
||||
// `cache.find` calls, safe to run concurrently) and their hits can
|
||||
// land on arbitrary, possibly-shared source entries — a lock-free
|
||||
// `fetch_or` avoids needing any synchronisation beyond that.
|
||||
// Reclaimed as a plain owned value (`Arc::try_unwrap`) once every
|
||||
// concurrent phase below is done, for the sequential finalisation
|
||||
// pass. ─────────────────────────────────────────────────────────
|
||||
let annex_path = layer_dir.join(ANNEX_FILE_NAME);
|
||||
let builder = Arc::new(SiblingAnnexBuilder::new(n, &annex_path)?);
|
||||
|
||||
// ── obipipeline: a *batch* transform, not a per-k-mer `Flat` one —
|
||||
// the actual cross-partition lookup reuses
|
||||
// `KmerPartition::query_partition_with` (the same batching mechanism
|
||||
// `obikmer query` already uses: open a partition's files once,
|
||||
// answer a whole batch of queries against it) instead of one lookup
|
||||
// per pipeline item. A per-item lookup (tried first) reopened/
|
||||
// re-mmap'd every target partition's files on every single variant
|
||||
// — fine at toy scale, but ~90% system time against a real index,
|
||||
// observed in practice. A *later* attempt still pushed one pipeline
|
||||
// message per generated variant (a `Flat` stage, `SourceItem` =>
|
||||
// `VariantQuery`, one k-mer in => up to 3 variants out as separate
|
||||
// messages) — cheaper than reopening files, but sampling a real run
|
||||
// showed most wall-clock time going into per-message channel
|
||||
// send/notify syscalls instead of the lookup itself: the pipeline's
|
||||
// whole point is amortising synchronisation over a batch, and a
|
||||
// single k-mer's ≤3 variants is far too fine a granularity for
|
||||
// that. Batching `BATCH_SIZE` source k-mers into one pipeline item
|
||||
// — a plain 1-to-1 (`|`, not `||`) transform, batch in, batch of
|
||||
// variants out, one message either way — keeps the per-message
|
||||
// synchronisation cost amortised over thousands of lookups instead
|
||||
// of one to three. `BATCH_SIZE` was originally tuned back when the
|
||||
// per-k-mer cost inside this stage (minimiser via `RollingStat`)
|
||||
// was ~3x higher than it is now (see `CanonicalKmerOf::minimizer`,
|
||||
// a direct O(k) bit-arithmetic replacement) — at the old per-k-mer
|
||||
// cost, dispatch overhead (the scheduler's single-threaded `Select`
|
||||
// loop: one channel round-trip per batch) was negligible next to
|
||||
// the work each batch represented; now that the work itself is
|
||||
// cheaper, that fixed per-batch overhead is proportionally larger,
|
||||
// so a bigger batch amortises it over more k-mers again. ─────────
|
||||
const BATCH_SIZE: usize = 32768;
|
||||
let n_workers = obisys::effective_parallelism();
|
||||
let capacity = 256;
|
||||
|
||||
// Throttling limits how many *batches* are in flight at once — the
|
||||
// permit is acquired per batch (not per k-mer) in the source
|
||||
// thread, and released once its `VariantBatch` has been read out of
|
||||
// the pipeline by the accumulation loop below. See
|
||||
// `obipipeline::throttle`'s docs for why this is required, not
|
||||
// optional, once a `Flat`-style stage sits in the pipeline.
|
||||
//
|
||||
// Batches stream straight from `unitigs.bin` via
|
||||
// `enumerate_kmers_batch` — never a full-layer `Vec` collect (a
|
||||
// layer can hold billions of k-mers; see the "no full collect"
|
||||
// rule). Each k-mer's own base is seeded straight into the annex in
|
||||
// this same pass, since this is exactly the iteration order the
|
||||
// annex is keyed on — no separate seeding pass needed.
|
||||
let seed_builder = Arc::clone(&builder);
|
||||
let batches = mphf
|
||||
.enumerate_kmers_batch(BATCH_SIZE)
|
||||
.map(move |(start, kmers)| {
|
||||
kmers
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, kmer)| {
|
||||
let base = central_base(kmer, k);
|
||||
let bits = if fast_mode {
|
||||
FamilyMask::layer_bits(base, l)
|
||||
} else {
|
||||
FamilyMask::presence_bits(base)
|
||||
};
|
||||
seed_builder
|
||||
.atomic_slot(start + i)
|
||||
.fetch_or(bits, Ordering::Relaxed);
|
||||
(start + i, kmer)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
|
||||
// Diagnostic: count still-empty annex slots after seeding + cross-partition
|
||||
// resolution. A non-zero count here means some k-mer's own base was never
|
||||
// written (should be impossible after the batch_offset fix above).
|
||||
let check_empty = |label: &str| {
|
||||
let empty = (0..n).filter(|&slot| builder.get(slot).is_none()).count();
|
||||
if empty > 0 {
|
||||
tracing::warn!(
|
||||
"{label}: {empty} mask slots still empty after resolution (layer {layer_dir:?})"
|
||||
);
|
||||
}
|
||||
};
|
||||
let throttled = obipipeline::throttle(batches, n_workers).map(|t| SourceBatch {
|
||||
items: t.item,
|
||||
_permit: t.guard,
|
||||
});
|
||||
|
||||
let pipe = obipipeline::make_pipe! {
|
||||
SibData : SourceBatch => VariantBatch,
|
||||
| {
|
||||
move |batch: SourceBatch| -> VariantBatch {
|
||||
let mut items = Vec::with_capacity(batch.items.len() * 3);
|
||||
for (order, kmer) in batch.items {
|
||||
for variant in kmer.central_canonical_neighbors() {
|
||||
if variant == kmer {
|
||||
continue;
|
||||
}
|
||||
items.push((
|
||||
variant.partition(n_parts),
|
||||
variant,
|
||||
order,
|
||||
central_base(variant, k),
|
||||
));
|
||||
}
|
||||
}
|
||||
VariantBatch { items, _permit: batch._permit }
|
||||
}
|
||||
} : Batch => Variants,
|
||||
};
|
||||
|
||||
// ── Group generated variants by destination partition. `cache`
|
||||
// holds every partition already mmap'd (no more `open()` cost), but
|
||||
// `mmap` pages are still loaded on demand and can be evicted — a
|
||||
// lookup is not free just because the file isn't reopened. Grouping
|
||||
// keeps one partition's pages hot while its whole batch is resolved,
|
||||
// instead of faulting pages in and out as lookups jump between
|
||||
// partitions in whatever order the pipeline happens to produce
|
||||
// them. Each batch's throttle permit drops here, once accumulated.
|
||||
let mut outgoing: Vec<Vec<(CanonicalKmer, usize, u8)>> =
|
||||
(0..n_parts).map(|_| Vec::new()).collect();
|
||||
for vb in pipe.apply(throttled, n_workers, capacity) {
|
||||
for (dest_partition, variant, source_order, base) in vb.items {
|
||||
outgoing[dest_partition].push((variant, source_order, base));
|
||||
}
|
||||
}
|
||||
|
||||
check_empty("after_seeding");
|
||||
|
||||
// ── Resolve each partition's batch against the cache, parallelised
|
||||
// over roughly equal-sized *chunks*, not over partitions — a plain
|
||||
// `outgoing.par_iter()` over `n_parts` buckets gives one thread the
|
||||
// whole of one partition's bucket, however large, and a lot of
|
||||
// buckets are far from equal: a central-base substitution changes
|
||||
// the minimiser (and thus routes to a different partition) only
|
||||
// when the winning minimiser window overlaps the central position.
|
||||
// For k=31/m=11 that is ~11 of the 21 possible windows — so *most*
|
||||
// of the remaining ~10/21 send the variant right back to the
|
||||
// partition already being built. That self-partition bucket ends up
|
||||
// far larger than any other, so the naive per-partition split
|
||||
// leaves one thread grinding through it alone long after every
|
||||
// other partition's (tiny) bucket is done — confirmed by sampling a
|
||||
// real run: one thread solely in `MphfLayer::find` while the rest
|
||||
// of the pool sits idle. Splitting each non-empty bucket into
|
||||
// `chunk_size`-sized pieces first keeps this pass's per-partition
|
||||
// locality (each chunk is still contiguous within one partition,
|
||||
// still resolved via `cache.find` grouped by that partition) while
|
||||
// letting Rayon spread a single oversized bucket across several
|
||||
// threads instead of pinning it to one. ──────────────────────────
|
||||
let chunk_size = ((outgoing.iter().map(Vec::len).sum::<usize>() / n_workers).max(1)).min(4096);
|
||||
let work: Vec<(usize, &[(CanonicalKmer, usize, u8)])> = outgoing
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, q)| !q.is_empty())
|
||||
.flat_map(|(dest, q)| q.chunks(chunk_size).map(move |c| (dest, c)))
|
||||
.collect();
|
||||
work.par_iter().for_each(|&(dest, chunk)| {
|
||||
for &(variant, source_order, base) in chunk {
|
||||
if let Some(layer) = cache.find(dest, variant) {
|
||||
let bits = if fast_mode {
|
||||
FamilyMask::layer_bits(base, layer)
|
||||
} else {
|
||||
FamilyMask::presence_bits(base)
|
||||
};
|
||||
builder
|
||||
.atomic_slot(source_order)
|
||||
.fetch_or(bits, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
check_empty("after_resolution");
|
||||
|
||||
// ── Write the layer's annex file, indexed by iteration order — a
|
||||
// second streamed pass over `unitigs.bin` (via `enumerate_kmers`),
|
||||
// now that every entry's mask is final; never a `Vec` hold of the
|
||||
// whole layer. The minorant flag is computed here, not by every
|
||||
// later reader: this is the one place the whole family's *final*
|
||||
// mask and this entry's own k-mer (already in hand, no extra
|
||||
// lookup) are both available together. Every consumer that only
|
||||
// needs to know "is this k-mer the family's minorant" — the common
|
||||
// case, since a family is tallied once, at its minorant — reads
|
||||
// the bit straight back instead of re-deriving it (re-scanning
|
||||
// `unitigs.bin` and re-hashing through the MPHF, the cost
|
||||
// `is_minorant` was cheap to *compute* but expensive to *get the
|
||||
// inputs for* every time).
|
||||
// Every concurrent phase above is done and its `Arc` clone (the
|
||||
// seeding pass's `seed_builder`) has already been dropped along
|
||||
// with the now-fully-drained `batches` iterator — this is the only
|
||||
// remaining strong reference, so reclaiming plain ownership for the
|
||||
// sequential `set` calls below is guaranteed to succeed.
|
||||
let mut builder = Arc::try_unwrap(builder).unwrap_or_else(|_| {
|
||||
panic!("sibling-annex builder still shared after every concurrent phase completed")
|
||||
});
|
||||
for (order, kmer) in mphf.enumerate_kmers() {
|
||||
let final_mask = builder.get(order).expect("seeded by construction");
|
||||
let minorant = is_minorant(kmer, final_mask, k);
|
||||
builder.set(order, final_mask.with_minorant(minorant));
|
||||
}
|
||||
builder.close()?;
|
||||
|
||||
Ok(n as u64)
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikindex::layer::KmerLayer;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::{KmerIndex, OKIResult};
|
||||
|
||||
/// Every partition's already-open layers, built **once** for the whole
|
||||
/// `build_sibling_annex` run and shared (read-only) across every lookup, in
|
||||
/// every source layer, for the rest of the run — not reopened/re-mmap'd per
|
||||
/// query, nor per source layer.
|
||||
///
|
||||
/// Confirmed necessary by sampling a real run: routing lookups through
|
||||
/// `KmerPartition::query_partition_with` (the same batching `obikmer query`
|
||||
/// uses) still reopens+re-mmaps every target partition's files on every
|
||||
/// call, and it is called once per destination partition **per source
|
||||
/// layer** — for an index with many layers this repeats the same
|
||||
/// `MphfLayer::open`/`Evidence::open`/`PersistentBitMatrix::open` work over
|
||||
/// and over. Parallelising those calls (see the gather step below) spread
|
||||
/// the redundant work across more cores but did not reduce it: sampling
|
||||
/// showed Rayon workers spending their time inside repeated `open()`
|
||||
/// syscalls, not computation. This cache amortises that cost to once per
|
||||
/// partition for the entire run, regardless of how many source layers or
|
||||
/// lookups follow.
|
||||
///
|
||||
/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the
|
||||
/// format-erased `Count`/`Presence` handle, not a sibling-specific type:
|
||||
/// this module used to bundle its own `Mat` enum here, duplicating exactly
|
||||
/// what `Layer` now does one crate down (see
|
||||
/// `DevDocMD/implementation/partition_layer_cache.md`). Its
|
||||
/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`.
|
||||
pub(super) struct PartitionCache {
|
||||
mats: Vec<Vec<KmerLayer>>,
|
||||
/// Whether every `FamilyMask` field in this index's sibling annexes can
|
||||
/// be trusted as a real layer index (`n_layers <= 7`, the same decision
|
||||
/// `build_layer_sibling_annex` makes when writing them) — computed once
|
||||
/// here, from `PartitionMeta::n_layers` (not `mats[p].len()`, which can
|
||||
/// undercount if a layer failed to open), reused by both the annex
|
||||
/// writer (`build.rs`) and reader (`family_scan.rs`) so the two never
|
||||
/// disagree. `n_layers` is guaranteed identical across every partition
|
||||
/// (a merge adds one layer to all of them at once), so the first
|
||||
/// non-empty partition's count speaks for the whole index.
|
||||
fast_mode: bool,
|
||||
}
|
||||
|
||||
impl PartitionCache {
|
||||
pub(super) fn build(index: &KmerIndex, n_parts: usize, with_counts: bool) -> OKIResult<Self> {
|
||||
let pb = progress_bar("open_partitions", n_parts as u64, "partitions");
|
||||
let built: Vec<(Vec<KmerLayer>, usize)> = (0..n_parts)
|
||||
.into_par_iter()
|
||||
.map(|part| -> OKIResult<(Vec<KmerLayer>, usize)> {
|
||||
let index_dir = index.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
pb.inc(1);
|
||||
return Ok((Vec::new(), 0));
|
||||
}
|
||||
let meta = index.partition_meta(part)?;
|
||||
let mut mats = Vec::with_capacity(meta.n_layers);
|
||||
for l in 0..meta.n_layers {
|
||||
let Ok(mat) =
|
||||
KmerLayer::open(&index.layer_dir(part, l), &meta.mode, with_counts)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
mats.push(mat);
|
||||
}
|
||||
pb.inc(1);
|
||||
Ok((mats, meta.n_layers))
|
||||
})
|
||||
.collect::<OKIResult<Vec<_>>>()?;
|
||||
pb.finish_and_clear();
|
||||
|
||||
let n_layers = built.iter().map(|&(_, n)| n).find(|&n| n > 0).unwrap_or(0);
|
||||
let fast_mode = n_layers <= 7;
|
||||
if !fast_mode {
|
||||
tracing::warn!(
|
||||
"index has {n_layers} layers (>7) — sibling-annex fast layer lookup disabled; \
|
||||
consider compacting this index"
|
||||
);
|
||||
}
|
||||
|
||||
let mats = built.into_iter().map(|(m, _)| m).collect();
|
||||
Ok(Self { mats, fast_mode })
|
||||
}
|
||||
|
||||
/// See the [`fast_mode`](Self::fast_mode) field docs.
|
||||
pub(super) fn fast_mode(&self) -> bool {
|
||||
self.fast_mode
|
||||
}
|
||||
|
||||
/// Existence lookup of `variant` in partition `dest_partition`: tries
|
||||
/// each of the partition's already-open layers in turn, stopping at the
|
||||
/// first hit and reporting which layer it was — `find_slot`, not
|
||||
/// `sub_matrix`/`carries`, no data read needed for a plain existence
|
||||
/// check.
|
||||
pub(super) fn find(&self, dest_partition: usize, variant: CanonicalKmer) -> Option<usize> {
|
||||
self.mats
|
||||
.get(dest_partition)?
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(li, mat)| mat.find_slot(variant).map(|_| li))
|
||||
}
|
||||
|
||||
/// Resolve many `(variant, family_idx, base)` queries against one
|
||||
/// partition's matrices at once, calling `on_hit(family_idx, base, g)`
|
||||
/// for every genome `g` that carries the resolved variant.
|
||||
///
|
||||
/// Genome-major, not query-major: `PersistentBitMatrix`/
|
||||
/// `PersistentCompactIntMatrix` are stored one contiguous block per
|
||||
/// genome (column), slot as the offset within it (see
|
||||
/// `obicompactvec::bitmatrix::packed::PackedBitMatrix` — each column's
|
||||
/// own `mmap` region). Resolving query-by-query (`for query { for genome
|
||||
/// { mat.carries(genome, slot) } }`, this function's predecessor) visits
|
||||
/// every genome's block once *per query* — for a batch of thousands of
|
||||
/// queries against ~90 genomes, that is thousands of jumps into each of
|
||||
/// ~90 widely separated multi-MB regions, in query order, not genome
|
||||
/// order: the access pattern a column-major layout is least suited to.
|
||||
/// Grouping first (by layer, since each layer's matrix is a separate
|
||||
/// column set) and letting `fill_sub_matrix_carries` sort each group by
|
||||
/// slot internally, then visiting genome by genome, turns that into ~90
|
||||
/// mostly-sequential sweeps through one column's own bytes — the
|
||||
/// layout's fast axis — confirmed by sampling a real run:
|
||||
/// `PersistentBitMatrix::get` dominated wall-clock time, mostly blocked
|
||||
/// on page faults, even after every partition/batch locality fix above
|
||||
/// this in the traversal.
|
||||
/// `queries`' 4th element (the annex-recorded layer) is ignored here —
|
||||
/// every layer is probed with evidence regardless, since this is the
|
||||
/// `!fast_mode` fallback where that field isn't trustworthy (see
|
||||
/// [`find_presence_batch_fast`](Self::find_presence_batch_fast)). Same
|
||||
/// tuple shape as the fast path purely so callers don't need two
|
||||
/// query representations.
|
||||
pub(super) fn find_presence_batch(
|
||||
&self,
|
||||
dest_partition: usize,
|
||||
queries: &[(CanonicalKmer, usize, u8, u8)],
|
||||
n_genomes: usize,
|
||||
mut on_hit: impl FnMut(usize, u8, usize),
|
||||
) {
|
||||
let Some(mats) = self.mats.get(dest_partition) else {
|
||||
return;
|
||||
};
|
||||
|
||||
// First hit wins, same semantics as the old per-query loop (a
|
||||
// variant present in an earlier layer shadows later ones).
|
||||
let mut by_layer: Vec<Vec<(usize, usize, u8)>> = vec![Vec::new(); mats.len()];
|
||||
for &(variant, family_idx, base, _layer) in queries {
|
||||
for (li, mat) in mats.iter().enumerate() {
|
||||
if let Some(slot) = mat.find_slot(variant) {
|
||||
by_layer[li].push((slot, family_idx, base));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (li, hits) in by_layer.into_iter().enumerate() {
|
||||
if hits.is_empty() {
|
||||
continue;
|
||||
}
|
||||
resolve_layer_hits(&mats[li], &hits, n_genomes, &mut on_hit);
|
||||
}
|
||||
}
|
||||
|
||||
/// Same contract as [`find_presence_batch`](Self::find_presence_batch),
|
||||
/// but only valid when [`fast_mode`](Self::fast_mode) is true for this
|
||||
/// whole index: every query's 4th element is trusted outright as its
|
||||
/// real destination layer (recorded once, evidence-checked, at
|
||||
/// annex-build time by [`find`](Self::find)) — no per-layer probing, a
|
||||
/// direct `index_batch` (no evidence) straight to that layer instead.
|
||||
/// Safe for the same reason `build_layer_sibling_annex`'s own
|
||||
/// iteration-pipeline lookups are: this exact variant's membership in
|
||||
/// this exact layer was already positively established once: replaying
|
||||
/// it via `index`/`index_batch` here isn't a fresh, unverified
|
||||
/// assumption, it's reusing a fact already paid for.
|
||||
pub(super) fn find_presence_batch_fast(
|
||||
&self,
|
||||
dest_partition: usize,
|
||||
queries: &[(CanonicalKmer, usize, u8, u8)],
|
||||
n_genomes: usize,
|
||||
mut on_hit: impl FnMut(usize, u8, usize),
|
||||
) {
|
||||
let Some(mats) = self.mats.get(dest_partition) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut by_layer: Vec<Vec<(CanonicalKmer, usize, u8)>> = vec![Vec::new(); mats.len()];
|
||||
for &(variant, family_idx, base, layer) in queries {
|
||||
if let Some(bucket) = by_layer.get_mut(layer as usize) {
|
||||
bucket.push((variant, family_idx, base));
|
||||
}
|
||||
}
|
||||
|
||||
for (li, entries) in by_layer.into_iter().enumerate() {
|
||||
if entries.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mat = &mats[li];
|
||||
let variants: Vec<CanonicalKmer> =
|
||||
entries.iter().map(|&(variant, _, _)| variant).collect();
|
||||
let slots = mat.hash_batch(&variants);
|
||||
let hits: Vec<(usize, usize, u8)> = slots
|
||||
.into_iter()
|
||||
.zip(entries.iter())
|
||||
.map(|(slot, &(_, family_idx, base))| (slot, family_idx, base))
|
||||
.collect();
|
||||
resolve_layer_hits(mat, &hits, n_genomes, &mut on_hit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared tail of both [`PartitionCache::find_presence_batch`] and
|
||||
/// [`PartitionCache::find_presence_batch_fast`]: given one layer's already-
|
||||
/// resolved `(slot, family_idx, base)` hits, sweep genome-major (via
|
||||
/// `fill_sub_matrix_carries`, buffer-reusing, no per-call allocation) and
|
||||
/// call `on_hit` for every carry. The two callers differ only in *how*
|
||||
/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this
|
||||
/// is everything after that.
|
||||
fn resolve_layer_hits(
|
||||
mat: &KmerLayer,
|
||||
hits: &[(usize, usize, u8)],
|
||||
n_genomes: usize,
|
||||
on_hit: &mut impl FnMut(usize, u8, usize),
|
||||
) {
|
||||
let n_cols = mat.n_cols().min(n_genomes);
|
||||
let slots: Vec<usize> = hits.iter().map(|&(slot, _, _)| slot).collect();
|
||||
let mut carries: Vec<Vec<bool>> = (0..n_cols).map(|_| Vec::new()).collect();
|
||||
mat.fill_sub_matrix_carries(&slots, &mut carries);
|
||||
for (g, col) in carries.iter().enumerate() {
|
||||
for (&(_, family_idx, base), &carries_it) in hits.iter().zip(col.iter()) {
|
||||
if carries_it {
|
||||
on_hit(family_idx, base, g);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,336 +0,0 @@
|
||||
//! Cardinality × composition decomposition of the 16-state transition cost
|
||||
//! matrix — replaces `sankoff::build_cost_matrix`'s elementary-edge graph
|
||||
//! and its Floyd-Warshall shortest-path closure, which double-counts
|
||||
//! multi-hop transitions once IQ-TREE's own matrix exponential composes
|
||||
//! them again. See `DevDocMD/theory/evolutionary_distances.md`, "`R` via
|
||||
//! `exp(-cost)` is wrong for a CTMC; cardinality/composition decomposition".
|
||||
//!
|
||||
//! Two small, directly-estimated first-order Markov models (diagonals
|
||||
//! included — "stay the same" is a real, calibrated outcome, not implicit):
|
||||
//! a 5-state cardinality model (`0..=4` members of a family) and a 4-state
|
||||
//! base-composition model (`A,C,G,T`, unambiguous/cardinality-1 loci only —
|
||||
//! composition bias is a substitution phenomenon, only meaningful when
|
||||
//! cardinality is conserved). Composed per pair of 16 states via a
|
||||
//! parsimony-style pairing of the non-shared elements (see
|
||||
//! `pairwise_transition_matrix`'s doc), then symmetrised — Sankoff
|
||||
//! parsimony's score is only independent of root placement (as required,
|
||||
//! since TNT/PhyG search *unrooted* trees) if the cost matrix is
|
||||
//! symmetric, the discrete-parsimony analogue of CTMC reversibility.
|
||||
|
||||
use super::{BasePairTally, CardinalityTally};
|
||||
|
||||
/// Row-stochastic 5×5 cardinality transition probabilities (`0..=4`),
|
||||
/// diagonal included ("stay at the same cardinality"), from
|
||||
/// [`CardinalityTally`]'s pooled, symmetric co-occurrence counts.
|
||||
pub fn cardinality_transition_probs(tally: &CardinalityTally) -> [[f64; 5]; 5] {
|
||||
let mut p = [[0.0f64; 5]; 5];
|
||||
for a in 0..5 {
|
||||
let row_sum: u64 = tally.counts[a].iter().sum();
|
||||
if row_sum == 0 {
|
||||
continue;
|
||||
}
|
||||
for b in 0..5 {
|
||||
p[a][b] = tally.counts[a][b] as f64 / row_sum as f64;
|
||||
}
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Row-stochastic 4×4 base-composition transition probabilities
|
||||
/// (`A,C,G,T`), diagonal included ("stay the same base"), from
|
||||
/// [`BasePairTally`]'s pooled substitution (off-diagonal) and agreement
|
||||
/// (`same`, diagonal) counts — unambiguous, cardinality-1 loci only.
|
||||
pub fn composition_transition_probs(tally: &BasePairTally) -> [[f64; 4]; 4] {
|
||||
let mut p = [[0.0f64; 4]; 4];
|
||||
for a in 0..4 {
|
||||
let row_sum = tally.same[a] + (0..4).map(|b| tally.counts[a][b]).sum::<u64>();
|
||||
if row_sum == 0 {
|
||||
continue;
|
||||
}
|
||||
p[a][a] = tally.same[a] as f64 / row_sum as f64;
|
||||
for b in 0..4 {
|
||||
if a != b {
|
||||
p[a][b] = tally.counts[a][b] as f64 / row_sum as f64;
|
||||
}
|
||||
}
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Enumerate every injective mapping from `small` (indices `0..small.len()`)
|
||||
/// into `large` (indices `0..large.len()`, `large.len() >= small.len()`),
|
||||
/// calling `visit` with each full permutation of chosen `large` indices.
|
||||
/// `small.len() <= 4` always here (at most 4 bases can be lost or gained),
|
||||
/// so a plain recursive enumeration (at most `4! = 24` calls) is simplest —
|
||||
/// not worth a dependency for.
|
||||
fn for_each_injection(small_len: usize, large_len: usize, visit: &mut dyn FnMut(&[usize])) {
|
||||
let mut chosen = vec![0usize; small_len];
|
||||
let mut used = vec![false; large_len];
|
||||
fn go(pos: usize, chosen: &mut [usize], used: &mut [bool], visit: &mut dyn FnMut(&[usize])) {
|
||||
if pos == chosen.len() {
|
||||
visit(chosen);
|
||||
return;
|
||||
}
|
||||
for l in 0..used.len() {
|
||||
if used[l] {
|
||||
continue;
|
||||
}
|
||||
used[l] = true;
|
||||
chosen[pos] = l;
|
||||
go(pos + 1, chosen, used, visit);
|
||||
used[l] = false;
|
||||
}
|
||||
}
|
||||
if small_len == 0 {
|
||||
visit(&[]);
|
||||
return;
|
||||
}
|
||||
go(0, &mut chosen, &mut used, visit);
|
||||
}
|
||||
|
||||
/// Parsimony-style pairing cost for the non-shared elements of a
|
||||
/// transition: pair every element of the smaller of `lost`/`gained` with
|
||||
/// some element of the larger one, choosing the pairing that minimises
|
||||
/// total `-ln(P_composition)` (maximises likelihood) — the discrete
|
||||
/// analogue of "prefer a substitution over an independent loss+gain".
|
||||
/// Leftover elements of the larger side (`|large|-|small|` of them) are
|
||||
/// *not* charged here — they're pure cardinality change, already priced by
|
||||
/// `P_cardinality(|A|→|B|)` in the caller.
|
||||
fn best_pairing_cost(lost: &[u8], gained: &[u8], p_comp: &[[f64; 4]; 4]) -> f64 {
|
||||
let (small, large) = if lost.len() <= gained.len() { (lost, gained) } else { (gained, lost) };
|
||||
if small.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut best = f64::INFINITY;
|
||||
for_each_injection(small.len(), large.len(), &mut |perm| {
|
||||
let mut cost = 0.0;
|
||||
for (i, &li) in perm.iter().enumerate() {
|
||||
let (x, y) = (small[i], large[li]);
|
||||
let p = p_comp[x as usize][y as usize];
|
||||
cost += if p > 0.0 { -p.ln() } else { f64::INFINITY };
|
||||
}
|
||||
if cost < best {
|
||||
best = cost;
|
||||
}
|
||||
});
|
||||
best
|
||||
}
|
||||
|
||||
/// The full 16-state transition cost matrix, replacing
|
||||
/// `sankoff::build_cost_matrix`. For every pair of states `A`, `B`
|
||||
/// (bitmasks, bit `0..4` = `A,C,G,T`): `shared = A∩B` contributes
|
||||
/// `P_composition(x→x)` per shared base (diagonal — "stayed"); the
|
||||
/// remaining `lost = A\B`, `gained = B\A` are parsimony-paired into
|
||||
/// substitutions via [`best_pairing_cost`], and whatever's left over after
|
||||
/// pairing (only possible on one side, since one side is fully consumed)
|
||||
/// is *pure* cardinality change, priced once via `P_cardinality(|A|→|B|)`
|
||||
/// — never chained through intermediate states. Row-normalised (`P(A→·)`
|
||||
/// sums to `1` over all `B`), converted to a cost via `-ln`, then
|
||||
/// symmetrised (`(cost(A,B)+cost(B,A))/2` — the row-normalised `P` is not
|
||||
/// symmetric in general, but a Sankoff parsimony cost must be, so the
|
||||
/// score is independent of where an unrooted tree gets rooted).
|
||||
///
|
||||
/// `free_loss`: drop the `P_cardinality(|A|→|B|)` factor entirely (never
|
||||
/// added to `log_p`) — the same low/incomplete-coverage argument that
|
||||
/// justifies recoding whole-family non-detection as `?` (see
|
||||
/// `DevDocMD/theory/evolutionary_distances.md`, "Locus dropout under
|
||||
/// incomplete coverage") applies one level down: whether a genome shows 1
|
||||
/// vs 2 (etc.) detected members of a *present* family is exactly as
|
||||
/// vulnerable to sampling failure as whether the family was detected at
|
||||
/// all. Without this, `∅`-involving transitions are neutralised (via the
|
||||
/// `?` recoding, bypassing this matrix's row/column 0 entirely) but
|
||||
/// cardinality changes *between two otherwise-detected, non-empty* states
|
||||
/// (e.g. `{A} -> {A,C}`) still carried the same calibrated
|
||||
/// `P_cardinality` penalty as any other gain/loss — inconsistent with
|
||||
/// `--free-loss`'s own rationale. With the factor dropped, cost is driven
|
||||
/// only by composition matching (shared-base retention and paired
|
||||
/// substitutions), never by a state pair's cardinality difference alone.
|
||||
pub fn pairwise_cost_matrix(p_card: &[[f64; 5]; 5], p_comp: &[[f64; 4]; 4], free_loss: bool) -> [[f64; 16]; 16] {
|
||||
let mut log_p = [[0.0f64; 16]; 16]; // ln(P), *before* row-normalisation
|
||||
for a in 0u8..16 {
|
||||
for b in 0u8..16 {
|
||||
let shared = a & b;
|
||||
let lost: Vec<u8> = (0..4).filter(|&i| a & (1 << i) != 0 && b & (1 << i) == 0).collect();
|
||||
let gained: Vec<u8> = (0..4).filter(|&i| b & (1 << i) != 0 && a & (1 << i) == 0).collect();
|
||||
|
||||
let mut lp = 0.0; // accumulate ln(P), so 0.0 = probability 1
|
||||
if !free_loss {
|
||||
let card_a = a.count_ones() as usize;
|
||||
let card_b = b.count_ones() as usize;
|
||||
let p_c = p_card[card_a][card_b];
|
||||
lp += if p_c > 0.0 { p_c.ln() } else { f64::NEG_INFINITY };
|
||||
}
|
||||
|
||||
for i in 0..4u8 {
|
||||
if shared & (1 << i) != 0 {
|
||||
let p = p_comp[i as usize][i as usize];
|
||||
lp += if p > 0.0 { p.ln() } else { f64::NEG_INFINITY };
|
||||
}
|
||||
}
|
||||
lp -= best_pairing_cost(&lost, &gained, p_comp);
|
||||
|
||||
log_p[a as usize][b as usize] = lp;
|
||||
}
|
||||
}
|
||||
|
||||
// Row-normalise and convert to cost entirely in log-space
|
||||
// (log-sum-exp), never exponentiating a raw `log_p` value directly —
|
||||
// for a transition reachable only through several low-probability
|
||||
// steps, `log_p.exp()` can underflow to exactly `0.0` (anything below
|
||||
// roughly `-709` does, in `f64`), silently turning a real, if small,
|
||||
// probability into a hard `+∞` cost. Observed in practice: a
|
||||
// calibrated matrix with many such exact-zero entries is numerically
|
||||
// unstable for IQ-TREE's own likelihood-derivative computation
|
||||
// ("Numerical underflow for lh-derivative"). `log_sum_exp` never
|
||||
// exponentiates anything above `0` (every term is shifted by the
|
||||
// row's own max first), so it stays accurate across the full range
|
||||
// `f64` can represent, not just what survives a direct `exp()`.
|
||||
let mut cost = [[0.0f64; 16]; 16];
|
||||
for a in 0..16 {
|
||||
let row_max = log_p[a].iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
||||
if row_max == f64::NEG_INFINITY {
|
||||
// Every transition out of this state has probability 0 in the
|
||||
// calibration data — genuinely unreachable, not underflow.
|
||||
cost[a] = [f64::INFINITY; 16];
|
||||
continue;
|
||||
}
|
||||
let log_sum = row_max + log_p[a].iter().map(|&lp| (lp - row_max).exp()).sum::<f64>().ln();
|
||||
for b in 0..16 {
|
||||
// `log_sum - log_p[a][b]` is `+∞` automatically when
|
||||
// `log_p[a][b] == -∞` (IEEE 754: finite − (−∞) = +∞) — no
|
||||
// separate branch needed for a genuinely zero-probability
|
||||
// transition.
|
||||
cost[a][b] = log_sum - log_p[a][b];
|
||||
}
|
||||
}
|
||||
|
||||
// Symmetrise (see doc comment: parsimony on an unrooted tree requires
|
||||
// a symmetric cost matrix).
|
||||
let mut sym = [[0.0f64; 16]; 16];
|
||||
for a in 0..16 {
|
||||
for b in 0..16 {
|
||||
sym[a][b] = if a == b { 0.0 } else { (cost[a][b] + cost[b][a]) / 2.0 };
|
||||
}
|
||||
}
|
||||
sym
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cardinality_probs_row_normalised() {
|
||||
let mut counts = [[0u64; 5]; 5];
|
||||
counts[0][0] = 10;
|
||||
counts[0][1] = 5;
|
||||
counts[1][0] = 5;
|
||||
counts[1][1] = 2;
|
||||
let tally = CardinalityTally { counts };
|
||||
let p = cardinality_transition_probs(&tally);
|
||||
assert!((p[0].iter().sum::<f64>() - 1.0).abs() < 1e-12);
|
||||
assert!((p[1].iter().sum::<f64>() - 1.0).abs() < 1e-12);
|
||||
assert!((p[0][0] - 10.0 / 15.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn composition_probs_include_diagonal() {
|
||||
let mut tally = BasePairTally { counts: [[0u64; 4]; 4], same: [0u64; 4] };
|
||||
tally.same[0] = 90; // A stays A 90 times
|
||||
tally.counts[0][1] = 10; // A -> C 10 times
|
||||
tally.counts[1][0] = 10;
|
||||
let p = composition_transition_probs(&tally);
|
||||
assert!((p[0][0] - 0.9).abs() < 1e-12);
|
||||
assert!((p[0][1] - 0.1).abs() < 1e-12);
|
||||
assert!((p[0].iter().sum::<f64>() - 1.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pairwise_cost_matrix_is_symmetric_and_zero_diagonal() {
|
||||
// A plausible-looking pair of models (not calibrated from real
|
||||
// data — just needs every state reachable with nonzero probability
|
||||
// for this structural test).
|
||||
let p_card = [
|
||||
[0.5, 0.3, 0.15, 0.04, 0.01],
|
||||
[0.3, 0.4, 0.2, 0.08, 0.02],
|
||||
[0.15, 0.2, 0.4, 0.2, 0.05],
|
||||
[0.04, 0.08, 0.2, 0.5, 0.18],
|
||||
[0.01, 0.02, 0.05, 0.18, 0.74],
|
||||
];
|
||||
let p_comp = [
|
||||
[0.7, 0.1, 0.15, 0.05],
|
||||
[0.1, 0.7, 0.05, 0.15],
|
||||
[0.15, 0.05, 0.7, 0.1],
|
||||
[0.05, 0.15, 0.1, 0.7],
|
||||
];
|
||||
let cost = pairwise_cost_matrix(&p_card, &p_comp, false);
|
||||
for a in 0..16 {
|
||||
assert_eq!(cost[a][a], 0.0);
|
||||
for b in 0..16 {
|
||||
assert!((cost[a][b] - cost[b][a]).abs() < 1e-9, "cost[{a}][{b}]={} cost[{b}][{a}]={}", cost[a][b], cost[b][a]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn free_loss_ignores_cardinality_transition_probs() {
|
||||
// A skewed cardinality model (cardinality change made artificially
|
||||
// expensive) must have zero effect on the cost matrix once
|
||||
// `free_loss` is set — the whole point of the flag.
|
||||
let p_card_uniform = [[0.2f64; 5]; 5];
|
||||
let p_card_skewed = [
|
||||
[0.96, 0.01, 0.01, 0.01, 0.01],
|
||||
[0.01, 0.96, 0.01, 0.01, 0.01],
|
||||
[0.01, 0.01, 0.96, 0.01, 0.01],
|
||||
[0.01, 0.01, 0.01, 0.96, 0.01],
|
||||
[0.01, 0.01, 0.01, 0.01, 0.96],
|
||||
];
|
||||
let p_comp = [
|
||||
[0.7, 0.1, 0.15, 0.05],
|
||||
[0.1, 0.7, 0.05, 0.15],
|
||||
[0.15, 0.05, 0.7, 0.1],
|
||||
[0.05, 0.15, 0.1, 0.7],
|
||||
];
|
||||
let cost_uniform = pairwise_cost_matrix(&p_card_uniform, &p_comp, true);
|
||||
let cost_skewed = pairwise_cost_matrix(&p_card_skewed, &p_comp, true);
|
||||
for a in 0..16 {
|
||||
for b in 0..16 {
|
||||
assert!(
|
||||
(cost_uniform[a][b] - cost_skewed[a][b]).abs() < 1e-9,
|
||||
"cost[{a}][{b}] differs between cardinality models under free_loss: {} vs {}",
|
||||
cost_uniform[a][b], cost_skewed[a][b],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn underflow_prone_transition_gets_finite_cost_not_infinite() {
|
||||
// Every off-diagonal composition probability is tiny (1e-200) but
|
||||
// not exactly zero — small enough that the naive `log_p.exp()`
|
||||
// (exponentiate before row-normalising) hard-underflows to `0.0`
|
||||
// in `f64` well before the probability is truly zero, silently
|
||||
// turning a real, if minuscule, probability into `+∞` cost. The
|
||||
// log-sum-exp normalisation in `pairwise_cost_matrix` must keep
|
||||
// this finite instead.
|
||||
let p_card = [[0.2f64; 5]; 5];
|
||||
let tiny = 1e-200;
|
||||
let mut p_comp = [[tiny; 4]; 4];
|
||||
for i in 0..4 {
|
||||
p_comp[i][i] = 1.0 - 3.0 * tiny;
|
||||
}
|
||||
|
||||
let cost = pairwise_cost_matrix(&p_card, &p_comp, false);
|
||||
|
||||
// {A,C} -> {G,T}: no shared bases, both members substituted — the
|
||||
// shape most prone to underflow (several tiny-probability factors
|
||||
// multiplied together).
|
||||
let a = 0b0011u8; // A, C
|
||||
let b = 0b1100u8; // G, T
|
||||
assert!(
|
||||
cost[a as usize][b as usize].is_finite(),
|
||||
"cost must stay finite for a merely tiny (not exactly zero) probability, got {}",
|
||||
cost[a as usize][b as usize]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use ndarray::Array2;
|
||||
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::distance::RawSnpDistanceOutput;
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
|
||||
/// See [`KmerIndex::cardinality_tally`].
|
||||
pub struct CardinalityTally {
|
||||
/// `counts[a][b] == counts[b][a]` = number of family sites, pooled over
|
||||
/// included genome pairs, where one genome's family cardinality
|
||||
/// (popcount of its presence mask, `0..=4`) is `a` and the other's is
|
||||
/// `b`. Diagonal is real data here (both genomes at the same
|
||||
/// cardinality), unlike [`super::distance::BasePairTally::counts`].
|
||||
pub counts: [[u64; 5]; 5],
|
||||
}
|
||||
|
||||
/// Adds [`cardinality_tally`](Self::cardinality_tally) to `KmerIndex`.
|
||||
pub trait CardinalityExt {
|
||||
/// Cardinality co-occurrence, pooled only over genome pairs whose
|
||||
/// overall SNP ratio in `raw` is at or below `ratio_ceiling` — same
|
||||
/// saturation/no-data exclusion discipline as
|
||||
/// [`base_pair_tally`](Self::base_pair_tally). Unlike
|
||||
/// `scan_family_pairs` (which resolves each genome to a single form and
|
||||
/// silently drops any genome carrying more than one member of the
|
||||
/// family), this needs the *full* per-genome presence mask — a family
|
||||
/// member count of 2, 3 or 4 is exactly the signal being tallied, not
|
||||
/// noise to discard — so it re-implements the traversal rather than
|
||||
/// reusing that helper.
|
||||
///
|
||||
/// Restricted to variable families (`family_size() >= 2`), matching
|
||||
/// `snp_pseudo_alignment`'s own scope — briefly removed, then
|
||||
/// reinstated: without it, the diagonal is dominated by genome-wide
|
||||
/// invariant background (family_size()<2 loci vastly outnumber the
|
||||
/// ones that ever vary anywhere), which is inconsistent with the
|
||||
/// `+ASC`-corrected alignment this matrix is ultimately used with —
|
||||
/// `+ASC` exists specifically because the likelihood only ever sees
|
||||
/// variable sites, so a rate model calibrated mostly from invariant
|
||||
/// background sites doesn't describe the population it's applied to.
|
||||
/// Verified empirically: removing the filter measurably worsened a
|
||||
/// real IQ-TREE run (log-likelihood dropped, `NNI search needs
|
||||
/// unusual large number of steps to converge` warnings appeared) — see
|
||||
/// `DevDocMD/theory/evolutionary_distances.md` for the full account.
|
||||
/// [`base_pair_tally`](Self::base_pair_tally)'s own diagonal (`same`)
|
||||
/// gets the matching restriction via `scan_family_pairs`'s new
|
||||
/// `variable` flag, rather than a `family_size()` check of its own (it
|
||||
/// doesn't have direct access to the family's mask).
|
||||
fn cardinality_tally(
|
||||
&self,
|
||||
raw: &RawSnpDistanceOutput,
|
||||
ratio_ceiling: f64,
|
||||
) -> OKIResult<CardinalityTally>;
|
||||
}
|
||||
|
||||
impl CardinalityExt for KmerIndex {
|
||||
fn cardinality_tally(
|
||||
&self,
|
||||
raw: &RawSnpDistanceOutput,
|
||||
ratio_ceiling: f64,
|
||||
) -> OKIResult<CardinalityTally> {
|
||||
let n_parts = self.n_partitions();
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = self.meta().config.with_counts;
|
||||
let k = self.kmer_size();
|
||||
let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| {
|
||||
if i == j {
|
||||
return false;
|
||||
}
|
||||
let snp = raw.snp[[i, j]];
|
||||
let total = snp + raw.shared[[i, j]];
|
||||
total > 0 && (snp as f64 / total as f64) <= ratio_ceiling
|
||||
});
|
||||
|
||||
let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?);
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(self)?;
|
||||
|
||||
let pb = progress_bar("cardinality_tally", layer_dirs.len() as u64, "layers");
|
||||
// One layer at a time — see `snp_pseudo_alignment`'s comment for why
|
||||
// `par_iter()` over layers would defeat `scan_layer_families`'s
|
||||
// partition-grouped locality. Tallied straight into `total` as each
|
||||
// family comes back — no per-layer buffer.
|
||||
let mut total = [[0u64; 5]; 5];
|
||||
for layer_dir in &layer_dirs {
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&Selection::All,
|
||||
|_family_idx, mask, genome_mask| {
|
||||
if mask.family_size() < 2 {
|
||||
// Fully invariant family (never varies anywhere in
|
||||
// the index) — genome-wide background, not
|
||||
// SNP-adjacent signal; would otherwise swamp the
|
||||
// diagonal (`c=1/c=1` etc.), which needs to reflect
|
||||
// the same variable-families-only population the
|
||||
// `+ASC`-corrected alignment/likelihood actually
|
||||
// models. See `base_pair_tally`'s `variable` gate
|
||||
// on its own `same` diagonal for the matching fix.
|
||||
return;
|
||||
}
|
||||
|
||||
for i in 0..n_genomes {
|
||||
let card_i = genome_mask[i].count_ones() as usize;
|
||||
for j in (i + 1)..n_genomes {
|
||||
if !included[[i, j]] {
|
||||
continue;
|
||||
}
|
||||
let card_j = genome_mask[j].count_ones() as usize;
|
||||
total[card_i][card_j] += 1;
|
||||
if card_i != card_j {
|
||||
total[card_j][card_i] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(CardinalityTally { counts: total })
|
||||
}
|
||||
}
|
||||
@@ -1,244 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use ndarray::Array2;
|
||||
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
|
||||
/// Raw p-distance restricted to loci that are single-copy in **both**
|
||||
/// genomes of a pair — the "stringent / paralogy-aware" locus eligibility
|
||||
/// rule (`DevDocMD/theory/evolutionary_distances.md`, "Locus eligibility"),
|
||||
/// without the JC/K2P/LogDet finalisation math: `snp[i,j]` + `shared[i,j]`
|
||||
/// is the number of eligible loci, `snp[i,j] / (snp[i,j] + shared[i,j])` is
|
||||
/// `p_hat`. A quick, self-contained way to sanity-check the estimator
|
||||
/// against a real index before the full `SnpTally` design is built.
|
||||
///
|
||||
/// A locus (family, tallied once at its minorant) is eligible for pair
|
||||
/// `(i, j)` iff genome `i` carries exactly one of the family's observed
|
||||
/// forms **and** genome `j` carries exactly one (possibly a different one)
|
||||
/// — presence-only: a genome carrying the same form twice (a same-allele
|
||||
/// duplicate) is indistinguishable from carrying it once when only a
|
||||
/// presence matrix is available, so such cases are not excluded here even
|
||||
/// when a count index exists. See "Locus eligibility", stringent rule, for
|
||||
/// why this matters and how a count index would close the gap — left as a
|
||||
/// follow-up, not applied here.
|
||||
pub struct RawSnpDistanceOutput {
|
||||
/// n×n count of eligible loci where the two genomes' single forms differ.
|
||||
pub snp: Array2<u64>,
|
||||
/// n×n count of eligible loci where the two genomes' single forms agree.
|
||||
pub shared: Array2<u64>,
|
||||
}
|
||||
|
||||
/// Shared traversal behind [`DistanceExt::raw_snp_distance`] and
|
||||
/// [`DistanceExt::base_pair_tally`]: for every family (tallied once, at its
|
||||
/// minorant) of every layer of the already-built sibling annex, resolves
|
||||
/// each genome's single observed form (`None` if absent or
|
||||
/// ambiguous/multi-copy), then calls `on_pair(acc, i, j, bi, bj, variable)`
|
||||
/// for every genome pair `(i, j)` where both are unambiguous and
|
||||
/// single-copy (`bi == bj` means shared at that locus, `bi != bj` means a
|
||||
/// SNP). `variable` is the family's own `family_size() >= 2` (true if more
|
||||
/// than one member is observed *anywhere* in the family, i.e. it isn't
|
||||
/// fully invariant across the whole index) — `raw_snp_distance` ignores it
|
||||
/// (a fully-invariant family is still legitimately "shared"), but callers
|
||||
/// whose diagonal should only reflect genuine SNP-adjacent agreement, not
|
||||
/// the genome-wide invariant background, need it (see
|
||||
/// [`DistanceExt::base_pair_tally`]'s `same` field). Layers are processed
|
||||
/// one at a time, not in parallel — see `snp_pseudo_alignment`'s comment
|
||||
/// for why `par_iter()` over layers would defeat `scan_layer_families`'s
|
||||
/// partition-grouped locality; each layer gets its own accumulator from
|
||||
/// `zero()`, combined pairwise via `combine`. A free function, not a
|
||||
/// `KmerIndex` method — called only from this file's `DistanceExt` impl.
|
||||
fn scan_family_pairs<Acc, F, C>(
|
||||
index: &KmerIndex,
|
||||
label: &str,
|
||||
zero: impl Fn() -> Acc + Sync,
|
||||
on_pair: F,
|
||||
combine: C,
|
||||
) -> OKIResult<Acc>
|
||||
where
|
||||
Acc: Send,
|
||||
F: Fn(&mut Acc, usize, usize, u8, u8, bool) + Sync,
|
||||
C: Fn(Acc, Acc) -> Acc,
|
||||
{
|
||||
let n_parts = index.n_partitions();
|
||||
let n_genomes = index.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = index.meta().config.with_counts;
|
||||
let k = index.kmer_size();
|
||||
let cache = Arc::new(PartitionCache::build(index, n_parts, with_counts)?);
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(index)?;
|
||||
|
||||
let pb = progress_bar(label, layer_dirs.len() as u64, "layers");
|
||||
// One layer at a time — see `snp_pseudo_alignment`'s comment for why
|
||||
// `par_iter()` over layers would defeat `scan_layer_families`'s
|
||||
// partition-grouped locality.
|
||||
let mut total = zero();
|
||||
for layer_dir in &layer_dirs {
|
||||
let mut acc = zero();
|
||||
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&Selection::All,
|
||||
|_family_idx, mask, genome_mask| {
|
||||
let variable = mask.family_size() >= 2;
|
||||
|
||||
// Per genome: which single form (if exactly one) it
|
||||
// carries — `None` (a `popcount != 1` mask) once a
|
||||
// second form is seen, ambiguous/not single-copy,
|
||||
// ineligible for either side of a pair.
|
||||
let single_form = |g: usize| -> Option<u8> {
|
||||
let m = genome_mask[g];
|
||||
(m.count_ones() == 1).then(|| m.trailing_zeros() as u8)
|
||||
};
|
||||
|
||||
for i in 0..n_genomes {
|
||||
let Some(bi) = single_form(i) else { continue };
|
||||
for j in (i + 1)..n_genomes {
|
||||
let Some(bj) = single_form(j) else { continue };
|
||||
on_pair(&mut acc, i, j, bi, bj, variable);
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
|
||||
total = combine(total, acc);
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Adds [`raw_snp_distance`](Self::raw_snp_distance) and
|
||||
/// [`base_pair_tally`](Self::base_pair_tally) to `KmerIndex`.
|
||||
pub trait DistanceExt {
|
||||
/// Compute [`RawSnpDistanceOutput`] from an already-built sibling annex
|
||||
/// (run [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) first).
|
||||
fn raw_snp_distance(&self) -> OKIResult<RawSnpDistanceOutput>;
|
||||
|
||||
/// Symmetric 6-category base-pair substitution tally (AC, AG, AT, CG,
|
||||
/// CT, GT — indexed `0=A,1=C,2=G,3=T`), pooled only over genome pairs
|
||||
/// whose overall SNP ratio in `raw` is at or below `ratio_ceiling` —
|
||||
/// same saturation-exclusion discipline as
|
||||
/// [`cardinality_tally`](super::cardinality::CardinalityExt::cardinality_tally),
|
||||
/// for the same reason: a saturated pair's observed base-pair mix
|
||||
/// trends toward neutral base composition, not the true point-mutation
|
||||
/// spectrum.
|
||||
///
|
||||
/// A second full pass over the annex, sharing
|
||||
/// [`raw_snp_distance`](Self::raw_snp_distance)'s traversal (guided by
|
||||
/// it, not a blind re-scan) — needed because `raw_snp_distance` only
|
||||
/// keeps aggregate SNP/shared counts per genome pair, not which bases
|
||||
/// were actually involved at each locus, and the ratio-ceiling filter
|
||||
/// can only be evaluated once the aggregate counts are known.
|
||||
fn base_pair_tally(
|
||||
&self,
|
||||
raw: &RawSnpDistanceOutput,
|
||||
ratio_ceiling: f64,
|
||||
) -> OKIResult<BasePairTally>;
|
||||
}
|
||||
|
||||
impl DistanceExt for KmerIndex {
|
||||
fn raw_snp_distance(&self) -> OKIResult<RawSnpDistanceOutput> {
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let (snp, shared) = scan_family_pairs(
|
||||
self,
|
||||
"raw_snp_distance",
|
||||
|| {
|
||||
(
|
||||
Array2::<u64>::zeros((n_genomes, n_genomes)),
|
||||
Array2::<u64>::zeros((n_genomes, n_genomes)),
|
||||
)
|
||||
},
|
||||
|(snp, shared), i, j, bi, bj, _variable| {
|
||||
if bi == bj {
|
||||
shared[[i, j]] += 1;
|
||||
shared[[j, i]] += 1;
|
||||
} else {
|
||||
snp[[i, j]] += 1;
|
||||
snp[[j, i]] += 1;
|
||||
}
|
||||
},
|
||||
|(mut snp, mut shared), (s, sh)| {
|
||||
snp += &s;
|
||||
shared += &sh;
|
||||
(snp, shared)
|
||||
},
|
||||
)?;
|
||||
Ok(RawSnpDistanceOutput { snp, shared })
|
||||
}
|
||||
|
||||
fn base_pair_tally(
|
||||
&self,
|
||||
raw: &RawSnpDistanceOutput,
|
||||
ratio_ceiling: f64,
|
||||
) -> OKIResult<BasePairTally> {
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| {
|
||||
if i == j {
|
||||
return false;
|
||||
}
|
||||
let snp = raw.snp[[i, j]];
|
||||
let total = snp + raw.shared[[i, j]];
|
||||
total > 0 && (snp as f64 / total as f64) <= ratio_ceiling
|
||||
});
|
||||
|
||||
let (counts, same) = scan_family_pairs(
|
||||
self,
|
||||
"base_pair_tally",
|
||||
|| ([[0u64; 4]; 4], [0u64; 4]),
|
||||
|(counts, same), i, j, bi, bj, variable| {
|
||||
if !included[[i, j]] {
|
||||
return;
|
||||
}
|
||||
if bi != bj {
|
||||
counts[bi as usize][bj as usize] += 1;
|
||||
counts[bj as usize][bi as usize] += 1;
|
||||
} else if variable {
|
||||
// Only count "stayed the same" from families that vary
|
||||
// *somewhere* in the index — a fully invariant family
|
||||
// (never varies anywhere) isn't a SNP-adjacent
|
||||
// agreement, it's genome-wide background, and would
|
||||
// otherwise swamp the diagonal (see
|
||||
// `DevDocMD/theory/evolutionary_distances.md`, the
|
||||
// ascertainment-bias regression this was reverting).
|
||||
same[bi as usize] += 1;
|
||||
}
|
||||
},
|
||||
|(mut counts, mut same), (partial_counts, partial_same)| {
|
||||
for a in 0..4 {
|
||||
same[a] += partial_same[a];
|
||||
for b in 0..4 {
|
||||
counts[a][b] += partial_counts[a][b];
|
||||
}
|
||||
}
|
||||
(counts, same)
|
||||
},
|
||||
)?;
|
||||
Ok(BasePairTally { counts, same })
|
||||
}
|
||||
}
|
||||
|
||||
/// See [`DistanceExt::base_pair_tally`].
|
||||
pub struct BasePairTally {
|
||||
/// `counts[a][b] == counts[b][a]` = number of eligible loci, pooled
|
||||
/// over included genome pairs, where the two genomes' single forms are
|
||||
/// `a` and `b` (0=A, 1=C, 2=G, 3=T). Diagonal always `0` — an `a == b`
|
||||
/// locus is counted in `same`, not here.
|
||||
pub counts: [[u64; 4]; 4],
|
||||
/// `same[a]` = number of eligible loci, pooled over included genome
|
||||
/// pairs, where both genomes' single forms are `a` — the diagonal
|
||||
/// `counts` omits, needed to build a proper row-stochastic composition
|
||||
/// probability matrix (the "stay the same base" entries), not just the
|
||||
/// substitution-cost off-diagonal.
|
||||
pub same: [u64; 4],
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
//! `--shannon` — per-family Shannon entropy over the 15 non-empty subsets
|
||||
//! of `{A,C,G,T}`. See `DevDocMD/architecture/siblings.md`, "Entropy
|
||||
//! definition — settled 2026-08-15" and "`--subsample`/`--shannon`", for
|
||||
//! the design discussion this implements: the project's calibrated 16-state
|
||||
//! Sankoff cost matrix (`cardcomp::pairwise_cost_matrix`) treats every
|
||||
//! observed combination of bases as its own first-class state, so entropy
|
||||
//! over that same 15-symbol space (state `0`/`∅` excluded — a family is
|
||||
//! only informative among the genomes where it is actually observed) is the
|
||||
//! consistent informativeness proxy for this project, not a 4-symbol
|
||||
//! reduction.
|
||||
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::entropy_annex::{ENTROPY_ANNEX_FILE_NAME, EntropyAnnexBuilder};
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
use super::subsample::EntropyBias;
|
||||
|
||||
/// Shannon entropy (bits) of one family's per-genome states, over the 15
|
||||
/// non-empty subsets of `{A,C,G,T}` actually observed among the genomes
|
||||
/// carrying it (`genome_mask[g] != 0`) — genomes where the family is absent
|
||||
/// are excluded from the denominator, not scored as a 16th state. `None`
|
||||
/// only if the family is absent from every genome (shouldn't happen for a
|
||||
/// real minorant, guarded against rather than dividing by zero). Second
|
||||
/// value: the denominator (`n_genomes_present`) used, for traceability.
|
||||
pub(super) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> {
|
||||
let mut freq = [0u32; 16]; // index 0 (∅) never incremented, kept for direct `m as usize` indexing
|
||||
let mut present = 0u32;
|
||||
for &m in genome_mask {
|
||||
if m != 0 {
|
||||
freq[m as usize] += 1;
|
||||
present += 1;
|
||||
}
|
||||
}
|
||||
if present == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut h = 0.0f64;
|
||||
for &c in &freq[1..] {
|
||||
if c == 0 {
|
||||
continue;
|
||||
}
|
||||
let p = f64::from(c) / f64::from(present);
|
||||
h -= p * p.log2();
|
||||
}
|
||||
Some((h, present as usize))
|
||||
}
|
||||
|
||||
/// Shannon entropy (bits) of one family's per-genome states, reduced to the
|
||||
/// 4 plain nucleotide symbols instead of the 15-state space above — kept
|
||||
/// alongside it (not instead of it) purely to measure how much the two
|
||||
/// diverge on real data, per the discussion in
|
||||
/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`". A genome
|
||||
/// carrying several bases at once (`genome_mask[g]` with more than one bit
|
||||
/// set) contributes to *each* of those bases' counts — counted once per
|
||||
/// base present, not split fractionally, and not folded into a combined
|
||||
/// state the way [`family_entropy`] does: a genome polymorphic for the
|
||||
/// family is, by construction, present at more than one base, so it simply
|
||||
/// counts more than once here. The denominator is therefore the total
|
||||
/// number of base-occurrences summed over every present genome, not the
|
||||
/// genome count — the two coincide only when no genome carries more than
|
||||
/// one base. `None` under the same condition as [`family_entropy`].
|
||||
pub(super) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> {
|
||||
let mut freq = [0u32; 4];
|
||||
let mut total = 0u32;
|
||||
for &m in genome_mask {
|
||||
for (base, count) in freq.iter_mut().enumerate() {
|
||||
if m & (1 << base) != 0 {
|
||||
*count += 1;
|
||||
total += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut h = 0.0f64;
|
||||
for &c in &freq {
|
||||
if c == 0 {
|
||||
continue;
|
||||
}
|
||||
let p = f64::from(c) / f64::from(total);
|
||||
h -= p * p.log2();
|
||||
}
|
||||
Some((h, total as usize))
|
||||
}
|
||||
|
||||
/// Adds [`shannon_entropy_csv`](Self::shannon_entropy_csv) to `KmerIndex`.
|
||||
pub trait ShannonEntropyExt {
|
||||
/// Writes a CSV
|
||||
/// (`layer,family_idx,entropy15,entropy4,family_size,n_genomes_present`)
|
||||
/// of per-family Shannon entropy — both [`family_entropy`] (15 non-empty
|
||||
/// states, the project's settled definition) and [`family_entropy_4`]
|
||||
/// (plain 4-symbol reduction, kept alongside for comparison) — one row
|
||||
/// per non-monomorphic minorant visited, from an already-built sibling
|
||||
/// annex (run
|
||||
/// [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex)
|
||||
/// first). `layer` is this layer's ordinal among every layer carrying an
|
||||
/// annex (not a partition/layer pair — an opaque but stable id, unique
|
||||
/// together with `family_idx`); `family_idx` is the family's
|
||||
/// iteration-order index within that layer, the same numbering
|
||||
/// `--subsample`'s selection is drawn from.
|
||||
///
|
||||
/// `subsample`/`entropy_bias`: same meaning as
|
||||
/// [`SnpAlignmentExt::snp_pseudo_alignment`](super::SnpAlignmentExt::snp_pseudo_alignment)'s
|
||||
/// — `subsample = None`, `entropy_bias = None` streams every
|
||||
/// non-monomorphic minorant of the whole index (a time cost, not a
|
||||
/// memory one: entropy is computed and written per-family as soon as it
|
||||
/// resolves); `subsample = Some(n)` bounds the index's cross-partition
|
||||
/// resolution work to ~`n` sampled families; `entropy_bias = Some(_)`
|
||||
/// weights that sample by its own entropy (see
|
||||
/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection") —
|
||||
/// useful here specifically to see the biased sample's own entropy
|
||||
/// distribution, not just to speed up a distance computation.
|
||||
fn shannon_entropy_csv(
|
||||
&self,
|
||||
path: &Path,
|
||||
subsample: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<()>;
|
||||
}
|
||||
|
||||
impl ShannonEntropyExt for KmerIndex {
|
||||
fn shannon_entropy_csv(
|
||||
&self,
|
||||
path: &Path,
|
||||
subsample: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<()> {
|
||||
let n_parts = self.n_partitions();
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = self.meta().config.with_counts;
|
||||
let k = self.kmer_size();
|
||||
let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?);
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(self)?;
|
||||
let selections =
|
||||
super::subsample::compute_selections(self, &layer_dirs, subsample, entropy_bias)?;
|
||||
|
||||
let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?);
|
||||
writeln!(
|
||||
f,
|
||||
"layer,family_idx,entropy15,entropy4,family_size,n_genomes_present"
|
||||
)
|
||||
.map_err(OKIError::Io)?;
|
||||
|
||||
let pb = progress_bar("shannon_entropy", layer_dirs.len() as u64, "layers");
|
||||
for (layer_ord, (layer_dir, layer_selection)) in
|
||||
layer_dirs.iter().zip(selections.iter()).enumerate()
|
||||
{
|
||||
let selection = match layer_selection {
|
||||
None => Selection::All,
|
||||
Some(set) => Selection::Some(set),
|
||||
};
|
||||
let mut write_err = None;
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&selection,
|
||||
|family_idx, mask, genome_mask| {
|
||||
if write_err.is_some() {
|
||||
return;
|
||||
}
|
||||
if mask.family_size() < 2 {
|
||||
return; // monomorphic family — entropy trivially 0, no signal, skip
|
||||
}
|
||||
let Some((h15, n_present)) = family_entropy(genome_mask) else {
|
||||
return;
|
||||
};
|
||||
let h4 = family_entropy_4(genome_mask).map_or(0.0, |(h, _)| h);
|
||||
if let Err(e) = writeln!(
|
||||
f,
|
||||
"{layer_ord},{family_idx},{h15:.6},{h4:.6},{},{n_present}",
|
||||
mask.family_size()
|
||||
) {
|
||||
write_err = Some(e);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
if let Some(e) = write_err {
|
||||
return Err(OKIError::Io(e));
|
||||
}
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the entropy annex (`entropy_annex.rs`) for every layer in
|
||||
/// `layer_dirs` that doesn't already have one — a no-op, cheap existence
|
||||
/// check per layer, once every layer's annex file exists. Called from
|
||||
/// [`super::subsample::compute_selections`] whenever entropy-biased
|
||||
/// selection is requested; see `DevDocMD/architecture/siblings.md`,
|
||||
/// "Entropy-biased selection", for why this is the one place that pays a
|
||||
/// scan (every non-monomorphic minorant's `genome_mask` must be resolved
|
||||
/// to know its entropy) — every later read of the resulting file is a
|
||||
/// plain positional mmap lookup, not a re-scan.
|
||||
///
|
||||
/// Bounded to non-monomorphic minorants only
|
||||
/// ([`super::subsample::non_monomorphic_selection_layer`], a cheap
|
||||
/// annex-only pass — no cross-partition resolution), not
|
||||
/// `Selection::All`: monomorphism (`family_size() < 2`) is already known
|
||||
/// from the annex bits alone, so there is no reason to pay
|
||||
/// `scan_layer_families`'s expensive per-genome resolution for the ~98%
|
||||
/// of minorants that are monomorphic only to discard the result.
|
||||
pub(super) fn ensure_entropy_annexes(index: &KmerIndex, layer_dirs: &[PathBuf]) -> OKIResult<()> {
|
||||
let missing: Vec<usize> = layer_dirs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, dir)| !dir.join(ENTROPY_ANNEX_FILE_NAME).exists())
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
if missing.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let n_parts = index.n_partitions();
|
||||
let n_genomes = index.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = index.meta().config.with_counts;
|
||||
let k = index.kmer_size();
|
||||
let cache = Arc::new(PartitionCache::build(index, n_parts, with_counts)?);
|
||||
let minorant_counts = super::subsample::minorant_counts(layer_dirs)?;
|
||||
|
||||
let pb = progress_bar("entropy_annex_build", missing.len() as u64, "layers");
|
||||
for &li in &missing {
|
||||
let layer_dir = &layer_dirs[li];
|
||||
let mut builder = EntropyAnnexBuilder::new(
|
||||
minorant_counts[li] as usize,
|
||||
&layer_dir.join(ENTROPY_ANNEX_FILE_NAME),
|
||||
)
|
||||
.map_err(OKIError::Io)?;
|
||||
let eligible = super::subsample::non_monomorphic_selection_layer(layer_dir)?;
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&Selection::Some(&eligible),
|
||||
|family_idx, mask, genome_mask| {
|
||||
debug_assert!(
|
||||
mask.family_size() >= 2,
|
||||
"Selection::Some(eligible) must only visit non-monomorphic minorants"
|
||||
);
|
||||
if let Some((h15, _)) = family_entropy(genome_mask) {
|
||||
builder.set(family_idx, h15 as f32);
|
||||
}
|
||||
},
|
||||
)?;
|
||||
builder.close().map_err(OKIError::Io)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
//! Persisted, per-layer Shannon entropy (`entropy15`, see `entropy.rs`),
|
||||
//! one value per **minorant family**, indexed by `family_idx` — the same
|
||||
//! numbering `scan_layer_families`/`Selection`/`subsample::reservoir_sample_layer`
|
||||
//! already use (every minorant of the layer, monomorphic ones included,
|
||||
//! counted in iteration order; see `subsample.rs`'s module docs for why
|
||||
//! `family_idx` and the raw annex slot are kept as two different counters).
|
||||
//!
|
||||
//! Exists so entropy-biased selection (`--entropy`/`--entropy-sd`) can
|
||||
//! weight candidates without re-resolving each one's `genome_mask` (the
|
||||
//! expensive cross-partition step) on every run — see
|
||||
//! `DevDocMD/architecture/siblings.md`, "Entropy-biased selection": entropy is
|
||||
//! only knowable once presence is resolved, so *building* this file still
|
||||
//! costs a full, unsampled scan the first time (`ensure_entropy_annexes` in
|
||||
//! `entropy.rs`), but every subsequent read is a plain positional mmap
|
||||
//! lookup, restoring `Selection::Some`'s usual "skip resolving excluded
|
||||
//! families entirely" speedup for every run after the first.
|
||||
//!
|
||||
//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat
|
||||
//! per-entry data, read-only after build) but is not slot-indexed the way
|
||||
//! `SiblingAnnex` is: `SiblingAnnex` is indexed by every k-mer of the layer
|
||||
//! (needed because any slot might turn out to be a minorant), while this
|
||||
//! file is built *from* an already-open `SiblingAnnex`, so it only ever
|
||||
//! needs to be as long as the layer's minorant count — monomorphic
|
||||
//! minorants still get an entry (a `-1.0` sentinel, entropy not meaningful
|
||||
//! for them, never read), keeping the same family_idx numbering every
|
||||
//! other consumer uses, but non-minorant k-mers (the vast majority of
|
||||
//! slots) are never represented here at all.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PENT";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// Sentinel: monomorphic minorant (no signal, never sampled) or otherwise
|
||||
/// not applicable. Entropy is always `>= 0.0`, so any negative value is
|
||||
/// unambiguous — chosen over `NaN` to avoid `NaN`-comparison footguns at
|
||||
/// every read site.
|
||||
const SENTINEL: f32 = -1.0;
|
||||
|
||||
pub(super) const ENTROPY_ANNEX_FILE_NAME: &str = "entropy.pent";
|
||||
|
||||
pub(super) struct EntropyAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
}
|
||||
|
||||
impl EntropyAnnex {
|
||||
pub(super) fn open(path: &Path) -> io::Result<Self> {
|
||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||
if mmap.len() < HEADER_SIZE {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PENT file too short"));
|
||||
}
|
||||
if mmap[0..4] != MAGIC {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PENT magic"));
|
||||
}
|
||||
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
|
||||
if mmap.len() < HEADER_SIZE + n * 4 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PENT file truncated"));
|
||||
}
|
||||
Ok(Self { mmap, n })
|
||||
}
|
||||
|
||||
pub(super) fn len(&self) -> usize { self.n }
|
||||
|
||||
/// `None` for the sentinel (monomorphic minorant, or not yet computed).
|
||||
pub(super) fn get(&self, family_idx: usize) -> Option<f32> {
|
||||
let off = HEADER_SIZE + family_idx * 4;
|
||||
let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap());
|
||||
(v >= 0.0).then_some(v)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct EntropyAnnexBuilder {
|
||||
mmap: MmapMut,
|
||||
}
|
||||
|
||||
impl EntropyAnnexBuilder {
|
||||
/// Create a new annex of `n` entries (a layer's total minorant count) at
|
||||
/// `path`, pre-initialised to the sentinel — entries never written
|
||||
/// (monomorphic minorants) stay the sentinel, not an accidental `0.0`
|
||||
/// (a real, achievable entropy value).
|
||||
pub(super) fn new(n: usize, path: &Path) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n * 4;
|
||||
let file = OpenOptions::new()
|
||||
.read(true).write(true).create(true).truncate(true)
|
||||
.open(path)?;
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&MAGIC);
|
||||
mmap[4..8].copy_from_slice(&[0u8; 4]);
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
for i in 0..n {
|
||||
let off = HEADER_SIZE + i * 4;
|
||||
mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes());
|
||||
}
|
||||
Ok(Self { mmap })
|
||||
}
|
||||
|
||||
pub(super) fn set(&mut self, family_idx: usize, value: f32) {
|
||||
debug_assert!(value >= 0.0, "entropy must be non-negative, got {value}");
|
||||
let off = HEADER_SIZE + family_idx * 4;
|
||||
self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes());
|
||||
}
|
||||
|
||||
pub(super) fn close(self) -> io::Result<()> {
|
||||
self.mmap.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn sentinel_is_negative_and_unset_entries_read_as_none() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let builder = EntropyAnnexBuilder::new(4, &path).unwrap();
|
||||
builder.close().unwrap();
|
||||
let annex = EntropyAnnex::open(&path).unwrap();
|
||||
for i in 0..4 {
|
||||
assert_eq!(annex.get(i), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_including_zero_entropy() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = EntropyAnnexBuilder::new(4, &path).unwrap();
|
||||
let values = [0.0f32, 1.5, 3.9, 0.0001];
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
builder.set(i, v);
|
||||
}
|
||||
builder.close().unwrap();
|
||||
let annex = EntropyAnnex::open(&path).unwrap();
|
||||
for (i, &v) in values.iter().enumerate() {
|
||||
assert_eq!(annex.get(i), Some(v), "entry {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_sentinel_and_real_values() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.pent");
|
||||
let mut builder = EntropyAnnexBuilder::new(3, &path).unwrap();
|
||||
builder.set(1, 2.0);
|
||||
builder.close().unwrap();
|
||||
let annex = EntropyAnnex::open(&path).unwrap();
|
||||
assert_eq!(annex.get(0), None);
|
||||
assert_eq!(annex.get(1), Some(2.0));
|
||||
assert_eq!(annex.get(2), None);
|
||||
assert_eq!(annex.len(), 3);
|
||||
}
|
||||
}
|
||||
@@ -1,373 +0,0 @@
|
||||
//! Shared per-layer family traversal, used by every sibling-annex consumer
|
||||
//! (`snp_pseudo_alignment`, `cardinality_tally`, `scan_family_pairs`,
|
||||
//! `sibling_annex_stats`) — resolves each minorant family's per-genome
|
||||
//! base-presence, with the same locality discipline as
|
||||
//! `build_sibling_annex` (see `cache.rs`/`build.rs`'s docs): cross-partition
|
||||
//! lookups are grouped by destination partition and resolved in one
|
||||
//! contiguous sweep per partition, instead of one lookup jumping between
|
||||
//! partitions in family order.
|
||||
//!
|
||||
//! Two concerns, kept on two different mechanisms because they need
|
||||
//! opposite things:
|
||||
//!
|
||||
//! - **Generating** each batch's cross-partition queries is pure CPU (bit
|
||||
//! tests, one hash per variant) — cheap, and safe to run for several
|
||||
//! batches at once. It runs on an `obipipeline::throttle` + `make_pipe!`
|
||||
//! pipeline, the same mechanism `build_layer_sibling_annex` uses, for the
|
||||
//! same reason: it overlaps this CPU work with the *previous* batch's
|
||||
//! resolution instead of leaving cores idle between batches.
|
||||
//! - **Resolving** those queries against `PartitionCache` is I/O (mmap page
|
||||
//! faults on a real index far larger than RAM) — one batch at a time,
|
||||
//! `rayon`-parallel *across partitions* (like `build_sibling_annex`'s own
|
||||
//! `outgoing.par_iter()`), never several batches concurrently. An earlier
|
||||
//! version resolved each batch fully inside its own pipeline worker, so
|
||||
//! `n_workers` batches were resolved concurrently — each one spreading its
|
||||
//! queries thin across every partition of the layer at once. Sampling a
|
||||
//! real run showed the fix hadn't helped at all: 16 threads "busy" in
|
||||
//! `find_presence`, but mostly blocked on page faults (~1.7 GB/s pagein),
|
||||
//! because 16 concurrent sweeps across the same partition space is exactly
|
||||
//! the scattering the grouping was meant to prevent — just spread over
|
||||
//! threads instead of over layers this time. Resolving one batch's worth
|
||||
//! at a time, with each of `rayon`'s threads owning one partition
|
||||
//! contiguously until that batch is done, keeps only one partition set
|
||||
//! "hot" at once, restoring the locality the batching is for.
|
||||
//!
|
||||
//! `obipipeline` does not guarantee output order, so generated batches carry
|
||||
//! their own starting family index and are replayed through a small reorder
|
||||
//! buffer before resolution — bounded by the throttle's own concurrency
|
||||
//! (`n_workers` batches), not by the layer's size.
|
||||
//!
|
||||
//! [`FAMILY_BATCH`] bounds a single batch's memory to `FAMILY_BATCH *
|
||||
//! genome_count` bytes, and is chosen large enough that a batch still gives
|
||||
//! each partition a decent number of queries to resolve in one go — too
|
||||
//! small a batch starves that density regardless of how the resolution step
|
||||
//! is parallelised. An earlier version materialised the whole layer's
|
||||
//! families up front before resolving anything: O(layer's family count ×
|
||||
//! genome count) memory, measured pushing a real run into heavy VM
|
||||
//! compression/swap.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikindex::layer::meta::PartitionMeta;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obipipeline::{ThrottleGuard, throttle};
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use obikindex::layer::KmerLayer;
|
||||
|
||||
use super::cache::PartitionCache;
|
||||
use super::helpers::central_base;
|
||||
use super::iter::{SiblingEntry, SiblingLayerExt};
|
||||
use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex};
|
||||
|
||||
/// Families per batch — see the module docs for the memory-vs-per-partition-
|
||||
/// density trade-off this picks a point on. At ~90 genomes and a few
|
||||
/// hundred partitions, this keeps a batch's resolution memory in the low
|
||||
/// tens of MB while still giving each partition on the order of a thousand
|
||||
/// queries per batch to amortise against.
|
||||
const FAMILY_BATCH: usize = 65536;
|
||||
|
||||
/// Restricts [`scan_layer_families`] to a subset of a layer's families,
|
||||
/// identified by their iteration-order index — see
|
||||
/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`", for why:
|
||||
/// bounding which families actually pay the expensive cross-partition
|
||||
/// resolution cost is what makes a sampled run cheap on an index far larger
|
||||
/// than the sample itself. `All` (the default for every pre-existing
|
||||
/// caller) keeps today's behaviour exactly.
|
||||
pub(super) enum Selection<'a> {
|
||||
All,
|
||||
Some(&'a std::collections::HashSet<usize>),
|
||||
}
|
||||
|
||||
/// Shared by the pipeline worker (via the `Arc`-cloned `Option<HashSet>`,
|
||||
/// see below — a `Selection` itself doesn't cross the worker thread
|
||||
/// boundary) and the final replay loop, so both sides agree on membership
|
||||
/// without duplicating the match.
|
||||
fn is_selected(selected: &Option<std::collections::HashSet<usize>>, family_idx: usize) -> bool {
|
||||
match selected {
|
||||
None => true,
|
||||
Some(set) => set.contains(&family_idx),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every (partition, layer) directory carrying a sibling annex, checked
|
||||
/// up front so a missing one is reported before any real work starts.
|
||||
/// Shared by every sibling-annex consumer's extension-trait impl (`alignment`,
|
||||
/// `build`, `cardinality`, `distance`, `stats`) — a free function, not a
|
||||
/// `KmerIndex` method, since it is crate-internal only and `KmerIndex` lives
|
||||
/// in `obikindex`, a foreign crate from here (orphan rule).
|
||||
pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
|
||||
let n_parts = index.n_partitions();
|
||||
let mut layer_dirs = Vec::new();
|
||||
for part in 0..n_parts {
|
||||
let index_dir = index.index_dir(part);
|
||||
if !index_dir.exists() {
|
||||
continue;
|
||||
}
|
||||
let n_layers = index.n_layers(part)?;
|
||||
for l in 0..n_layers {
|
||||
let this_layer_dir = index.layer_dir(part, l);
|
||||
let annex_path = this_layer_dir.join(ANNEX_FILE_NAME);
|
||||
if !annex_path.exists() {
|
||||
return Err(OKIError::InvalidInput(format!(
|
||||
"no sibling annex at {} — run build_sibling_annex first",
|
||||
annex_path.display()
|
||||
)));
|
||||
}
|
||||
layer_dirs.push(this_layer_dir);
|
||||
}
|
||||
}
|
||||
Ok(layer_dirs)
|
||||
}
|
||||
|
||||
/// Read-only state shared (via `Arc`) across every pipeline worker
|
||||
/// generating this layer's batches — opened once, not per batch. No `cache`
|
||||
/// here: generation never touches the cross-partition cache, only this
|
||||
/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either —
|
||||
/// `mat` (a `TypedLayer<D>`) already bundles the MPHF, and each `SiblingEntry`
|
||||
/// arrives with its kmer and mask already in hand from `iter_minorants_batch`.
|
||||
struct LayerCtx {
|
||||
mat: KmerLayer,
|
||||
n_parts: usize,
|
||||
n_genomes: usize,
|
||||
n_cols: usize,
|
||||
k: usize,
|
||||
}
|
||||
|
||||
struct SourceBatch {
|
||||
start_family_idx: usize,
|
||||
/// One entry per minorant family in this batch, straight from
|
||||
/// `iter_minorants_batch` — `order` (iteration-order index, not an MPHF
|
||||
/// slot; see `DevDocMD/architecture/siblings.md`), `kmer`, and `mask`
|
||||
/// already carried through, no second annex read.
|
||||
entries: Vec<SiblingEntry>,
|
||||
_permit: ThrottleGuard,
|
||||
}
|
||||
|
||||
/// One batch's generated work: local presence already resolved (straight
|
||||
/// from this layer's own matrix, no lookup needed), cross-partition queries
|
||||
/// collected but not yet resolved against the cache.
|
||||
struct GeneratedBatch {
|
||||
start_family_idx: usize,
|
||||
masks: Vec<FamilyMask>,
|
||||
/// Flat `slots.len() * n_genomes` — `genome_mask[i * n_genomes + g]`.
|
||||
genome_mask: Vec<u8>,
|
||||
/// `outgoing[dest_partition]` = `(variant, family_idx_in_batch, base,
|
||||
/// layer)` — `layer` is the annex-recorded destination layer
|
||||
/// (`FamilyMask::family_members`'s `layer_value`, `0` if absent),
|
||||
/// trustworthy only when `PartitionCache::fast_mode()` is true.
|
||||
outgoing: Vec<Vec<(CanonicalKmer, usize, u8, u8)>>,
|
||||
_permit: ThrottleGuard,
|
||||
}
|
||||
|
||||
enum FamData {
|
||||
Batch(SourceBatch),
|
||||
Generated(GeneratedBatch),
|
||||
}
|
||||
|
||||
/// Visits every minorant family of one layer, in iteration order, batched
|
||||
/// [`FAMILY_BATCH`] at a time — see the module docs for why generation and
|
||||
/// resolution use different concurrency. `on_family` is called once per
|
||||
/// family, in iteration order, with that family's iteration-order index
|
||||
/// (the same numbering [`Selection`] indices are drawn from), its own annex
|
||||
/// mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set
|
||||
/// iff genome `g` carries the member whose own canonical central base is
|
||||
/// `b`), backed by a scratch buffer reused across every call — callers that
|
||||
/// need to keep data past the call must copy it themselves.
|
||||
pub(super) fn scan_layer_families(
|
||||
layer_dir: &Path,
|
||||
n_parts: usize,
|
||||
n_genomes: usize,
|
||||
with_counts: bool,
|
||||
k: usize,
|
||||
cache: &Arc<PartitionCache>,
|
||||
selection: &Selection,
|
||||
mut on_family: impl FnMut(usize, FamilyMask, &[u8]),
|
||||
) -> OKIResult<()> {
|
||||
let index_dir = layer_dir
|
||||
.parent()
|
||||
.expect("layer_dir has a parent index dir");
|
||||
let meta = PartitionMeta::load(index_dir)?;
|
||||
let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?);
|
||||
|
||||
let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts)?;
|
||||
let n_cols = mat.n_cols().min(n_genomes);
|
||||
|
||||
let ctx = Arc::new(LayerCtx {
|
||||
mat,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
n_cols,
|
||||
k,
|
||||
});
|
||||
|
||||
// Streamed straight from `iter_minorants_batch` (zips this layer's own
|
||||
// `iter_kmers()` with the annex, both in iteration order — never an
|
||||
// MPHF slot; see `DevDocMD/architecture/siblings.md`) — never collected
|
||||
// into a `Vec` first: a layer can hold billions of k-mers, so
|
||||
// materialising every minorant family up front is exactly the memory
|
||||
// blowup an earlier version of this traversal was rewritten to avoid
|
||||
// (see the module docs). `.scan()` computes each batch's starting
|
||||
// family index lazily, mirroring what the eager `chunks()`+running
|
||||
// `offset` used to do.
|
||||
let batches =
|
||||
ctx.mat
|
||||
.iter_minorants_batch(annex, FAMILY_BATCH)
|
||||
.scan(0usize, |offset, batch| {
|
||||
let start = *offset;
|
||||
*offset += batch.len();
|
||||
Some((start, batch))
|
||||
});
|
||||
|
||||
let n_workers = obisys::effective_parallelism();
|
||||
let capacity = 4;
|
||||
let throttled = throttle(batches, n_workers).map(|t| SourceBatch {
|
||||
start_family_idx: t.item.0,
|
||||
entries: t.item.1,
|
||||
_permit: t.guard,
|
||||
});
|
||||
|
||||
// `Selection::Some` borrows a `HashSet` whose lifetime doesn't span the
|
||||
// pipeline's worker threads — cloned once into an `Arc` so every worker
|
||||
// can share it cheaply instead of requiring `selection` itself to be
|
||||
// `'static`. `None` means [`Selection::All`], checked with a plain
|
||||
// `map_or` at each use instead of allocating an always-true set.
|
||||
let selected: Arc<Option<std::collections::HashSet<usize>>> = Arc::new(match selection {
|
||||
Selection::All => None,
|
||||
Selection::Some(set) => Some((*set).clone()),
|
||||
});
|
||||
|
||||
let worker_ctx = Arc::clone(&ctx);
|
||||
let worker_selected = Arc::clone(&selected);
|
||||
let pipe = obipipeline::make_pipe! {
|
||||
FamData : SourceBatch => GeneratedBatch,
|
||||
| {
|
||||
move |batch: SourceBatch| -> GeneratedBatch {
|
||||
let ctx = &worker_ctx;
|
||||
let n = batch.entries.len();
|
||||
let mut masks = Vec::with_capacity(n);
|
||||
let mut bases = Vec::with_capacity(n);
|
||||
let mut genome_mask = vec![0u8; n * ctx.n_genomes];
|
||||
let mut outgoing: Vec<Vec<(CanonicalKmer, usize, u8, u8)>> = (0..ctx.n_parts).map(|_| Vec::new()).collect();
|
||||
|
||||
// Pass 1: cheap, no matrix access — own base and this
|
||||
// batch's cross-partition queries. Kmer and mask already in
|
||||
// hand from `iter_minorants_batch`, no second annex read,
|
||||
// no slot lookup needed for this pass. `family_members`
|
||||
// (not a hand-rolled `central_canonical_neighbors` +
|
||||
// `mask.has` loop) already filters to present members and
|
||||
// hands back each one's annex-recorded layer alongside.
|
||||
// Families outside `selected` (sampling only) still get a
|
||||
// mask/base entry — pass 2 indexes uniformly by `i` — but
|
||||
// never an outgoing query: that's the expensive part a
|
||||
// sampled run exists to avoid paying for every family.
|
||||
for (i, entry) in batch.entries.iter().enumerate() {
|
||||
let (kmer, mask) = (entry.kmer, entry.mask);
|
||||
masks.push(mask);
|
||||
let base = central_base(kmer, ctx.k);
|
||||
bases.push(base);
|
||||
let family_idx = batch.start_family_idx + i;
|
||||
if !is_selected(&worker_selected, family_idx) {
|
||||
continue;
|
||||
}
|
||||
for (member, layer) in mask.family_members(kmer, ctx.k) {
|
||||
if member == kmer {
|
||||
continue; // local — resolved below straight from `mat`, no lookup
|
||||
}
|
||||
let b = central_base(member, ctx.k);
|
||||
let dest = member.partition(ctx.n_parts);
|
||||
outgoing[dest].push((member, i, b, layer.unwrap_or(0)));
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: genome-major, not family-major — `mat` is stored
|
||||
// one contiguous block per genome (column), slot as the
|
||||
// offset within it (see `PartitionCache::find_presence_batch`'s
|
||||
// docs for the full rationale). `fill_sub_matrix_carries`
|
||||
// sorts the slots internally for a sequential mmap sweep per
|
||||
// column, then restores this batch's order — no hand-rolled
|
||||
// sort/genome-major loop needed here. The presence/count
|
||||
// matrix is still MPHF-slot-indexed (unlike the annex), so
|
||||
// each entry's kmer is mapped to its slot via `index_batch`
|
||||
// — a pure MPHF lookup, no evidence check, since these are
|
||||
// this layer's own kmers, known members by construction.
|
||||
let kmers: Vec<CanonicalKmer> = batch.entries.iter().map(|e| e.kmer).collect();
|
||||
let slots = ctx.mat.hash_batch(&kmers);
|
||||
let mut carries: Vec<Vec<bool>> = (0..ctx.n_cols).map(|_| Vec::new()).collect();
|
||||
ctx.mat.fill_sub_matrix_carries(&slots, &mut carries);
|
||||
for (g, col) in carries.iter().enumerate() {
|
||||
for (i, &carries_it) in col.iter().enumerate() {
|
||||
if carries_it {
|
||||
genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GeneratedBatch { start_family_idx: batch.start_family_idx, masks, genome_mask, outgoing, _permit: batch._permit }
|
||||
}
|
||||
} : Batch => Generated,
|
||||
};
|
||||
|
||||
// Batches finish generation in whatever order their worker completes
|
||||
// them, not submission order — replay in order through a buffer bounded
|
||||
// by the throttle's own concurrency (`n_workers` batches can be in
|
||||
// flight at once, so at most that many can be waiting here).
|
||||
let mut pending: HashMap<usize, GeneratedBatch> = HashMap::new();
|
||||
let mut next_expected = 0usize;
|
||||
let mut scratch = vec![0u8; n_genomes];
|
||||
for generated in pipe.apply(throttled, n_workers, capacity) {
|
||||
pending.insert(generated.start_family_idx, generated);
|
||||
while let Some(batch) = pending.remove(&next_expected) {
|
||||
let n = batch.masks.len();
|
||||
|
||||
// ── Resolve this one batch's cross-partition queries — the
|
||||
// only place that touches `cache`. Parallel across partitions
|
||||
// (one batch at a time, never several concurrently — see the
|
||||
// module docs), each thread owning one partition's queries
|
||||
// contiguously until this batch is done.
|
||||
let genome_mask: Vec<AtomicU8> =
|
||||
batch.genome_mask.into_iter().map(AtomicU8::new).collect();
|
||||
let fast_mode = cache.fast_mode();
|
||||
batch
|
||||
.outgoing
|
||||
.par_iter()
|
||||
.enumerate()
|
||||
.filter(|(_, q)| !q.is_empty())
|
||||
.for_each(|(dest, queries)| {
|
||||
let on_hit = |i: usize, base: u8, g: usize| {
|
||||
genome_mask[i * n_genomes + g].fetch_or(1 << base, Ordering::Relaxed);
|
||||
};
|
||||
if fast_mode {
|
||||
cache.find_presence_batch_fast(dest, queries, n_genomes, on_hit);
|
||||
} else {
|
||||
cache.find_presence_batch(dest, queries, n_genomes, on_hit);
|
||||
}
|
||||
});
|
||||
|
||||
for i in 0..n {
|
||||
let family_idx = batch.start_family_idx + i;
|
||||
if !is_selected(&selected, family_idx) {
|
||||
continue; // not in the sample — never resolved above, nothing to report
|
||||
}
|
||||
for (g, dst) in scratch.iter_mut().enumerate() {
|
||||
*dst = genome_mask[i * n_genomes + g].load(Ordering::Relaxed);
|
||||
}
|
||||
on_family(family_idx, batch.masks[i], &scratch);
|
||||
}
|
||||
next_expected += n;
|
||||
}
|
||||
}
|
||||
debug_assert!(
|
||||
pending.is_empty(),
|
||||
"every generated batch must have been replayed"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use super::FamilyMask;
|
||||
|
||||
/// Central-position base of a canonical k-mer, in the fixed 0=A/1=C/2=G/3=T
|
||||
/// encoding — the mask's bit index. `k` must be odd (project invariant).
|
||||
#[inline]
|
||||
pub(super) fn central_base(kmer: CanonicalKmer, k: usize) -> u8 {
|
||||
kmer.nucleotide((k - 1) / 2)
|
||||
}
|
||||
|
||||
/// Is `kmer` the minorant of its family, given the family's presence mask?
|
||||
/// Regenerates the family's 4 canonical forms from `kmer` itself (cheap, no
|
||||
/// lookup — see the design doc's "Definitions" section for why this is
|
||||
/// always safe: the set of 4 forms is invariant regardless of which member
|
||||
/// you start from), and compares the raw encodings of whichever are marked
|
||||
/// present in `mask`.
|
||||
pub(super) fn is_minorant(kmer: CanonicalKmer, mask: FamilyMask, k: usize) -> bool {
|
||||
kmer.central_canonical_neighbors().into_iter().all(|other| {
|
||||
other == kmer || !mask.has(central_base(other, k)) || kmer.raw() <= other.raw()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
//! Phylo/sibling-domain iteration over a layer — an extension trait, not a
|
||||
//! new field on `MphfLayer`/`TypedLayer<D>`: "family"/"minorant" are phylo
|
||||
//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see
|
||||
//! `DevDocMD/architecture/siblings.md`).
|
||||
//!
|
||||
//! The sibling annex is persisted in the same order as `iter_kmers()`
|
||||
//! (`build_sibling_annex`, see `build.rs`), so pairing them is a plain zip —
|
||||
//! no MPHF, no slot, no `kmer_at`. Both sides are already `Send + 'static`
|
||||
//! (`KmerIter` owns an `Arc<UnitigFileReader>` clone; `SiblingAnnex` is
|
||||
//! mmap-backed and handed in as an `Arc` by the caller), so `SiblingIter`
|
||||
//! streams straight from disk and can be fed to `obipipeline` batch by
|
||||
//! batch — never collected whole into memory (see the project's "no full
|
||||
//! collect" rule).
|
||||
//!
|
||||
//! Four iterator types, deliberately mirroring `obikindex::layer`'s own
|
||||
//! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the
|
||||
//! minorant-filtered variant of each, since "all siblings" and "one row per
|
||||
//! family" are both common cases:
|
||||
//!
|
||||
//! | | all entries | minorants only |
|
||||
//! |------------|-------------------|---------------------|
|
||||
//! | single | [`SiblingIter`] | [`MinorantIter`] |
|
||||
//! | batch | [`SiblingBatchIter`] | [`MinorantBatchIter`] |
|
||||
//!
|
||||
//! No separate "enumerate" variant (unlike `KmerIter`/`enumerate_kmers`):
|
||||
//! [`SiblingEntry`] already carries `order` for free, since pairing with
|
||||
//! the annex requires it anyway.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use obikindex::layer::{KmerIter, LayerData, TypedLayer};
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use super::{FamilyMask, SiblingAnnex};
|
||||
|
||||
/// One layer entry: a k-mer's position in the layer's iteration order (the
|
||||
/// same index the sibling annex is keyed on — not an MPHF slot), the k-mer
|
||||
/// itself, and its family mask.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct SiblingEntry {
|
||||
pub order: usize,
|
||||
pub kmer: CanonicalKmer,
|
||||
pub mask: FamilyMask,
|
||||
}
|
||||
|
||||
/// Streams `(order, kmer, mask)` triples for one layer, in iteration order.
|
||||
/// Produced by [`SiblingLayerExt::iter_siblings`].
|
||||
pub struct SiblingIter {
|
||||
kmers: KmerIter,
|
||||
annex: Arc<SiblingAnnex>,
|
||||
order: usize,
|
||||
}
|
||||
|
||||
impl Iterator for SiblingIter {
|
||||
type Item = SiblingEntry;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
loop {
|
||||
let kmer = self.kmers.next()?;
|
||||
let order = self.order;
|
||||
self.order += 1;
|
||||
// `None` means "not yet computed" (see `SiblingAnnex` module
|
||||
// docs) — shouldn't happen against a fully-built annex, but
|
||||
// skip rather than misalign the two streams if it does.
|
||||
if let Some(mask) = self.annex.get(order) {
|
||||
return Some(SiblingEntry { order, kmer, mask });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Batches of [`SiblingIter`]'s entries, `batch_size` at a time — the last
|
||||
/// batch may be shorter. Produced by [`SiblingLayerExt::iter_siblings_batch`].
|
||||
pub struct SiblingBatchIter {
|
||||
inner: SiblingIter,
|
||||
batch_size: usize,
|
||||
}
|
||||
|
||||
impl Iterator for SiblingBatchIter {
|
||||
type Item = Vec<SiblingEntry>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
collect_batch(&mut self.inner, self.batch_size)
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`SiblingIter`], filtered to the minorant of each family — the
|
||||
/// common case, since a family is tallied once, at its minorant. Produced
|
||||
/// by [`SiblingLayerExt::iter_minorants`].
|
||||
pub struct MinorantIter {
|
||||
inner: SiblingIter,
|
||||
}
|
||||
|
||||
impl Iterator for MinorantIter {
|
||||
type Item = SiblingEntry;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
self.inner.by_ref().find(|e| e.mask.is_minorant())
|
||||
}
|
||||
}
|
||||
|
||||
/// Batches of [`MinorantIter`]'s entries, `batch_size` at a time — the last
|
||||
/// batch may be shorter. Produced by
|
||||
/// [`SiblingLayerExt::iter_minorants_batch`].
|
||||
pub struct MinorantBatchIter {
|
||||
inner: MinorantIter,
|
||||
batch_size: usize,
|
||||
}
|
||||
|
||||
impl Iterator for MinorantBatchIter {
|
||||
type Item = Vec<SiblingEntry>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
collect_batch(&mut self.inner, self.batch_size)
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared by [`SiblingBatchIter`] and [`MinorantBatchIter`] — pull up to
|
||||
/// `batch_size` items, `None` once the source is exhausted with nothing left.
|
||||
fn collect_batch<I: Iterator>(inner: &mut I, batch_size: usize) -> Option<Vec<I::Item>> {
|
||||
let mut batch = Vec::with_capacity(batch_size);
|
||||
for _ in 0..batch_size {
|
||||
match inner.next() {
|
||||
Some(item) => batch.push(item),
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
if batch.is_empty() { None } else { Some(batch) }
|
||||
}
|
||||
|
||||
/// Adds phylo/sibling iteration to any `TypedLayer<D>` — the extension that
|
||||
/// turns a plain layer into a "sibling layer". Generic over `D`
|
||||
/// (`LayerData`) rather than implemented once per matrix kind: kmer
|
||||
/// iteration doesn't depend on the data payload, and `TypedLayer<D>` already
|
||||
/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every
|
||||
/// `D` — reusing that instead of going back through a separate `MphfLayer`.
|
||||
pub trait SiblingLayerExt {
|
||||
/// Zip this layer's k-mers with their sibling-annex entry, in iteration
|
||||
/// order. `annex` must have been built from this same layer (its length
|
||||
/// must match the layer's k-mer count).
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter;
|
||||
|
||||
/// Like [`iter_siblings`](Self::iter_siblings), yielding `batch_size`
|
||||
/// entries at a time.
|
||||
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter;
|
||||
|
||||
/// Like [`iter_siblings`](Self::iter_siblings), filtered to the
|
||||
/// minorant of each family — the common case, since a family is
|
||||
/// tallied once, at its minorant.
|
||||
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter;
|
||||
|
||||
/// Like [`iter_minorants`](Self::iter_minorants), yielding `batch_size`
|
||||
/// minorants at a time.
|
||||
fn iter_minorants_batch(
|
||||
&self,
|
||||
annex: Arc<SiblingAnnex>,
|
||||
batch_size: usize,
|
||||
) -> MinorantBatchIter;
|
||||
}
|
||||
|
||||
impl<D: LayerData> SiblingLayerExt for TypedLayer<D> {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
SiblingIter {
|
||||
kmers: self.iter_kmers(),
|
||||
annex,
|
||||
order: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
|
||||
SiblingBatchIter {
|
||||
inner: self.iter_siblings(annex),
|
||||
batch_size,
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
|
||||
MinorantIter {
|
||||
inner: self.iter_siblings(annex),
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants_batch(
|
||||
&self,
|
||||
annex: Arc<SiblingAnnex>,
|
||||
batch_size: usize,
|
||||
) -> MinorantBatchIter {
|
||||
MinorantBatchIter {
|
||||
inner: self.iter_minorants(annex),
|
||||
batch_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Same extension, over `obikindex::layer::Layer` (the format-erased
|
||||
/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch
|
||||
/// only, both arms return the same concrete iterator types (they don't
|
||||
/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer`
|
||||
/// itself can't implement this: "family"/"minorant" are phylo concepts, it
|
||||
/// stays kmer/slot-mapping only (see the module docs above).
|
||||
impl SiblingLayerExt for obikindex::layer::KmerLayer {
|
||||
fn iter_siblings(&self, annex: Arc<SiblingAnnex>) -> SiblingIter {
|
||||
match self {
|
||||
obikindex::layer::KmerLayer::Count(l) => l.iter_siblings(annex),
|
||||
obikindex::layer::KmerLayer::Presence(l) => l.iter_siblings(annex),
|
||||
obikindex::layer::KmerLayer::Empty { .. } => {
|
||||
panic!("iter_siblings() called on an Empty layer")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_siblings_batch(&self, annex: Arc<SiblingAnnex>, batch_size: usize) -> SiblingBatchIter {
|
||||
match self {
|
||||
obikindex::layer::KmerLayer::Count(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obikindex::layer::KmerLayer::Presence(l) => l.iter_siblings_batch(annex, batch_size),
|
||||
obikindex::layer::KmerLayer::Empty { .. } => {
|
||||
panic!("iter_siblings_batch() called on an Empty layer")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants(&self, annex: Arc<SiblingAnnex>) -> MinorantIter {
|
||||
match self {
|
||||
obikindex::layer::KmerLayer::Count(l) => l.iter_minorants(annex),
|
||||
obikindex::layer::KmerLayer::Presence(l) => l.iter_minorants(annex),
|
||||
obikindex::layer::KmerLayer::Empty { .. } => {
|
||||
panic!("iter_minorants() called on an Empty layer")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn iter_minorants_batch(
|
||||
&self,
|
||||
annex: Arc<SiblingAnnex>,
|
||||
batch_size: usize,
|
||||
) -> MinorantBatchIter {
|
||||
match self {
|
||||
obikindex::layer::KmerLayer::Count(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obikindex::layer::KmerLayer::Presence(l) => l.iter_minorants_batch(annex, batch_size),
|
||||
obikindex::layer::KmerLayer::Empty { .. } => {
|
||||
panic!("iter_minorants_batch() called on an Empty layer")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
//! Family presence-mask annex construction.
|
||||
//!
|
||||
//! See `DevDocMD/theory/evolutionary_distances.md`, "Definitions: family, and
|
||||
//! the canonical form of a family" and "Step 2b", for the full design
|
||||
//! discussion this implements.
|
||||
//!
|
||||
//! For each distinct k-mer of each layer of the (already built/merged)
|
||||
//! index, computes a 4-bit presence mask for its "family" (the up to 4
|
||||
//! k-mers sharing its flanks, differing only at the central base —
|
||||
//! well-defined for odd k): bit `b` set iff the family member whose own
|
||||
//! canonical central base is `b` (0=A, 1=C, 2=G, 3=T) is observed anywhere
|
||||
//! in the current multi-genome index — a property of the whole index, not
|
||||
//! of any one genome. Sibling count and minorant are *derived* from the
|
||||
//! mask by callers, not stored (see `FamilyMask` and
|
||||
//! [`sibling_annex_stats`](stats::SiblingStatsExt::sibling_annex_stats)
|
||||
//! below).
|
||||
//!
|
||||
//! Per layer, an `obipipeline` batch transform (throttled — see
|
||||
//! `obipipeline::throttle`) generates a whole batch's central variants at
|
||||
//! once (`BATCH_SIZE` source k-mers in, that batch's variants out as one
|
||||
//! pipeline message), interleaved across many in-flight batches by the
|
||||
//! scheduler's shared worker pool rather than processed on a single
|
||||
//! thread. The actual cross-partition lookup reuses a `PartitionCache` of
|
||||
//! every partition's already-open MPHF layers, built once for the whole
|
||||
//! `build_sibling_annex` run, rather than reopening files per lookup or
|
||||
//! per source layer. Two earlier, coarser-grained designs were tried and
|
||||
//! measured (not guessed) to be worse, in order: (1) reopening/re-mmap'ing
|
||||
//! every target partition's files on every single lookup — fine at toy
|
||||
//! scale, ~90% system time against a real index; (2) a `Flat` pipeline
|
||||
//! stage pushing one message per generated *variant* (up to 3 per source
|
||||
//! k-mer) — cheaper than reopening files, but sampling a real run showed
|
||||
//! most wall-clock time going into per-message channel send/notify
|
||||
//! syscalls rather than the lookup itself, because a single k-mer's ≤3
|
||||
//! variants is far too fine a granularity to amortise a pipeline's
|
||||
//! synchronisation cost over. See `DevDocMD/theory/evolutionary_distances.md`,
|
||||
//! Step 2b, "Mechanism".
|
||||
//!
|
||||
//! Submodules, in the order data flows through them: [`cache`] (shared
|
||||
//! whole-run partition cache), [`helpers`] (small pure functions used
|
||||
//! throughout), [`build`] (annex construction), [`stats`] (family-size
|
||||
//! diagnostics), [`distance`] (raw SNP distance + base-pair tally),
|
||||
//! [`cardinality`] (cardinality co-occurrence), [`alignment`] (SNP-only
|
||||
//! pseudo-alignment).
|
||||
|
||||
mod alignment;
|
||||
mod build;
|
||||
mod cache;
|
||||
mod cardcomp;
|
||||
mod cardinality;
|
||||
mod distance;
|
||||
mod entropy;
|
||||
mod entropy_annex;
|
||||
mod family_scan;
|
||||
mod helpers;
|
||||
mod iter;
|
||||
mod sankoff_bundle;
|
||||
mod siblingannex;
|
||||
mod stats;
|
||||
mod subsample;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use alignment::{SnpAlignment, SnpAlignmentExt};
|
||||
pub use build::SiblingAnnexBuildExt;
|
||||
pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix};
|
||||
pub use cardinality::{CardinalityExt, CardinalityTally};
|
||||
pub use distance::{BasePairTally, DistanceExt, RawSnpDistanceOutput};
|
||||
pub use entropy::ShannonEntropyExt;
|
||||
pub use iter::{MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt};
|
||||
pub use sankoff_bundle::{SankoffBundle, SankoffBundleExt};
|
||||
pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
|
||||
pub use stats::{SiblingAnnexStats, SiblingStatsExt};
|
||||
pub use subsample::EntropyBias;
|
||||
|
||||
pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib";
|
||||
@@ -1,250 +0,0 @@
|
||||
//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree`
|
||||
//! pipeline — see `DevDocMD/architecture/siblings.md`, "`--free-loss`/`--tnt`
|
||||
//! pipeline: four independent scans, three of them unsampled" and "Wired
|
||||
//! into `pack` and the sibling-annex build path" for the problem this
|
||||
//! replaces: `raw_snp_distance`, `base_pair_tally`, `cardinality_tally` and
|
||||
//! `snp_pseudo_alignment` used to each open their own `PartitionCache` and
|
||||
//! independently re-scan the whole annex, three of the four ignoring
|
||||
//! `--subsample` entirely.
|
||||
//!
|
||||
//! `base_pair_tally`/`cardinality_tally` both need `raw_snp_distance`'s
|
||||
//! *complete* aggregate SNP/shared counts before they can derive
|
||||
//! `included[i,j]` (the `ratio_ceiling` filter) — a genuine sequential
|
||||
//! dependency, not an accidental one, so this is two passes, not one:
|
||||
//!
|
||||
//! - **Pass A**: aggregate SNP/shared counts per genome pair, over the
|
||||
//! shared selection (computed once, [`super::subsample::compute_selections`]
|
||||
//! — the *same* selection every other pass below uses, per the user's
|
||||
//! own requirement: "il faut qu'on sélectionne tout le monde avec la
|
||||
//! même sélection"). Excluded genomes (`--exclude-genome`) are zeroed
|
||||
//! here, before `included` is derived from it — the ratio filter must
|
||||
//! see the post-exclusion counts, not the raw ones.
|
||||
//! - **Pass B**: base-pair tally, cardinality tally, and the pseudo-
|
||||
//! alignment, fused into *one* scan over the same selection — the three
|
||||
//! are mutually independent once `included` is known, so they are
|
||||
//! accumulated together family-by-family instead of three more separate
|
||||
//! scans.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use ndarray::Array2;
|
||||
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::alignment::{SnpAlignment, iupac_code};
|
||||
use super::cache::PartitionCache;
|
||||
use super::cardinality::CardinalityTally;
|
||||
use super::distance::{BasePairTally, RawSnpDistanceOutput};
|
||||
use super::family_scan::{Selection, scan_layer_families, sibling_layer_dirs};
|
||||
use super::subsample::{EntropyBias, compute_selections};
|
||||
|
||||
/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs,
|
||||
/// computed together from one shared, possibly-subsampled/entropy-biased
|
||||
/// selection — see the module docs.
|
||||
pub struct SankoffBundle {
|
||||
pub raw: RawSnpDistanceOutput,
|
||||
pub base_pair_tally: BasePairTally,
|
||||
pub cardinality_tally: CardinalityTally,
|
||||
pub alignment: SnpAlignment,
|
||||
}
|
||||
|
||||
/// Adds [`sankoff_bundle`](Self::sankoff_bundle) to `KmerIndex`.
|
||||
pub trait SankoffBundleExt {
|
||||
/// `subsample`/`entropy_bias`: see
|
||||
/// [`SnpAlignmentExt::snp_pseudo_alignment`](super::SnpAlignmentExt::snp_pseudo_alignment).
|
||||
/// `ratio_ceiling`: see
|
||||
/// [`DistanceExt::base_pair_tally`](super::DistanceExt::base_pair_tally).
|
||||
/// `exclude_mask[g]`: zero genome `g`'s row/column out of the aggregate
|
||||
/// SNP/shared counts *before* `included[i,j]` is derived from them —
|
||||
/// same semantics as `--exclude-genome` had on the old, separate
|
||||
/// `raw_snp_distance` call.
|
||||
fn sankoff_bundle(
|
||||
&self,
|
||||
subsample: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
ratio_ceiling: f64,
|
||||
exclude_mask: &[bool],
|
||||
) -> OKIResult<SankoffBundle>;
|
||||
}
|
||||
|
||||
impl SankoffBundleExt for KmerIndex {
|
||||
fn sankoff_bundle(
|
||||
&self,
|
||||
subsample: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
ratio_ceiling: f64,
|
||||
exclude_mask: &[bool],
|
||||
) -> OKIResult<SankoffBundle> {
|
||||
let n_parts = self.n_partitions();
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = self.meta().config.with_counts;
|
||||
let k = self.kmer_size();
|
||||
let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?);
|
||||
let layer_dirs = sibling_layer_dirs(self)?;
|
||||
// Computed once — every pass below iterates the exact same
|
||||
// families, in the exact same layers, per the shared-selection
|
||||
// requirement this module exists to satisfy.
|
||||
let selections = compute_selections(self, &layer_dirs, subsample, entropy_bias)?;
|
||||
|
||||
// ── Pass A: aggregate SNP/shared counts ────────────────────────────
|
||||
let pb = progress_bar("raw_snp_distance", layer_dirs.len() as u64, "layers");
|
||||
let mut snp = Array2::<u64>::zeros((n_genomes, n_genomes));
|
||||
let mut shared = Array2::<u64>::zeros((n_genomes, n_genomes));
|
||||
for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) {
|
||||
let selection = match layer_selection {
|
||||
None => Selection::All,
|
||||
Some(set) => Selection::Some(set),
|
||||
};
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&selection,
|
||||
|_family_idx, _mask, genome_mask| {
|
||||
let single_form = |g: usize| -> Option<u8> {
|
||||
let m = genome_mask[g];
|
||||
(m.count_ones() == 1).then(|| m.trailing_zeros() as u8)
|
||||
};
|
||||
for i in 0..n_genomes {
|
||||
let Some(bi) = single_form(i) else { continue };
|
||||
for j in (i + 1)..n_genomes {
|
||||
let Some(bj) = single_form(j) else { continue };
|
||||
if bi == bj {
|
||||
shared[[i, j]] += 1;
|
||||
shared[[j, i]] += 1;
|
||||
} else {
|
||||
snp[[i, j]] += 1;
|
||||
snp[[j, i]] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
// `--exclude-genome`: zero excluded rows/columns before `included`
|
||||
// is derived below — same effect the old standalone
|
||||
// `zero_excluded_pairs` had, applied here since this is now the
|
||||
// only place `included` gets computed for this pipeline.
|
||||
for (i, &excluded) in exclude_mask.iter().enumerate() {
|
||||
if !excluded {
|
||||
continue;
|
||||
}
|
||||
for j in 0..n_genomes {
|
||||
snp[[i, j]] = 0;
|
||||
snp[[j, i]] = 0;
|
||||
shared[[i, j]] = 0;
|
||||
shared[[j, i]] = 0;
|
||||
}
|
||||
}
|
||||
let raw = RawSnpDistanceOutput { snp, shared };
|
||||
|
||||
let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| {
|
||||
if i == j {
|
||||
return false;
|
||||
}
|
||||
let s = raw.snp[[i, j]];
|
||||
let total = s + raw.shared[[i, j]];
|
||||
total > 0 && (s as f64 / total as f64) <= ratio_ceiling
|
||||
});
|
||||
|
||||
// ── Pass B: base-pair tally + cardinality tally + alignment, fused ──
|
||||
let pb = progress_bar("sankoff_pass_b", layer_dirs.len() as u64, "layers");
|
||||
let mut bp_counts = [[0u64; 4]; 4];
|
||||
let mut bp_same = [0u64; 4];
|
||||
let mut card_counts = [[0u64; 5]; 5];
|
||||
let mut sequences: Vec<Vec<u8>> = vec![Vec::new(); n_genomes];
|
||||
for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) {
|
||||
let selection = match layer_selection {
|
||||
None => Selection::All,
|
||||
Some(set) => Selection::Some(set),
|
||||
};
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&selection,
|
||||
|_family_idx, mask, genome_mask| {
|
||||
let variable = mask.family_size() >= 2;
|
||||
|
||||
// base-pair tally — same pairwise single-form resolution as pass A.
|
||||
let single_form = |g: usize| -> Option<u8> {
|
||||
let m = genome_mask[g];
|
||||
(m.count_ones() == 1).then(|| m.trailing_zeros() as u8)
|
||||
};
|
||||
for i in 0..n_genomes {
|
||||
let Some(bi) = single_form(i) else { continue };
|
||||
for j in (i + 1)..n_genomes {
|
||||
let Some(bj) = single_form(j) else { continue };
|
||||
if !included[[i, j]] {
|
||||
continue;
|
||||
}
|
||||
if bi != bj {
|
||||
bp_counts[bi as usize][bj as usize] += 1;
|
||||
bp_counts[bj as usize][bi as usize] += 1;
|
||||
} else if variable {
|
||||
bp_same[bi as usize] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cardinality tally — restricted to variable families, see
|
||||
// `CardinalityExt::cardinality_tally`'s docs for why.
|
||||
if variable {
|
||||
for i in 0..n_genomes {
|
||||
let card_i = genome_mask[i].count_ones() as usize;
|
||||
for j in (i + 1)..n_genomes {
|
||||
if !included[[i, j]] {
|
||||
continue;
|
||||
}
|
||||
let card_j = genome_mask[j].count_ones() as usize;
|
||||
card_counts[card_i][card_j] += 1;
|
||||
if card_i != card_j {
|
||||
card_counts[card_j][card_i] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pseudo-alignment — same variable-family gate as
|
||||
// `snp_pseudo_alignment`. Every family the shared selection
|
||||
// actually visits already satisfies this when the
|
||||
// selection is `Selection::Some` (only non-monomorphic
|
||||
// minorants are ever selected), but the explicit check
|
||||
// still matters under `Selection::All` (no `--subsample`),
|
||||
// which visits monomorphic minorants too.
|
||||
if variable {
|
||||
for (g, &m) in genome_mask.iter().enumerate() {
|
||||
sequences[g].push(iupac_code(m));
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(SankoffBundle {
|
||||
raw,
|
||||
base_pair_tally: BasePairTally {
|
||||
counts: bp_counts,
|
||||
same: bp_same,
|
||||
},
|
||||
cardinality_tally: CardinalityTally {
|
||||
counts: card_counts,
|
||||
},
|
||||
alignment: SnpAlignment { sequences },
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,433 +0,0 @@
|
||||
//! Family presence-mask annex: a compact, read-only-after-build, per-slot
|
||||
//! derived value used by the central-position SNP distance estimator (see
|
||||
//! `DevDocMD/theory/evolutionary_distances.md`, "Step 2b" and "Definitions:
|
||||
//! family, and the canonical form of a family").
|
||||
//!
|
||||
//! Two bytes are stored per MPHF slot of a partition/layer, packed as four
|
||||
//! 3-bit fields (one per central base, in the fixed A/C/G/T = 0/1/2/3
|
||||
//! encoding) plus a minorant flag:
|
||||
//!
|
||||
//! - Each 3-bit field: `0` = the family member whose *own* central base —
|
||||
//! in its own canonical orientation — is that base is absent from the
|
||||
//! whole multi-genome index (a property of the whole index, not of any
|
||||
//! one genome); a non-zero value `v` (`1..=7`) means it's present, and
|
||||
//! *may* additionally encode `layer = v - 1` (the destination layer
|
||||
//! within its partition) once a caller populates it via
|
||||
//! [`with_layer`](FamilyMask::with_layer) — see that method's docs.
|
||||
//! `FamilyMask` itself is policy-free about whether a given non-zero
|
||||
//! value is a trustworthy layer index or just "present, no layer
|
||||
//! recorded": that decision belongs to callers, who already have
|
||||
//! `PartitionMeta::n_layers` in scope.
|
||||
//! - The minorant bit: whether *this slot's own k-mer* is its family's
|
||||
//! minorant (the smallest raw encoding among the family's observed
|
||||
//! members) — computed once, when the whole family's mask is already
|
||||
//! final (see `build::build_layer_sibling_annex`), and read back by every
|
||||
//! consumer that would otherwise have to reconstruct this slot's k-mer
|
||||
//! from `unitigs.bin` and hash it through the MPHF again just to ask the
|
||||
//! same question.
|
||||
//!
|
||||
//! Both facts an even earlier design stored explicitly are still derived,
|
||||
//! not stored:
|
||||
//! - sibling count = `family_size() - 1`;
|
||||
//! - which of the (up to 4) family members are present, and their canonical
|
||||
//! form — [`FamilyMask::family_members`], regenerated from the slot's own
|
||||
//! k-mer (`CanonicalKmer::central_canonical_neighbors`, cheap, no
|
||||
//! lookup), not stored.
|
||||
//!
|
||||
//! Mask value 0 is logically unreachable as a real result (a slot's own
|
||||
//! base is always present in its own family) and is reused as the "not yet
|
||||
//! computed" sentinel: annex files are pre-initialised to all-zero, and a
|
||||
//! real value is only ever written once, by the computation pass.
|
||||
//!
|
||||
//! Widened from the original 1-byte/slot design (4 presence bits + 1
|
||||
//! minorant bit) to carry a per-member layer number without an extra
|
||||
//! lookup pass — see the project discussion this implements. An old
|
||||
//! (1-byte/slot) `.psib` file is *not* silently misread by the new reader:
|
||||
//! its length no longer matches `HEADER_SIZE + n * 2`, so `SiblingAnnex::open`
|
||||
//! fails loudly ("PSIB file truncated") rather than producing garbage.
|
||||
//! Annexes built before this change must be rebuilt (`--sibling-annex`).
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::AtomicU16;
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
|
||||
use super::helpers::central_base;
|
||||
|
||||
const MAGIC: [u8; 4] = *b"PSIB";
|
||||
|
||||
// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (2 bytes/slot) follows.
|
||||
const HEADER_SIZE: usize = 16;
|
||||
|
||||
/// Width, in bits, of one base's field.
|
||||
const FIELD_BITS: u32 = 3;
|
||||
/// Mask for one base's field once shifted into position.
|
||||
const FIELD_MASK: u16 = 0b111;
|
||||
/// Highest raw field value a base can carry (`1..=MAX_FIELD_VALUE`, `0` = absent).
|
||||
const MAX_FIELD_VALUE: u8 = 7;
|
||||
|
||||
const MINORANT_BIT: u16 = 1 << 12;
|
||||
|
||||
/// A family presence mask — see the module docs for the full bit layout.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct FamilyMask(u16);
|
||||
|
||||
impl FamilyMask {
|
||||
/// The empty mask — never a valid *computed* result (a slot's own base
|
||||
/// is always present in its own family) — used only to build up a mask
|
||||
/// via repeated [`with`](Self::with)/[`with_layer`](Self::with_layer)
|
||||
/// calls before storing it.
|
||||
pub const EMPTY: FamilyMask = FamilyMask(0);
|
||||
|
||||
#[inline]
|
||||
fn field_shift(base: u8) -> u32 {
|
||||
debug_assert!(base < 4, "base out of range: {base}");
|
||||
base as u32 * FIELD_BITS
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn field(self, base: u8) -> u8 {
|
||||
((self.0 >> Self::field_shift(base)) & FIELD_MASK) as u8
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn with_field(self, base: u8, value: u8) -> Self {
|
||||
debug_assert!(value <= MAX_FIELD_VALUE, "field value out of range: {value}");
|
||||
let shift = Self::field_shift(base);
|
||||
let cleared = self.0 & !(FIELD_MASK << shift);
|
||||
FamilyMask(cleared | ((value as u16) << shift))
|
||||
}
|
||||
|
||||
/// Raw bits to `fetch_or` into a shared `AtomicU16` accumulator to mark
|
||||
/// `base` present at `layer`, without allocating a whole `FamilyMask`
|
||||
/// per write — for `build::build_layer_sibling_annex`'s concurrent
|
||||
/// construction, where each `(word, base)` field is written by exactly
|
||||
/// one thread (see that module's docs), so a lock-free OR is safe.
|
||||
#[inline]
|
||||
pub(crate) fn layer_bits(base: u8, layer: usize) -> u16 {
|
||||
debug_assert!(layer < MAX_FIELD_VALUE as usize, "layer out of range: {layer}");
|
||||
(layer as u16 + 1) << Self::field_shift(base)
|
||||
}
|
||||
|
||||
/// Same as [`layer_bits`](Self::layer_bits), presence-only (no layer
|
||||
/// recorded) — the accumulator-side equivalent of [`with`](Self::with).
|
||||
#[inline]
|
||||
pub(crate) fn presence_bits(base: u8) -> u16 {
|
||||
1u16 << Self::field_shift(base)
|
||||
}
|
||||
|
||||
/// Mark the member with central base `base` (0=A, 1=C, 2=G, 3=T) as
|
||||
/// present, without recording a layer (compat path: same observable
|
||||
/// effect as the original 1-byte design's `with`). Use
|
||||
/// [`with_layer`](Self::with_layer) instead when the destination layer
|
||||
/// is already known.
|
||||
#[inline]
|
||||
pub fn with(self, base: u8) -> Self {
|
||||
self.with_field(base, 1)
|
||||
}
|
||||
|
||||
/// Mark the member with central base `base` as present *and* record its
|
||||
/// destination layer. `layer` must be `< 7` (`debug_assert`ed) — a
|
||||
/// caller facing more layers than that has no compact field to record
|
||||
/// them in and must fall back to [`with`](Self::with) instead (still
|
||||
/// correct, just without the fast-path payoff); see the module docs on
|
||||
/// why `FamilyMask` doesn't decide that threshold itself.
|
||||
#[inline]
|
||||
pub fn with_layer(self, base: u8, layer: usize) -> Self {
|
||||
debug_assert!(layer < MAX_FIELD_VALUE as usize, "layer out of range: {layer}");
|
||||
self.with_field(base, layer as u8 + 1)
|
||||
}
|
||||
|
||||
/// Is the member with central base `base` (0..3) present?
|
||||
#[inline]
|
||||
pub fn has(self, base: u8) -> bool {
|
||||
self.field(base) != 0
|
||||
}
|
||||
|
||||
/// Raw stored value for `base`'s field, if present: `Some(v - 1)` where
|
||||
/// `v` is the non-zero field value. This is the *raw* stored value, not
|
||||
/// a validated layer index — a caller must cross-check it against
|
||||
/// `PartitionMeta::n_layers` (`<= 7`) before trusting it as a real
|
||||
/// layer, since a mask written via [`with`](Self::with) (no layer
|
||||
/// known) also reads back as `Some(0)` here.
|
||||
#[inline]
|
||||
pub fn layer_value(self, base: u8) -> Option<u8> {
|
||||
let v = self.field(base);
|
||||
if v == 0 { None } else { Some(v - 1) }
|
||||
}
|
||||
|
||||
/// Number of family members observed anywhere in the index (1..=4).
|
||||
#[inline]
|
||||
pub fn family_size(self) -> u32 {
|
||||
(0..4).filter(|&b| self.has(b)).count() as u32
|
||||
}
|
||||
|
||||
/// Number of *other* members observed (0..=3) — `family_size() - 1`.
|
||||
#[inline]
|
||||
pub fn siblings(self) -> u32 {
|
||||
self.family_size() - 1
|
||||
}
|
||||
|
||||
/// Set or clear the minorant flag (see the module docs).
|
||||
#[inline]
|
||||
pub fn with_minorant(self, is_minorant: bool) -> Self {
|
||||
if is_minorant {
|
||||
FamilyMask(self.0 | MINORANT_BIT)
|
||||
} else {
|
||||
FamilyMask(self.0 & !MINORANT_BIT)
|
||||
}
|
||||
}
|
||||
|
||||
/// Is this slot's own k-mer its family's minorant? Only meaningful once
|
||||
/// `with_minorant` has been called with the family's *final* mask (i.e.
|
||||
/// after construction) — see the module docs.
|
||||
#[inline]
|
||||
pub fn is_minorant(self) -> bool {
|
||||
self.0 & MINORANT_BIT != 0
|
||||
}
|
||||
|
||||
/// Raw presence bitmask (bit `b` = base `b` present, low 4 bits only —
|
||||
/// never includes the minorant flag or any layer information) — for
|
||||
/// callers that only need the old 1-bit-per-base view, e.g. to compare
|
||||
/// against a previously-computed value built via their own bit
|
||||
/// operations.
|
||||
#[inline]
|
||||
pub fn bits(self) -> u8 {
|
||||
(0..4).fold(0u8, |acc, b| if self.has(b) { acc | (1 << b) } else { acc })
|
||||
}
|
||||
|
||||
/// Construct from a raw presence bitmask (only the low 4 bits are kept
|
||||
/// — the minorant flag is not part of this, use
|
||||
/// [`with_minorant`](Self::with_minorant) separately). Each set bit is
|
||||
/// recorded as "present, no layer known" — see [`with`](Self::with).
|
||||
#[inline]
|
||||
pub fn from_bits(bits: u8) -> Self {
|
||||
(0..4).fold(FamilyMask::EMPTY, |mask, b| {
|
||||
if bits & (1 << b) != 0 { mask.with(b) } else { mask }
|
||||
})
|
||||
}
|
||||
|
||||
/// This family's present members, in canonical form, paired with their
|
||||
/// raw stored field value (see [`layer_value`](Self::layer_value) for
|
||||
/// why it's not directly a validated layer index). `owner` is this
|
||||
/// slot's own k-mer (any member works — the 4 canonical forms are
|
||||
/// invariant regardless of which member you start from).
|
||||
pub fn family_members(self, owner: CanonicalKmer, k: usize) -> impl Iterator<Item = (CanonicalKmer, Option<u8>)> {
|
||||
owner
|
||||
.central_canonical_neighbors()
|
||||
.into_iter()
|
||||
.filter_map(move |member| {
|
||||
let base = central_base(member, k);
|
||||
self.has(base).then(|| (member, self.layer_value(base)))
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn encode(self) -> u16 {
|
||||
self.0
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn decode(word: u16) -> Option<Self> {
|
||||
if word == 0 {
|
||||
// Unreachable for a real result — reserved as the "not yet
|
||||
// computed" sentinel. A real entry always has at least one
|
||||
// non-zero base field, so word == 0 always means "nothing
|
||||
// written yet", never a genuine minorant-only value.
|
||||
return None;
|
||||
}
|
||||
Some(FamilyMask(word))
|
||||
}
|
||||
}
|
||||
|
||||
// ── SiblingAnnex (reader) ───────────────────────────────────────────────────
|
||||
|
||||
pub struct SiblingAnnex {
|
||||
mmap: Mmap,
|
||||
n: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SiblingAnnex {
|
||||
pub fn open(path: &Path) -> io::Result<Self> {
|
||||
let mmap = unsafe { Mmap::map(&File::open(path)?)? };
|
||||
if mmap.len() < HEADER_SIZE {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short"));
|
||||
}
|
||||
if mmap[0..4] != MAGIC {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic"));
|
||||
}
|
||||
let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
|
||||
if mmap.len() < HEADER_SIZE + n * 2 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated"));
|
||||
}
|
||||
Ok(Self { mmap, n, path: path.to_path_buf() })
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path { &self.path }
|
||||
pub fn len(&self) -> usize { self.n }
|
||||
pub fn is_empty(&self) -> bool { self.n == 0 }
|
||||
|
||||
/// `None` means the slot has not (yet) been computed — see module docs.
|
||||
pub fn get(&self, slot: usize) -> Option<FamilyMask> {
|
||||
let off = HEADER_SIZE + slot * 2;
|
||||
FamilyMask::decode(u16::from_le_bytes(self.mmap[off..off + 2].try_into().unwrap()))
|
||||
}
|
||||
}
|
||||
|
||||
// ── SiblingAnnexBuilder (writer) ────────────────────────────────────────────
|
||||
|
||||
pub struct SiblingAnnexBuilder {
|
||||
mmap: MmapMut,
|
||||
}
|
||||
|
||||
impl SiblingAnnexBuilder {
|
||||
/// Create a new annex of `n` slots at `path`, pre-initialised to the
|
||||
/// "not yet computed" sentinel (all-zero).
|
||||
pub fn new(n: usize, path: &Path) -> io::Result<Self> {
|
||||
let file_size = HEADER_SIZE + n * 2;
|
||||
let file = OpenOptions::new()
|
||||
.read(true).write(true).create(true).truncate(true)
|
||||
.open(path)?;
|
||||
file.set_len(file_size as u64)?;
|
||||
let mut mmap = unsafe { MmapMut::map_mut(&file)? };
|
||||
mmap[0..4].copy_from_slice(&MAGIC);
|
||||
mmap[4..8].copy_from_slice(&[0u8; 4]);
|
||||
mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
|
||||
// Data region left at 0 by `set_len`/mmap — the sentinel value.
|
||||
Ok(Self { mmap })
|
||||
}
|
||||
|
||||
/// `None` means the slot has not (yet) been computed — see module
|
||||
/// docs. Used to read back a slot's concurrently-accumulated value
|
||||
/// (via [`atomic_slot`](Self::atomic_slot)) before finalising it with
|
||||
/// [`set`](Self::set).
|
||||
pub fn get(&self, slot: usize) -> Option<FamilyMask> {
|
||||
let off = HEADER_SIZE + slot * 2;
|
||||
FamilyMask::decode(u16::from_le_bytes(self.mmap[off..off + 2].try_into().unwrap()))
|
||||
}
|
||||
|
||||
/// Atomic handle to `slot`'s raw storage, for lock-free concurrent
|
||||
/// construction straight into the mmap — no separate in-memory
|
||||
/// accumulator needed (see `build::build_layer_sibling_annex`, the
|
||||
/// only caller). Safe: every concurrent write goes through
|
||||
/// `fetch_or`/atomic RMW, and by that caller's own invariant, a given
|
||||
/// 3-bit field within a slot's word is written by exactly one thread —
|
||||
/// only *different* fields of the same word can race, which `fetch_or`
|
||||
/// already serialises correctly. Alignment holds: `HEADER_SIZE` is
|
||||
/// even and every slot is 2 bytes, so `HEADER_SIZE + slot * 2` is
|
||||
/// always even, and the mmap's own base address is page-aligned.
|
||||
pub(crate) fn atomic_slot(&self, slot: usize) -> &AtomicU16 {
|
||||
let off = HEADER_SIZE + slot * 2;
|
||||
debug_assert!(off + 2 <= self.mmap.len());
|
||||
let ptr = unsafe { self.mmap.as_ptr().add(off) as *mut u16 };
|
||||
unsafe { AtomicU16::from_ptr(ptr) }
|
||||
}
|
||||
|
||||
/// Plain (non-atomic) store — for the sequential finalisation pass
|
||||
/// only, once every concurrent write (via
|
||||
/// [`atomic_slot`](Self::atomic_slot)) is done.
|
||||
pub fn set(&mut self, slot: usize, mask: FamilyMask) {
|
||||
let off = HEADER_SIZE + slot * 2;
|
||||
self.mmap[off..off + 2].copy_from_slice(&mask.encode().to_le_bytes());
|
||||
}
|
||||
|
||||
pub fn close(self) -> io::Result<()> { self.mmap.flush() }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let builder = SiblingAnnexBuilder::new(4, &path).unwrap();
|
||||
builder.close().unwrap();
|
||||
let annex = SiblingAnnex::open(&path).unwrap();
|
||||
for slot in 0..4 {
|
||||
assert_eq!(annex.get(slot), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_all_valid_masks() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.psib");
|
||||
let mut builder = SiblingAnnexBuilder::new(4, &path).unwrap();
|
||||
|
||||
let masks = [
|
||||
FamilyMask::EMPTY.with(0), // just A: family size 1
|
||||
FamilyMask::EMPTY.with(0).with(3), // A + T: size 2
|
||||
FamilyMask::EMPTY.with(1).with(2).with(3), // C+G+T: size 3
|
||||
FamilyMask::EMPTY.with(0).with(1).with(2).with(3), // all 4
|
||||
];
|
||||
for (slot, mask) in masks.iter().enumerate() {
|
||||
builder.set(slot, *mask);
|
||||
}
|
||||
builder.close().unwrap();
|
||||
let annex = SiblingAnnex::open(&path).unwrap();
|
||||
for (slot, mask) in masks.iter().enumerate() {
|
||||
assert_eq!(annex.get(slot), Some(*mask));
|
||||
}
|
||||
assert_eq!(annex.get(0).unwrap().siblings(), 0);
|
||||
assert_eq!(annex.get(1).unwrap().siblings(), 1);
|
||||
assert_eq!(annex.get(2).unwrap().siblings(), 2);
|
||||
assert_eq!(annex.get(3).unwrap().siblings(), 3);
|
||||
assert_eq!(annex.get(3).unwrap().family_size(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_reflects_individual_bits() {
|
||||
let mask = FamilyMask::EMPTY.with(0).with(2);
|
||||
assert!(mask.has(0));
|
||||
assert!(!mask.has(1));
|
||||
assert!(mask.has(2));
|
||||
assert!(!mask.has(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_layer_roundtrips_and_leaves_absent_bases_none() {
|
||||
let mask = FamilyMask::EMPTY.with_layer(0, 0).with_layer(2, 5);
|
||||
assert_eq!(mask.layer_value(0), Some(0));
|
||||
assert_eq!(mask.layer_value(2), Some(5));
|
||||
assert_eq!(mask.layer_value(1), None);
|
||||
assert_eq!(mask.layer_value(3), None);
|
||||
assert!(mask.has(0) && mask.has(2));
|
||||
assert!(!mask.has(1) && !mask.has(3));
|
||||
assert_eq!(mask.family_size(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_marks_present_without_a_layer() {
|
||||
// Compat path: `with` still means "present", now reading back as
|
||||
// `layer_value == Some(0)` (raw field value 1, i.e. "no layer
|
||||
// recorded") — distinguishable from a real layer 0 only by a
|
||||
// caller that already knows whether layers were ever wired in.
|
||||
let mask = FamilyMask::EMPTY.with(1);
|
||||
assert!(mask.has(1));
|
||||
assert_eq!(mask.layer_value(1), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn family_members_reconstructs_present_canonical_forms() {
|
||||
use obikseq::{Kmer, Sequence};
|
||||
|
||||
const K: usize = 11;
|
||||
obikseq::params::set_k(K);
|
||||
// Centre (index 5) is 'C' (base 1) in this k-mer's own orientation.
|
||||
let owner = Kmer::from_ascii(b"AACCGCTTAAG").unwrap().canonical();
|
||||
let mask = FamilyMask::EMPTY.with_layer(1, 0).with_layer(2, 3); // C (own) + G present
|
||||
|
||||
let members: Vec<_> = mask.family_members(owner, K).collect();
|
||||
assert_eq!(members.len(), 2, "only the 2 present members should be yielded");
|
||||
assert!(members.iter().any(|(k, layer)| *k == owner && *layer == Some(0)));
|
||||
assert!(members.iter().any(|(k, layer)| *k != owner && *layer == Some(3)));
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obisys::progress_bar;
|
||||
|
||||
use obikindex::KmerIndex;
|
||||
use obikindex::{OKIError, OKIResult};
|
||||
|
||||
use super::ANNEX_FILE_NAME;
|
||||
use super::SiblingAnnex;
|
||||
use super::cache::PartitionCache;
|
||||
use super::family_scan::{Selection, scan_layer_families};
|
||||
|
||||
/// Distribution of family sizes (1-4), read back from an already-built
|
||||
/// annex (see [`super::build::SiblingAnnexBuildExt::build_sibling_annex`])
|
||||
/// plus the index's presence/count data — a separate, occasional diagnostic
|
||||
/// pass, not fused into construction.
|
||||
///
|
||||
/// Every count here is **per family, not per slot**: a family with `F`
|
||||
/// members occupies `F` annex slots (one per observed member), all sharing
|
||||
/// the same mask. Counting every slot would count each family up to 4
|
||||
/// times over; only the minorant's slot is tallied (minorant is derived on
|
||||
/// the fly — see `is_minorant` — not stored, but cheap: no lookup, pure
|
||||
/// bit arithmetic on already-in-hand data).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct SiblingAnnexStats {
|
||||
/// `counts[s]` = number of *families* of size `s + 1` (index 0 = size 1,
|
||||
/// i.e. 0 siblings, ... index 3 = size 4, i.e. 3 siblings).
|
||||
pub counts: [u64; 4],
|
||||
/// `per_genome[g][s]` = number of families of size `s + 1` for which
|
||||
/// genome `g` (index into `KmerIndex::meta().genomes`) carries at least
|
||||
/// one member.
|
||||
pub per_genome: Vec<[u64; 4]>,
|
||||
}
|
||||
|
||||
/// Adds [`sibling_family_size_histogram`](Self::sibling_family_size_histogram)
|
||||
/// and [`sibling_annex_stats`](Self::sibling_annex_stats) to `KmerIndex`.
|
||||
pub trait SiblingStatsExt {
|
||||
/// The global family-size histogram alone (`SiblingAnnexStats::counts`,
|
||||
/// no `per_genome`) — reads only the already-built annex (`mask.siblings()`
|
||||
/// + `mask.is_minorant()`, 1 byte/slot, mmap'd), nothing else: no
|
||||
/// `unitigs.bin` scan, no MPHF lookup, no `PartitionCache`. Cost is a
|
||||
/// linear scan of one small file per layer, independent of the rest of
|
||||
/// the index's size or how much of it is paged in. An earlier version
|
||||
/// re-derived minorant-ness per slot (re-reading `unitigs.bin`, hashing
|
||||
/// every k-mer through the MPHF again to reconstruct what
|
||||
/// `build_sibling_annex` already knew) — sampling a real run showed
|
||||
/// `MphfLayer::find` alone at 71% of wall-clock time for what should be
|
||||
/// a near-instant four-bucket count. `mask.is_minorant()` is that same
|
||||
/// fact, computed once at construction (see `build_layer_sibling_annex`)
|
||||
/// and stored in the annex's own spare bits — free to read back.
|
||||
/// [`sibling_annex_stats`](Self::sibling_annex_stats) computes the same
|
||||
/// `counts` but pays the full per-genome cross-partition resolution
|
||||
/// cost to get there too; use this when only the global histogram is
|
||||
/// needed. Requires an annex built after the minorant flag was added —
|
||||
/// re-run `build_sibling_annex` if this reads as all-zero on an older one.
|
||||
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]>;
|
||||
|
||||
/// Tally the family-size distribution of an already-built annex
|
||||
/// (globally, and per genome), counting each family once (at its
|
||||
/// minorant slot). Errors if
|
||||
/// [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex)
|
||||
/// has not been run on this index first.
|
||||
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats>;
|
||||
}
|
||||
|
||||
impl SiblingStatsExt for KmerIndex {
|
||||
fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> {
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(self)?;
|
||||
|
||||
// Each layer's annex file is independent — no shared `PartitionCache`
|
||||
// and no `scan_layer_families` partition-grouped locality to protect
|
||||
// (unlike `sibling_annex_stats`/`distance`/`cardinality_tally`), so
|
||||
// layers can be scanned concurrently.
|
||||
layer_dirs
|
||||
.par_iter()
|
||||
.try_fold(
|
||||
|| [0u64; 4],
|
||||
|mut counts, layer_dir| -> OKIResult<[u64; 4]> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?;
|
||||
for slot in 0..annex.len() {
|
||||
let Some(mask) = annex.get(slot) else {
|
||||
continue;
|
||||
};
|
||||
if !mask.is_minorant() {
|
||||
continue; // family tallied once, at its minorant
|
||||
}
|
||||
counts[mask.siblings() as usize] += 1;
|
||||
}
|
||||
Ok(counts)
|
||||
},
|
||||
)
|
||||
.try_reduce(
|
||||
|| [0u64; 4],
|
||||
|a, b| {
|
||||
let mut sum = a;
|
||||
for i in 0..4 {
|
||||
sum[i] += b[i];
|
||||
}
|
||||
Ok(sum)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn sibling_annex_stats(&self) -> OKIResult<SiblingAnnexStats> {
|
||||
let n_parts = self.n_partitions();
|
||||
let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len();
|
||||
let with_counts = self.meta().config.with_counts;
|
||||
let k = self.kmer_size();
|
||||
// Same whole-run cache as `build_sibling_annex` — see its docs for
|
||||
// why re-opening per lookup (or per call to a batching helper) is
|
||||
// not good enough on a real index.
|
||||
let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?);
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(self)?;
|
||||
|
||||
// One layer at a time, not parallelised across layers — see
|
||||
// `snp_pseudo_alignment`'s comment for why `par_iter()` over layers
|
||||
// would defeat `scan_layer_families`'s partition-grouped locality.
|
||||
let pb = progress_bar("sibling_annex_stats", layer_dirs.len() as u64, "layers");
|
||||
let mut stats = SiblingAnnexStats {
|
||||
per_genome: vec![[0u64; 4]; n_genomes],
|
||||
..Default::default()
|
||||
};
|
||||
for layer_dir in &layer_dirs {
|
||||
scan_layer_families(
|
||||
layer_dir,
|
||||
n_parts,
|
||||
n_genomes,
|
||||
with_counts,
|
||||
k,
|
||||
&cache,
|
||||
&Selection::All,
|
||||
|_family_idx, mask, genome_mask| {
|
||||
// "Genome g represents this family" means g carries
|
||||
// *any* of its members, not just the minorant's own —
|
||||
// `genome_mask[g] != 0` is exactly that.
|
||||
let s = mask.siblings() as usize;
|
||||
stats.counts[s] += 1;
|
||||
for (g, &m) in genome_mask.iter().enumerate() {
|
||||
if m != 0 {
|
||||
stats.per_genome[g][s] += 1;
|
||||
}
|
||||
}
|
||||
},
|
||||
)?;
|
||||
pb.inc(1);
|
||||
}
|
||||
pb.finish_and_clear();
|
||||
|
||||
Ok(stats)
|
||||
}
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
//! Proportional per-layer sampling of non-monomorphic minorants
|
||||
//! (`family_size() >= 2`) — see `DevDocMD/architecture/siblings.md`,
|
||||
//! "`--subsample`/`--shannon`", for the full design discussion.
|
||||
//!
|
||||
//! Three steps, in order, each cheaper than the one before because it
|
||||
//! narrows what the next one has to touch:
|
||||
//!
|
||||
//! 1. [`non_monomorphic_counts`] — one cheap structural pass per layer
|
||||
//! (annex bits only, no cross-partition resolution), giving `count_layer`
|
||||
//! for every layer.
|
||||
//! 2. [`sample_layers`] — given a target `n` and those counts, computes
|
||||
//! `n_layer = round(n * count_layer / total_count)` per layer (never
|
||||
//! exceeds `count_layer` as long as `n <= total_count`; if
|
||||
//! `total_count <= n`, sampling is skipped entirely — every
|
||||
//! non-monomorphic minorant of every layer is kept) and reservoir-samples
|
||||
//! (Algorithm R, one more cheap structural pass, `O(n_layer)` memory) the
|
||||
//! selected iteration-order indices.
|
||||
//! 3. The result feeds [`super::family_scan::scan_layer_families`]'s
|
||||
//! `Selection` parameter, which bounds the expensive cross-partition
|
||||
//! resolution to just the selected families.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use rand::RngExt;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikindex::{KmerIndex, OKIError, OKIResult};
|
||||
|
||||
use super::ANNEX_FILE_NAME;
|
||||
use super::SiblingAnnex;
|
||||
use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME};
|
||||
|
||||
/// Gaussian-kernel entropy bias parameters — see
|
||||
/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". `mu`/
|
||||
/// `sigma` are the `--entropy`/`--entropy-sd` CLI values (defaulted by the
|
||||
/// caller, not here — this type carries only the resolved numbers). `pub`
|
||||
/// (not `pub(super)`): part of the public signature of
|
||||
/// `SnpAlignmentExt::snp_pseudo_alignment`,
|
||||
/// `ShannonEntropyExt::shannon_entropy_csv`, and
|
||||
/// `SankoffBundleExt::sankoff_bundle`, all consumed from `obikmer`.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct EntropyBias {
|
||||
pub mu: f64,
|
||||
pub sigma: f64,
|
||||
}
|
||||
|
||||
/// A family is non-monomorphic (informative-eligible) iff at least one
|
||||
/// sibling is registered alongside it — `family_size() >= 2`, i.e.
|
||||
/// `siblings() >= 1`. Mirrors the filter already applied post-hoc in
|
||||
/// `snp_pseudo_alignment`, but checked here *before* any resolution work.
|
||||
fn is_non_monomorphic_minorant(mask: super::FamilyMask) -> bool {
|
||||
mask.is_minorant() && mask.siblings() >= 1
|
||||
}
|
||||
|
||||
/// Step 1: per-layer count of non-monomorphic minorants — cheap, parallel
|
||||
/// across layers, annex bits only (same shape as
|
||||
/// `SiblingStatsExt::sibling_family_size_histogram`, but keeping the
|
||||
/// per-layer breakdown instead of collapsing it into one index-wide sum,
|
||||
/// since proportional sampling needs each layer's own share).
|
||||
pub(super) fn non_monomorphic_counts(layer_dirs: &[PathBuf]) -> OKIResult<Vec<u64>> {
|
||||
layer_dirs
|
||||
.par_iter()
|
||||
.map(|layer_dir| -> OKIResult<u64> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?;
|
||||
let mut count = 0u64;
|
||||
for slot in 0..annex.len() {
|
||||
if annex.get(slot).is_some_and(is_non_monomorphic_minorant) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Step 2, per layer: Algorithm-R reservoir sampling of `n_layer`
|
||||
/// non-monomorphic minorant iteration-indices — single pass, no
|
||||
/// cross-partition resolution, `O(n_layer)` memory regardless of the
|
||||
/// layer's true size (never materialises the full non-monomorphic index
|
||||
/// list, only the reservoir itself).
|
||||
///
|
||||
/// The stored values are **`family_idx`, `scan_layer_families`'s own
|
||||
/// numbering** — the index among *every* minorant of the layer (monomorphic
|
||||
/// ones included, since `iter_minorants_batch` only filters on
|
||||
/// `is_minorant()`), not the raw annex slot (`for slot in 0..annex.len()`
|
||||
/// spans every k-mer, minorant or not) and not a counter over
|
||||
/// non-monomorphic minorants alone. Conflating any of these with
|
||||
/// `family_idx` silently drifts apart as soon as the layer contains a
|
||||
/// monomorphic minorant — i.e. almost immediately, since ~98% of minorants
|
||||
/// are monomorphic — so the two counters below are deliberately kept
|
||||
/// separate: `family_idx` tracks every minorant (matching
|
||||
/// `scan_layer_families`), `seen` tracks only the non-monomorphic ones
|
||||
/// (what Algorithm R actually samples over).
|
||||
fn reservoir_sample_layer(layer_dir: &Path, n_layer: usize) -> OKIResult<HashSet<usize>> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?;
|
||||
let mut reservoir: Vec<usize> = Vec::with_capacity(n_layer);
|
||||
let mut seen: u64 = 0;
|
||||
let mut family_idx: usize = 0;
|
||||
let mut rng = rand::rng();
|
||||
for slot in 0..annex.len() {
|
||||
let Some(mask) = annex.get(slot) else { continue };
|
||||
if !mask.is_minorant() {
|
||||
continue; // not a minorant at all — doesn't advance `family_idx` either
|
||||
}
|
||||
let this_family_idx = family_idx;
|
||||
family_idx += 1;
|
||||
if mask.siblings() < 1 {
|
||||
continue; // monomorphic minorant — consumed a family_idx above, but not sample-eligible
|
||||
}
|
||||
if reservoir.len() < n_layer {
|
||||
reservoir.push(this_family_idx);
|
||||
} else {
|
||||
let j = rng.random_range(0..=seen);
|
||||
if j < n_layer as u64 {
|
||||
reservoir[j as usize] = this_family_idx;
|
||||
}
|
||||
}
|
||||
seen += 1;
|
||||
}
|
||||
Ok(reservoir.into_iter().collect())
|
||||
}
|
||||
|
||||
/// Every non-monomorphic minorant's `family_idx`, for one layer — not a
|
||||
/// sample, the *complete* set. Structural, annex-only, no cross-partition
|
||||
/// resolution (same discipline as [`reservoir_sample_layer`], just keeping
|
||||
/// everything instead of a bounded reservoir). Used to bound
|
||||
/// [`super::entropy::ensure_entropy_annexes`]'s `scan_layer_families` call
|
||||
/// to a `Selection::Some` of exactly the families that need their entropy
|
||||
/// computed, instead of `Selection::All` — monomorphic minorants (~98% of
|
||||
/// them, measured) are known to be ineligible straight from the annex bits
|
||||
/// already, with no need to pay `scan_layer_families`'s expensive
|
||||
/// per-genome resolution (`fill_sub_matrix_carries`) only to discard the
|
||||
/// result once `family_size() < 2` is checked *after* resolving.
|
||||
pub(super) fn non_monomorphic_selection_layer(layer_dir: &Path) -> OKIResult<HashSet<usize>> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?;
|
||||
let mut selected = HashSet::new();
|
||||
let mut family_idx: usize = 0;
|
||||
for slot in 0..annex.len() {
|
||||
let Some(mask) = annex.get(slot) else { continue };
|
||||
if !mask.is_minorant() {
|
||||
continue; // doesn't advance `family_idx` either
|
||||
}
|
||||
let this_family_idx = family_idx;
|
||||
family_idx += 1;
|
||||
if is_non_monomorphic_minorant(mask) {
|
||||
selected.insert(this_family_idx);
|
||||
}
|
||||
}
|
||||
Ok(selected)
|
||||
}
|
||||
|
||||
/// Total minorant count per layer (`is_minorant()`, monomorphic included)
|
||||
/// — sizes the entropy annex (`entropy_annex.rs`), which needs exactly one
|
||||
/// entry per minorant to keep its `family_idx` numbering aligned with
|
||||
/// every other consumer (`scan_layer_families`, `Selection`,
|
||||
/// `reservoir_sample_layer`'s own `family_idx` counter above).
|
||||
pub(super) fn minorant_counts(layer_dirs: &[PathBuf]) -> OKIResult<Vec<u64>> {
|
||||
layer_dirs
|
||||
.par_iter()
|
||||
.map(|layer_dir| -> OKIResult<u64> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?;
|
||||
let mut count = 0u64;
|
||||
for slot in 0..annex.len() {
|
||||
if annex.get(slot).is_some_and(|m| m.is_minorant()) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Entropy-biased alternative to [`reservoir_sample_layer`]: an
|
||||
/// independent accept/reject draw per non-monomorphic minorant, probability
|
||||
/// `p0 * w(entropy)` where `w(entropy) = exp(-(entropy-mu)^2 / (2*sigma^2))`
|
||||
/// (the unnormalised Gaussian kernel — `1` exactly at `entropy == mu`,
|
||||
/// smoothly decaying away from it, no hard cutoff) and `p0` is the same
|
||||
/// uniform rate the plain reservoir targets in expectation (`n / total_count`
|
||||
/// index-wide, or `1.0` when there is no size target at all — `--entropy`
|
||||
/// given without `--subsample`, a pure soft entropy filter over the whole
|
||||
/// index). See `DevDocMD/architecture/siblings.md`, "Entropy-biased
|
||||
/// selection", for why this is a single streaming pass with no
|
||||
/// cross-partition resolution: entropy is read from the already-built
|
||||
/// entropy annex (`ensure_entropy_annexes`), not recomputed here.
|
||||
fn entropy_biased_sample_layer(layer_dir: &Path, p0: f64, bias: EntropyBias) -> OKIResult<HashSet<usize>> {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?;
|
||||
let entropy_annex = EntropyAnnex::open(&layer_dir.join(ENTROPY_ANNEX_FILE_NAME)).map_err(OKIError::Io)?;
|
||||
let mut selected = HashSet::new();
|
||||
let mut family_idx: usize = 0;
|
||||
let mut rng = rand::rng();
|
||||
for slot in 0..annex.len() {
|
||||
let Some(mask) = annex.get(slot) else { continue };
|
||||
if !mask.is_minorant() {
|
||||
continue; // not a minorant at all — doesn't advance `family_idx` either
|
||||
}
|
||||
let this_family_idx = family_idx;
|
||||
family_idx += 1;
|
||||
debug_assert!(
|
||||
this_family_idx < entropy_annex.len(),
|
||||
"entropy annex ({}) shorter than this layer's minorant count — built against a different annex?",
|
||||
entropy_annex.len(),
|
||||
);
|
||||
if mask.siblings() < 1 {
|
||||
continue; // monomorphic minorant — no entropy, never eligible
|
||||
}
|
||||
let Some(entropy) = entropy_annex.get(this_family_idx) else { continue };
|
||||
let d = (entropy as f64 - bias.mu) / bias.sigma;
|
||||
let w = (-0.5 * d * d).exp();
|
||||
if rng.random::<f64>() < p0 * w {
|
||||
selected.insert(this_family_idx);
|
||||
}
|
||||
}
|
||||
Ok(selected)
|
||||
}
|
||||
|
||||
/// Per-layer sampling outcome — `None` at a given layer means "keep every
|
||||
/// non-monomorphic minorant of this layer" (either the `total_count <= n`
|
||||
/// case, or a layer whose entire non-monomorphic set was already smaller
|
||||
/// than its computed share and got fully absorbed by rounding).
|
||||
pub(super) type LayerSelection = Option<HashSet<usize>>;
|
||||
|
||||
/// Full entry point for every caller (`snp_pseudo_alignment`, the Shannon
|
||||
/// entropy report, `sankoff_bundle`): `n = None` and `entropy_bias = None`
|
||||
/// means "no `--subsample`, no `--entropy`/`--entropy-sd`", i.e. keep every
|
||||
/// non-monomorphic minorant of every layer — skips steps 1 and 2 entirely,
|
||||
/// since there is nothing to proportion.
|
||||
///
|
||||
/// `entropy_bias = Some(_)` switches selection from the plain uniform
|
||||
/// reservoir to [`entropy_biased_sample_layer`] — see
|
||||
/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". `index` is
|
||||
/// only used on that path, to build any missing per-layer entropy annex
|
||||
/// first ([`super::entropy::ensure_entropy_annexes`]) — a one-time cost,
|
||||
/// not paid again once every layer's annex file exists. The target size
|
||||
/// `n` still proportions the *expected* rate across the whole index (a
|
||||
/// single `p0 = n / total_count` applied uniformly at every layer already
|
||||
/// yields each layer its proportional share, with no need to compute a
|
||||
/// per-layer target the way [`sample_layers`] does for the uniform
|
||||
/// reservoir) — but unlike the uniform reservoir, the result is
|
||||
/// approximate, not exact (see [`entropy_biased_sample_layer`]'s docs).
|
||||
pub(super) fn compute_selections(
|
||||
index: &KmerIndex,
|
||||
layer_dirs: &[PathBuf],
|
||||
n: Option<usize>,
|
||||
entropy_bias: Option<EntropyBias>,
|
||||
) -> OKIResult<Vec<LayerSelection>> {
|
||||
let Some(bias) = entropy_bias else {
|
||||
return match n {
|
||||
None => Ok(vec![None; layer_dirs.len()]),
|
||||
Some(n) => {
|
||||
let counts = non_monomorphic_counts(layer_dirs)?;
|
||||
sample_layers(layer_dirs, &counts, n)
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
super::entropy::ensure_entropy_annexes(index, layer_dirs)?;
|
||||
let p0 = match n {
|
||||
None => 1.0,
|
||||
Some(n) => {
|
||||
let counts = non_monomorphic_counts(layer_dirs)?;
|
||||
let total_count: u64 = counts.iter().sum();
|
||||
if total_count == 0 { 0.0 } else { (n as f64 / total_count as f64).min(1.0) }
|
||||
}
|
||||
};
|
||||
layer_dirs
|
||||
.par_iter()
|
||||
.map(|layer_dir| -> OKIResult<LayerSelection> {
|
||||
Ok(Some(entropy_biased_sample_layer(layer_dir, p0, bias)?))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Step 2 entry point: `n` is the total number of families requested across
|
||||
/// the whole index (`--subsample N`); `layer_dirs` and `counts` must be the
|
||||
/// same length and in the same order as returned by
|
||||
/// [`super::family_scan::sibling_layer_dirs`] /
|
||||
/// [`non_monomorphic_counts`].
|
||||
fn sample_layers(layer_dirs: &[PathBuf], counts: &[u64], n: usize) -> OKIResult<Vec<LayerSelection>> {
|
||||
let total_count: u64 = counts.iter().sum();
|
||||
|
||||
if total_count <= n as u64 {
|
||||
// Fewer non-monomorphic minorants in the whole index than requested
|
||||
// — no sampling needed, keep everything everywhere.
|
||||
return Ok(vec![None; layer_dirs.len()]);
|
||||
}
|
||||
|
||||
layer_dirs
|
||||
.par_iter()
|
||||
.zip(counts.par_iter())
|
||||
.map(|(layer_dir, &count_layer)| -> OKIResult<LayerSelection> {
|
||||
if count_layer == 0 {
|
||||
return Ok(Some(HashSet::new()));
|
||||
}
|
||||
let n_layer = ((n as u128 * count_layer as u128) / total_count as u128) as usize;
|
||||
if n_layer == 0 {
|
||||
return Ok(Some(HashSet::new()));
|
||||
}
|
||||
let selected = reservoir_sample_layer(layer_dir, n_layer)?;
|
||||
Ok(Some(selected))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -1,557 +0,0 @@
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use obikseq::{CanonicalKmer, Kmer, Sequence};
|
||||
use obikindex::layer::MphfLayer;
|
||||
use obisys::Reporter;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use obikalgorithm::Algorithm;
|
||||
use obikindexer::algorithms::counter::Counter;
|
||||
use obikindexer::algorithms::dereplicator::Dereplicator;
|
||||
use obikindexer::algorithms::layer_builder::LayerBuilder;
|
||||
use obikindexer::algorithms::partitionner::PartitionRouter;
|
||||
use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode};
|
||||
|
||||
use super::alignment::SnpAlignmentExt;
|
||||
use super::build::SiblingAnnexBuildExt;
|
||||
use super::cardinality::CardinalityExt;
|
||||
use super::distance::DistanceExt;
|
||||
use super::entropy::ShannonEntropyExt;
|
||||
use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME};
|
||||
use super::helpers::is_minorant;
|
||||
use super::sankoff_bundle::SankoffBundleExt;
|
||||
use super::stats::SiblingStatsExt;
|
||||
use super::subsample::EntropyBias;
|
||||
use super::{FamilyMask, SiblingAnnex, ANNEX_FILE_NAME};
|
||||
|
||||
// k must be >= 11 (project constraint, "k ∈ [11,31]"); k=11, level_max=1,
|
||||
// theta=0.0 mirror `obiskbuilder`'s own tests (smaller k/level_max
|
||||
// combinations trip an unrelated pre-existing bug in `obikentropy`'s
|
||||
// sliding-window ring buffer — not this feature's concern).
|
||||
const K: usize = 11;
|
||||
const M: usize = 5;
|
||||
|
||||
/// Build a single-genome index from one in-memory FASTA sequence, driving
|
||||
/// the same primitives `obikmer`'s `scatter` step uses (minus the
|
||||
/// multi-file `obipipeline` wrapper — a single sequence needs none of
|
||||
/// that): normalise -> build superkmers -> route -> write.
|
||||
/// `cargo test` doesn't install a `tracing` subscriber the way `obikmer`'s
|
||||
/// CLI does, so `debug!`/etc. are silent no-ops by default — including the
|
||||
/// `PartitionRunner` instrumentation that would matter most for
|
||||
/// re-diagnosing a hang here. `try_init` is idempotent across concurrently
|
||||
/// running tests (later calls just find a subscriber already installed).
|
||||
fn init_tracing() {
|
||||
let _ = tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.with_writer(std::io::stderr)
|
||||
.try_init();
|
||||
}
|
||||
|
||||
fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex {
|
||||
init_tracing();
|
||||
let fasta_path = dir.join(format!("{label}.fasta"));
|
||||
let mut f = std::fs::File::create(&fasta_path).unwrap();
|
||||
writeln!(f, ">{label}").unwrap();
|
||||
f.write_all(seq).unwrap();
|
||||
writeln!(f).unwrap();
|
||||
drop(f);
|
||||
|
||||
let index_path = dir.join(format!("{label}.idx"));
|
||||
let config = IndexConfig {
|
||||
kmer_size: K,
|
||||
minimizer_size: M,
|
||||
n_bits: 0, // 1 partition — keeps the test deterministic and simple
|
||||
with_counts: false,
|
||||
evidence: obikindex::layer::IndexMode::Exact,
|
||||
block_bits: 0,
|
||||
};
|
||||
let idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label)))
|
||||
.expect("create");
|
||||
|
||||
let stream = obiread::open_nuc_stream(fasta_path.to_str().unwrap(), K).expect("open fasta");
|
||||
let mut router = PartitionRouter::new(&idx);
|
||||
for page in stream {
|
||||
let batch = obiskbuilder::build_superkmers_page(page, K, /* level_max */ 1, /* theta */ 0.0);
|
||||
router.write_batch(batch).expect("write_batch");
|
||||
}
|
||||
router.close().expect("close partition writers"); // also marks scatter done
|
||||
drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope
|
||||
|
||||
Dereplicator::new(&idx).run().expect("dereplicate");
|
||||
Counter::new(&idx).run().expect("count_kmer"); // also writes the spectrum + marks counted
|
||||
LayerBuilder::new(&idx)
|
||||
.run()
|
||||
.expect("build_layers"); // also marks indexed
|
||||
idx
|
||||
}
|
||||
|
||||
fn canonical(ascii: &[u8]) -> CanonicalKmer {
|
||||
Kmer::from_ascii(ascii).unwrap().canonical()
|
||||
}
|
||||
|
||||
/// Read back the annex entry for a given canonical k-mer from the merged
|
||||
/// index's (single) partition/layer, asserting it was found at all.
|
||||
fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask {
|
||||
let meta = idx.partition_meta(0).unwrap();
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = idx.layer_dir(0, l);
|
||||
let mphf = MphfLayer::open(&layer_dir).unwrap();
|
||||
if let Some(slot) = mphf.find(kmer) {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).unwrap();
|
||||
return annex.get(slot).expect("slot must have a computed annex entry");
|
||||
}
|
||||
}
|
||||
panic!("kmer not found in any layer of partition 0");
|
||||
}
|
||||
|
||||
fn merge_two(dir: &Path, g1: &KmerIndex, g2: &KmerIndex) -> KmerIndex {
|
||||
let mut rep = Reporter::new();
|
||||
KmerIndex::merge(
|
||||
&dir.join("merged.idx"),
|
||||
&[g1, g2],
|
||||
MergeMode::Presence,
|
||||
false,
|
||||
false,
|
||||
1.0,
|
||||
&mut rep,
|
||||
)
|
||||
.expect("merge")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_annex_one_sibling_each() {
|
||||
// k=11, centre = index 5 (0-based). Two genomes, each exactly one
|
||||
// k-mer, sharing every base except the centre:
|
||||
// g1 = "AACCGCTTAAG" (centre 'C', base index 1)
|
||||
// g2 = "AACCGGTTAAG" (centre 'G', base index 2)
|
||||
// Hand-verified: both stay forward-oriented under canonicalisation
|
||||
// (each is lexicographically smaller than its own reverse
|
||||
// complement, since both start with "AA"), and raw(g1) < raw(g2)
|
||||
// (only differing base: C=0b01 < G=0b10 at the centre) — so g1 is
|
||||
// the minorant, g2 is not. The mask is a family-wide value: both
|
||||
// slots must read back the *same* presence bits (1 and 2 set) — but
|
||||
// only g1's slot should carry the stored minorant flag.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let g1_kmer = canonical(b"AACCGCTTAAG");
|
||||
let g2_kmer = canonical(b"AACCGGTTAAG");
|
||||
let expected_mask = FamilyMask::EMPTY.with(1).with(2);
|
||||
|
||||
let a = annex_info_for(&merged, g1_kmer);
|
||||
assert_eq!(a.bits(), expected_mask.bits(), "AACCGCTTAAG");
|
||||
assert_eq!(a.siblings(), 1);
|
||||
assert!(is_minorant(g1_kmer, a, K), "g1 should be the minorant");
|
||||
assert!(a.is_minorant(), "g1's stored minorant flag should be set at build time");
|
||||
|
||||
let b = annex_info_for(&merged, g2_kmer);
|
||||
assert_eq!(b.bits(), expected_mask.bits(), "AACCGGTTAAG");
|
||||
assert_eq!(b.siblings(), 1);
|
||||
assert!(!is_minorant(g2_kmer, b, K), "g2 should not be the minorant");
|
||||
assert!(!b.is_minorant(), "g2's stored minorant flag should not be set");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_annex_works_after_pack_sparse() {
|
||||
// Same fixture as `sibling_annex_one_sibling_each`, but with
|
||||
// `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged
|
||||
// index before building the sibling annex — proves `PartitionCache`'s
|
||||
// sparse-detection (`Mat::SparsePresence`, gated on `presence/
|
||||
// is_multi.prsb`) and the generic `TypedLayer<D>` methods it relies on
|
||||
// actually round-trip through the real build pipeline, not just the
|
||||
// unit-level `TypedLayer<PersistentSparseBitMatrix>` tests in
|
||||
// `obikindex::layer`.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.pack_matrices(true).expect("pack_matrices(sparse)");
|
||||
|
||||
let index_dir = merged.index_dir(0);
|
||||
assert!(
|
||||
index_dir.join("layer_0").join("presence").join("is_multi.prsb").exists(),
|
||||
"pack_matrices(true) must leave the sparse marker file behind"
|
||||
);
|
||||
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let g1_kmer = canonical(b"AACCGCTTAAG");
|
||||
let g2_kmer = canonical(b"AACCGGTTAAG");
|
||||
let expected_mask = FamilyMask::EMPTY.with(1).with(2);
|
||||
|
||||
let a = annex_info_for(&merged, g1_kmer);
|
||||
assert_eq!(a.bits(), expected_mask.bits(), "AACCGCTTAAG");
|
||||
assert_eq!(a.siblings(), 1);
|
||||
assert!(is_minorant(g1_kmer, a, K), "g1 should be the minorant");
|
||||
assert!(a.is_minorant(), "g1's stored minorant flag should be set at build time");
|
||||
|
||||
let b = annex_info_for(&merged, g2_kmer);
|
||||
assert_eq!(b.bits(), expected_mask.bits(), "AACCGGTTAAG");
|
||||
assert_eq!(b.siblings(), 1);
|
||||
assert!(!is_minorant(g2_kmer, b, K), "g2 should not be the minorant");
|
||||
assert!(!b.is_minorant(), "g2's stored minorant flag should not be set");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_annex_zero_siblings_when_identical_and_no_variant_exists() {
|
||||
// Same k-mer in both genomes, no other genome around to carry a
|
||||
// variant -> 0 siblings, trivially its own minorant.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"GATTACAGATC");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"GATTACAGATC");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let kmer = canonical(b"GATTACAGATC");
|
||||
let mask = annex_info_for(&merged, kmer);
|
||||
assert_eq!(mask.siblings(), 0, "GATTACAGATC");
|
||||
assert_eq!(mask.family_size(), 1);
|
||||
assert!(is_minorant(kmer, mask, K));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_annex_stats_counts_each_family_once_and_per_genome() {
|
||||
// Reuses the one-sibling-each fixture: a single family of size 2
|
||||
// (g1's centre-C form + g2's centre-G form), each genome carrying
|
||||
// exactly one of the two members. Stats must report exactly one
|
||||
// family of size 2 (`counts[1] == 1`, since index 1 = size 2), not
|
||||
// two (which naively summing both slots would give), and both
|
||||
// genomes represented at size 2, neither at any other size.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let stats = merged.sibling_annex_stats().expect("sibling_annex_stats");
|
||||
|
||||
assert_eq!(stats.counts, [0, 1, 0, 0], "one family of size 2, counted once");
|
||||
assert_eq!(stats.per_genome.len(), 2);
|
||||
for g in 0..2 {
|
||||
assert_eq!(
|
||||
stats.per_genome[g], [0, 1, 0, 0],
|
||||
"genome {g} should represent exactly one size-2 family"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_family_size_histogram_matches_full_stats_global_counts() {
|
||||
// Same fixture as `sibling_annex_stats_counts_each_family_once_and_per_genome`
|
||||
// — the cheap, annex-only histogram must agree with the `counts` half of
|
||||
// the full (cross-partition) stats pass, without needing a `PartitionCache`
|
||||
// at all.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let histogram = merged.sibling_family_size_histogram().expect("sibling_family_size_histogram");
|
||||
assert_eq!(histogram, [0, 1, 0, 0], "one family of size 2, counted once");
|
||||
|
||||
let stats = merged.sibling_annex_stats().expect("sibling_annex_stats");
|
||||
assert_eq!(histogram, stats.counts, "must agree with the full stats pass's global counts");
|
||||
}
|
||||
|
||||
/// Exercises the four sibling-annex consumers that were rewritten to share
|
||||
/// `family_scan::scan_layer_families` (partition-grouped lookups instead of
|
||||
/// one lookup per family) — same one-sibling-each fixture as the tests
|
||||
/// above (g1 carries the family's centre-C member, g2 the centre-G one),
|
||||
/// hand-verified expected output for each.
|
||||
#[test]
|
||||
fn family_scan_consumers_agree_on_one_sibling_each() {
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
// Merge doesn't promise to preserve source order, so resolve each
|
||||
// genome's index by label rather than assuming g1 -> 0, g2 -> 1.
|
||||
let idx_of = |label: &str| merged.meta().genomes().unwrap().iter().position(|g| g.label == label).unwrap();
|
||||
let (i1, i2) = (idx_of("g1"), idx_of("g2"));
|
||||
|
||||
// snp_pseudo_alignment: one variable family, one column — g1's row
|
||||
// reads 'C' (its own member), g2's reads 'G'.
|
||||
let alignment = merged.snp_pseudo_alignment(None, None).expect("snp_pseudo_alignment");
|
||||
assert_eq!(alignment.sequences[i1], vec![b'C']);
|
||||
assert_eq!(alignment.sequences[i2], vec![b'G']);
|
||||
|
||||
// raw_snp_distance: g1's single form (C) != g2's (G) at the family's
|
||||
// one eligible locus -> a SNP, not a shared site.
|
||||
let raw = merged.raw_snp_distance().expect("raw_snp_distance");
|
||||
assert_eq!(raw.snp[[i1, i2]], 1);
|
||||
assert_eq!(raw.snp[[i2, i1]], 1);
|
||||
assert_eq!(raw.shared[[i1, i2]], 0);
|
||||
assert_eq!(raw.shared[[i2, i1]], 0);
|
||||
|
||||
// cardinality_tally: both genomes carry exactly one member of the
|
||||
// family (cardinality 1 each) -> one co-occurrence at [1][1].
|
||||
let cardinality = merged.cardinality_tally(&raw, 1.0).expect("cardinality_tally");
|
||||
assert_eq!(cardinality.counts[1][1], 1);
|
||||
let total: u64 = cardinality.counts.iter().flatten().sum();
|
||||
assert_eq!(total, 1, "no other cardinality pair should be tallied");
|
||||
|
||||
// base_pair_tally: the one eligible, differing locus is C (base 1) vs
|
||||
// G (base 2).
|
||||
let base_pairs = merged.base_pair_tally(&raw, 1.0).expect("base_pair_tally");
|
||||
assert_eq!(base_pairs.counts[1][2], 1);
|
||||
assert_eq!(base_pairs.counts[2][1], 1);
|
||||
let total: u64 = base_pairs.counts.iter().flatten().sum();
|
||||
assert_eq!(total, 2, "no other base pair should be tallied");
|
||||
assert_eq!(base_pairs.same, [0, 0, 0, 0], "the two genomes never agree at this locus");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_annex_no_empty_masks_after_build() {
|
||||
let dir = tempdir().unwrap();
|
||||
let seq = b"ACGTACGTACGT".repeat(500);
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", &seq);
|
||||
g1.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let n_layers = g1.n_layers(0).expect("partition meta");
|
||||
for l in 0..n_layers {
|
||||
let layer_dir = g1.layer_dir(0, l);
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open");
|
||||
for slot in 0..annex.len() {
|
||||
let mask = annex.get(slot).expect("slot must have an entry");
|
||||
assert!(
|
||||
mask.family_size() >= 1,
|
||||
"layer {l} slot {slot}: empty family mask (bits={:04b}) after build — \
|
||||
indicates batch-offset bug or unseeded mask slot",
|
||||
mask.bits()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sibling_histogram_does_not_panic_on_partial_last_batch() {
|
||||
// Regression test for the bug where enumerate_kmers_batch().enumerate()
|
||||
// used the batch index instead of the sequence offset, leaving the
|
||||
// trailing slots of a non-multiple-of-BATCH_SIZE layer unseeded and
|
||||
// producing the minorant-only sentinel (0x10) that caused
|
||||
// sibling_family_size_histogram to panic with index u32::MAX.
|
||||
let dir = tempdir().unwrap();
|
||||
let seq = b"ACGTACGTACGT".repeat(500);
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", &seq);
|
||||
g1.build_sibling_annex().expect("build_sibling_annex");
|
||||
let hist = g1.sibling_family_size_histogram().expect("sibling_family_size_histogram");
|
||||
let total: u64 = hist.iter().sum();
|
||||
assert!(total > 0, "histogram should contain at least one family");
|
||||
}
|
||||
|
||||
/// `sibling_annex_one_sibling_each`'s fixture happens to put g2's k-mer in
|
||||
/// layer 0 and g1's in layer 1 (a fresh single-genome index, then one
|
||||
/// merge that adds exactly one new layer for content absent from the
|
||||
/// first — confirmed empirically, not assumed) — a real two-layer index,
|
||||
/// not a contrived one, so it exercises both write paths
|
||||
/// `build_layer_sibling_annex` now has to get right: the seeding pass (a
|
||||
/// slot's own base gets *this* layer's index directly) and the
|
||||
/// cross-partition/cross-layer resolution pass (`PartitionCache::find`
|
||||
/// reporting which layer the other family member actually lives in, not
|
||||
/// just that it exists).
|
||||
#[test]
|
||||
fn sibling_annex_records_the_real_layer_of_each_family_member() {
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let n_layers = merged.n_layers(0).unwrap();
|
||||
assert_eq!(n_layers, 2, "fixture assumption: one merge, one new layer");
|
||||
|
||||
let g1_kmer = canonical(b"AACCGCTTAAG"); // own base C (1), sibling base G (2) — lives in layer 1
|
||||
let g2_kmer = canonical(b"AACCGGTTAAG"); // own base G (2), sibling base C (1) — lives in layer 0
|
||||
|
||||
let a = annex_info_for(&merged, g1_kmer);
|
||||
assert_eq!(a.layer_value(1), Some(1), "g1's own base: seeded with its own layer (1)");
|
||||
assert_eq!(a.layer_value(2), Some(0), "g1's sibling (g2's form): resolved to its real layer (0)");
|
||||
|
||||
let b = annex_info_for(&merged, g2_kmer);
|
||||
assert_eq!(b.layer_value(2), Some(0), "g2's own base: seeded with its own layer (0)");
|
||||
assert_eq!(b.layer_value(1), Some(1), "g2's sibling (g1's form): resolved to its real layer (1)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn subsample_and_shannon_on_one_variable_family() {
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let idx_of = |label: &str| merged.meta().genomes().unwrap().iter().position(|g| g.label == label).unwrap();
|
||||
let (i1, i2) = (idx_of("g1"), idx_of("g2"));
|
||||
|
||||
// This fixture has exactly one non-monomorphic family (see
|
||||
// `family_scan_consumers_agree_on_one_sibling_each`) — total_count = 1.
|
||||
// Asking for far more than that must fall back to "keep everything",
|
||||
// identical to the unsampled (`None`) result.
|
||||
let sampled = merged.snp_pseudo_alignment(Some(1000), None).expect("snp_pseudo_alignment (subsample >> total)");
|
||||
assert_eq!(sampled.sequences[i1], vec![b'C']);
|
||||
assert_eq!(sampled.sequences[i2], vec![b'G']);
|
||||
|
||||
// Asking for 0 must keep nothing — an empty (but still one-row-per-genome)
|
||||
// alignment, not an error.
|
||||
let empty = merged.snp_pseudo_alignment(Some(0), None).expect("snp_pseudo_alignment (subsample 0)");
|
||||
assert!(empty.sequences[i1].is_empty());
|
||||
assert!(empty.sequences[i2].is_empty());
|
||||
|
||||
// shannon_entropy_csv: one row, both entropy15 and entropy4 = 1 bit
|
||||
// exactly (2 genomes present, each carrying exactly one distinct base,
|
||||
// uniform split -> -2*(0.5*log2(0.5)) = 1 — the two definitions
|
||||
// coincide here since no genome carries more than one base).
|
||||
let csv_path = dir.path().join("shannon.csv");
|
||||
merged.shannon_entropy_csv(&csv_path, Some(1000), None).expect("shannon_entropy_csv");
|
||||
let csv = std::fs::read_to_string(&csv_path).unwrap();
|
||||
let mut lines = csv.lines();
|
||||
assert_eq!(lines.next(), Some("layer,family_idx,entropy15,entropy4,family_size,n_genomes_present"));
|
||||
let row = lines.next().expect("exactly one data row");
|
||||
assert!(lines.next().is_none(), "exactly one non-monomorphic family in this fixture");
|
||||
let fields: Vec<&str> = row.split(',').collect();
|
||||
let entropy15: f64 = fields[2].parse().unwrap();
|
||||
let entropy4: f64 = fields[3].parse().unwrap();
|
||||
assert!((entropy15 - 1.0).abs() < 1e-6, "entropy15 should be exactly 1 bit, got {entropy15}");
|
||||
assert!((entropy4 - 1.0).abs() < 1e-6, "entropy4 should be exactly 1 bit, got {entropy4}");
|
||||
assert_eq!(fields[4], "2", "family_size");
|
||||
assert_eq!(fields[5], "2", "n_genomes_present");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sankoff_bundle_matches_old_separate_calls() {
|
||||
// Same fixture as `sibling_annex_one_sibling_each`/
|
||||
// `subsample_and_shannon_on_one_variable_family`: one non-monomorphic
|
||||
// family, g1='C' (minorant), g2='G'. Regression check that fusing
|
||||
// raw_snp_distance/base_pair_tally/cardinality_tally/snp_pseudo_alignment
|
||||
// into `sankoff_bundle`'s two passes produces identical results to the
|
||||
// old four independent calls on `Selection::All` (no `--subsample`,
|
||||
// no `--entropy`) — see `DevDocMD/architecture/siblings.md`, "Wired into
|
||||
// `pack`...".
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let n_genomes = merged.meta().genomes().unwrap().len();
|
||||
let exclude_mask = vec![false; n_genomes];
|
||||
let ratio_ceiling = 0.5;
|
||||
|
||||
let expected_raw = merged.raw_snp_distance().expect("raw_snp_distance");
|
||||
let expected_base = merged.base_pair_tally(&expected_raw, ratio_ceiling).expect("base_pair_tally");
|
||||
let expected_card = merged.cardinality_tally(&expected_raw, ratio_ceiling).expect("cardinality_tally");
|
||||
let expected_alignment = merged.snp_pseudo_alignment(None, None).expect("snp_pseudo_alignment");
|
||||
|
||||
let bundle = merged.sankoff_bundle(None, None, ratio_ceiling, &exclude_mask).expect("sankoff_bundle");
|
||||
|
||||
assert_eq!(bundle.raw.snp, expected_raw.snp);
|
||||
assert_eq!(bundle.raw.shared, expected_raw.shared);
|
||||
assert_eq!(bundle.base_pair_tally.counts, expected_base.counts);
|
||||
assert_eq!(bundle.base_pair_tally.same, expected_base.same);
|
||||
assert_eq!(bundle.cardinality_tally.counts, expected_card.counts);
|
||||
assert_eq!(bundle.alignment.sequences, expected_alignment.sequences);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base_pair_tally_accumulates_base_a_diagnostic() {
|
||||
// Diagnostic for a user-reported observation on real data:
|
||||
// `composition_transitions`'s row/column for base 'A' (index 0) was
|
||||
// *entirely* zero, including the diagonal (`same[0]`, "A stays A"),
|
||||
// despite 'A' appearing at ~6% frequency in the pseudo-alignment
|
||||
// itself. g1/g2 are identical (both central base 'A'), g3 differs
|
||||
// (central base 'C') at the same family — makes the family variable
|
||||
// (family_size=2) while g1/g2 form an eligible (ratio 0), ordinary
|
||||
// pair both resolving `single_form` to `Some(0)` = 'A'.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AAAAAAAAAAA");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AAAAAAAAAAA");
|
||||
let g3 = build_single_genome_index(dir.path(), "g3", b"AAAAACAAAAA");
|
||||
let mut rep = Reporter::new();
|
||||
let merged = KmerIndex::merge(
|
||||
&dir.path().join("merged.idx"),
|
||||
&[&g1, &g2, &g3],
|
||||
MergeMode::Presence,
|
||||
false,
|
||||
false,
|
||||
1.0,
|
||||
&mut rep,
|
||||
).expect("merge");
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let n_genomes = merged.meta().genomes().unwrap().len();
|
||||
let exclude_mask = vec![false; n_genomes];
|
||||
let bundle = merged.sankoff_bundle(None, None, 0.5, &exclude_mask).expect("sankoff_bundle");
|
||||
|
||||
assert!(
|
||||
bundle.base_pair_tally.same[0] >= 1,
|
||||
"base 'A' (index 0) must accumulate in BasePairTally.same for an identical, included g1/g2 \
|
||||
pair both resolving to 'A' — got same={:?}, counts={:?}",
|
||||
bundle.base_pair_tally.same, bundle.base_pair_tally.counts,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_annex_builds_on_demand_and_biases_selection() {
|
||||
// Same fixture, same single non-monomorphic family, known entropy15 =
|
||||
// 1.0 bit exactly (see `subsample_and_shannon_on_one_variable_family`).
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG");
|
||||
let merged = merge_two(dir.path(), &g1, &g2);
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(&merged).expect("layer dirs");
|
||||
for layer_dir in &layer_dirs {
|
||||
assert!(
|
||||
!layer_dir.join(ENTROPY_ANNEX_FILE_NAME).exists(),
|
||||
"entropy annex must not exist before any entropy-biased selection is requested"
|
||||
);
|
||||
}
|
||||
|
||||
// This fixture spans 2 layers (a single-genome bootstrap layer plus the
|
||||
// merged one, see `merge_two`) — the one non-monomorphic family lives
|
||||
// in whichever layer actually got a non-empty selection, not
|
||||
// necessarily `layer_dirs[0]`.
|
||||
let find_layer = |selections: &[super::subsample::LayerSelection]| {
|
||||
selections.iter().position(|s| matches!(s, Some(set) if !set.is_empty()))
|
||||
};
|
||||
|
||||
// mu == the family's real entropy, tiny sigma -> weight == 1 exactly,
|
||||
// p0 == 1.0 (no `--subsample`) -> the family is selected with
|
||||
// probability 1 (up to a measure-zero event on the uniform draw).
|
||||
let bias_center = EntropyBias { mu: 1.0, sigma: 0.5 };
|
||||
let selections = super::subsample::compute_selections(&merged, &layer_dirs, None, Some(bias_center))
|
||||
.expect("compute_selections (entropy-biased)");
|
||||
for layer_dir in &layer_dirs {
|
||||
assert!(
|
||||
layer_dir.join(ENTROPY_ANNEX_FILE_NAME).exists(),
|
||||
"compute_selections must build the entropy annex for every layer on first use"
|
||||
);
|
||||
}
|
||||
let li = find_layer(&selections).expect("mu at the true entropy must select the one non-monomorphic family, in some layer");
|
||||
assert_eq!(selections[li], Some(std::collections::HashSet::from([0])));
|
||||
|
||||
// The annex now exists — read it back directly and check the stored
|
||||
// value matches `--shannon`'s own entropy15 computation exactly.
|
||||
let entropy_annex = EntropyAnnex::open(&layer_dirs[li].join(ENTROPY_ANNEX_FILE_NAME)).expect("open entropy annex");
|
||||
let stored = entropy_annex.get(0).expect("family 0 has a real (non-sentinel) entropy value");
|
||||
assert!((stored as f64 - 1.0).abs() < 1e-6, "stored entropy should be 1 bit, got {stored}");
|
||||
|
||||
// mu far from any achievable entropy, tiny sigma -> weight underflows
|
||||
// to exactly 0.0 -> never selected, deterministically, in any layer.
|
||||
let bias_far = EntropyBias { mu: 100.0, sigma: 0.001 };
|
||||
let selections_far = super::subsample::compute_selections(&merged, &layer_dirs, None, Some(bias_far))
|
||||
.expect("compute_selections (entropy-biased, far)");
|
||||
assert!(find_layer(&selections_far).is_none(), "mu far from the true entropy must never select it, in any layer");
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user