Refactor indexing, sampling, and entropy documentation

This commit introduces new theoretical frameworks for k-mer indexing, pair coverage sampling, and sibling partition unification. It also implements new algorithms for calculating family entropy and statistical tests for overlap analysis. Documentation is updated across the board to reflect these new features and theoretical concepts.
This commit is contained in:
Eric Coissac committed 2026-10-13 08:54:21 +02:00
1 parent 95bbc0b2ae
commit 0d12494f1a
12 files changed
+1182 -56

No files matched your search

@@ -51,28 +51,25 @@ pub(crate) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> {
Some((h, present as usize))
}
/// Shannon entropy (bits) of one family's per-genome states, reduced to the
/// 4 plain nucleotide symbols instead of the 15-state space above — kept
/// alongside it (not instead of it) purely to measure how much the two
/// diverge on real data. A genome carrying several bases at once
/// (`genome_mask[g]` with more than one bit set) contributes to *each* of
/// those bases' counts — counted once per base present, not split
/// fractionally, and not folded into a combined state the way
/// [`family_entropy`] does. The denominator is therefore the total number
/// of base-occurrences summed over every present genome, not the genome
/// count. `None` under the same condition as [`family_entropy`].
/// Shannon entropy (bits) of one family's per-genome states over the 4 plain
/// nucleotide symbols, restricted to the genomes carrying exactly one member
/// of the family (`genome_mask[g].count_ones() == 1`) — the genomes that stay
/// informative under `--no-ambiguity`, where a genome carrying several
/// members (paralogues, not read ambiguity) becomes missing data. Genomes
/// where the family is absent (`genome_mask[g] == 0`) and genomes carrying
/// several members are both left out of the denominator, which is returned
/// as the second value (`n_genomes_unambiguous`). `None` if no genome carries
/// exactly one member.
pub(crate) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> {
let mut freq = [0u32; 4];
let mut total = 0u32;
let mut n_single = 0u32;
for &m in genome_mask {
for (base, count) in freq.iter_mut().enumerate() {
if m & (1 << base) != 0 {
*count += 1;
total += 1;
}
if m.count_ones() == 1 {
freq[m.trailing_zeros() as usize] += 1;
n_single += 1;
}
}
if total == 0 {
if n_single == 0 {
return None;
}
let mut h = 0.0f64;
@@ -80,10 +77,37 @@ pub(crate) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> {
if c == 0 {
continue;
}
let p = f64::from(c) / f64::from(total);
let p = f64::from(c) / f64::from(n_single);
h -= p * p.log2();
}
Some((h, total as usize))
Some((h, n_single as usize))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn entropy_4_balanced_two_states_is_one_bit() {
// two genomes carry base 0 (A), two carry base 2 (G); one is absent
let (h, n) = family_entropy_4(&[0b0001, 0b0001, 0b0100, 0b0100, 0]).unwrap();
assert!((h - 1.0).abs() < 1e-12);
assert_eq!(n, 4);
}
#[test]
fn entropy_4_ignores_genomes_with_several_members() {
// the two-member genomes (0b0011, 0b1100) are left out entirely
let (h, n) = family_entropy_4(&[0b0001, 0b0001, 0b0011, 0b1100]).unwrap();
assert_eq!(h, 0.0);
assert_eq!(n, 2);
}
#[test]
fn entropy_4_is_none_without_unambiguous_genome() {
assert!(family_entropy_4(&[0, 0b0011, 0b0110]).is_none());
assert!(family_entropy_4(&[]).is_none());
}
}
/// Builds one layer's entropy annex if it doesn't already exist, and/or
@@ -45,10 +45,13 @@ pub trait SiblingExt {
fn build_sibling_annex(&self) -> OKIResult<()>;
/// Write a CSV (`partition,layer,family_idx,entropy15,entropy4,
/// family_size,n_genomes_present`) of per-family Shannon entropy — both
/// [`family_entropy`] (15 non-empty states, the project's settled
/// definition) and [`family_entropy_4`] (plain 4-symbol reduction, kept
/// alongside for comparison) — one row per non-monomorphic minorant of
/// family_size,n_genomes_present,n_genomes_unambiguous`) of per-family
/// Shannon entropy — both [`family_entropy`] (15 non-empty states over
/// every genome carrying the family, the project's settled definition)
/// and [`family_entropy_4`] (4 plain bases over the genomes carrying
/// exactly one member only, i.e. what survives `--no-ambiguity`;
/// `n_genomes_unambiguous` is its denominator, `0` meaning `entropy4`
/// is not defined) — one row per non-monomorphic minorant of
/// every cached layer, from an already-built sibling annex (run
/// [`build_sibling_annex`](Self::build_sibling_annex) first).
///
@@ -166,7 +169,7 @@ impl SiblingExt for IndexCache {
let fast_mode = is_fast_mode(self);
let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?);
writeln!(f, "partition,layer,family_idx,entropy15,entropy4,family_size,n_genomes_present")
writeln!(f, "partition,layer,family_idx,entropy15,entropy4,family_size,n_genomes_present,n_genomes_unambiguous")
.map_err(OKIError::Io)?;
let pb = progress_bar("shannon_entropy", self.n_partition() as u64, "partitions");
@@ -205,10 +208,10 @@ impl SiblingExt for IndexCache {
let Some((h15, n_present)) = family_entropy(genome_mask) else {
return;
};
let h4 = family_entropy_4(genome_mask).map_or(0.0, |(h, _)| h);
let (h4, n_unambiguous) = family_entropy_4(genome_mask).unwrap_or((0.0, 0));
if let Err(e) = writeln!(
f,
"{part},{l},{family_idx},{h15:.6},{h4:.6},{},{n_present}",
"{part},{l},{family_idx},{h15:.6},{h4:.6},{},{n_present},{n_unambiguous}",
mask.family_size()
) {
write_err = Some(e);