feat(phylo): add --iqtree-min-freq to filter rare nucleotide states
Introduces --iqtree-min-freq (default 0.001) to treat low-frequency nucleotide states as missing data during IQ-TREE alignment generation when --free-loss is active. This triggers a recoding pass that folds rare states into the missing symbol, followed by non-informative site removal and alphabet recomputation to maintain output consistency. The change also adds Sankoff model configuration files and updates related tests and documentation.
This commit is contained in:
Eric Coissac
@@ -15,7 +15,7 @@ use super::cardinality::CardinalityExt;
|
||||
use super::distance::DistanceExt;
|
||||
use super::entropy::ShannonEntropyExt;
|
||||
use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME};
|
||||
use super::helpers::is_minorant;
|
||||
use super::helpers::{central_base, is_minorant};
|
||||
use super::sankoff_bundle::SankoffBundleExt;
|
||||
use super::stats::SiblingStatsExt;
|
||||
use super::subsample::EntropyBias;
|
||||
@@ -457,6 +457,44 @@ fn sankoff_bundle_matches_old_separate_calls() {
|
||||
assert_eq!(bundle.alignment.sequences, expected_alignment.sequences);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base_pair_tally_accumulates_base_a_diagnostic() {
|
||||
// Diagnostic for a user-reported observation on real data:
|
||||
// `composition_transitions`'s row/column for base 'A' (index 0) was
|
||||
// *entirely* zero, including the diagonal (`same[0]`, "A stays A"),
|
||||
// despite 'A' appearing at ~6% frequency in the pseudo-alignment
|
||||
// itself. g1/g2 are identical (both central base 'A'), g3 differs
|
||||
// (central base 'C') at the same family — makes the family variable
|
||||
// (family_size=2) while g1/g2 form an eligible (ratio 0), ordinary
|
||||
// pair both resolving `single_form` to `Some(0)` = 'A'.
|
||||
let dir = tempdir().unwrap();
|
||||
let g1 = build_single_genome_index(dir.path(), "g1", b"AAAAAAAAAAA");
|
||||
let g2 = build_single_genome_index(dir.path(), "g2", b"AAAAAAAAAAA");
|
||||
let g3 = build_single_genome_index(dir.path(), "g3", b"AAAAACAAAAA");
|
||||
let mut rep = Reporter::new();
|
||||
let merged = KmerIndex::merge(
|
||||
&dir.path().join("merged.idx"),
|
||||
&[&g1, &g2, &g3],
|
||||
MergeMode::Presence,
|
||||
false,
|
||||
false,
|
||||
1.0,
|
||||
&mut rep,
|
||||
).expect("merge");
|
||||
merged.build_sibling_annex().expect("build_sibling_annex");
|
||||
|
||||
let n_genomes = merged.meta().genomes.len();
|
||||
let exclude_mask = vec![false; n_genomes];
|
||||
let bundle = merged.sankoff_bundle(None, None, 0.5, &exclude_mask).expect("sankoff_bundle");
|
||||
|
||||
assert!(
|
||||
bundle.base_pair_tally.same[0] >= 1,
|
||||
"base 'A' (index 0) must accumulate in BasePairTally.same for an identical, included g1/g2 \
|
||||
pair both resolving to 'A' — got same={:?}, counts={:?}",
|
||||
bundle.base_pair_tally.same, bundle.base_pair_tally.counts,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entropy_annex_builds_on_demand_and_biases_selection() {
|
||||
// Same fixture, same single non-monomorphic family, known entropy15 =
|
||||
@@ -537,6 +575,132 @@ fn diag_real_index_layer_distribution() {
|
||||
println!("top10 counts: {:?}", &sorted_counts[..10.min(sorted_counts.len())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn diag_real_index_base_a_representation() {
|
||||
// Diagnostic for a user-reported observation, reproduced on two
|
||||
// unrelated real datasets (a plant and this bacterial index):
|
||||
// `composition_transitions`'s row/column for base 'A' is entirely
|
||||
// zero, even though 'A' appears at real (~6-22%) frequency in the
|
||||
// pseudo-alignment itself. A minimal synthetic reproduction
|
||||
// (`base_pair_tally_accumulates_base_a_diagnostic`) showed the
|
||||
// pairwise tally mechanism itself works correctly for base A in
|
||||
// isolation — so this checks one level lower, purely structural
|
||||
// (annex bits only, no cross-partition/genome-level resolution): does
|
||||
// base A even show up as *present* in minorant families' own
|
||||
// `FamilyMask`s at all, at a rate proportional to the other 3 bases?
|
||||
// If yes, the anomaly is specific to the pairwise/`single_form`
|
||||
// resolution stage; if `has_base[0]` is itself near-zero here, the
|
||||
// anomaly originates earlier, in `build_sibling_annex`/`central_base`.
|
||||
let idx = KmerIndex::open("/Users/coissac/Sync/travail/__MOI__/obikmer/benchmark/global_index_presence")
|
||||
.expect("open real index");
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(&idx).expect("layer dirs");
|
||||
let mut has_base = [0u64; 4];
|
||||
let mut minorant_total = 0u64;
|
||||
let mut minorant_central_a = 0u64;
|
||||
for layer_dir in &layer_dirs {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).unwrap();
|
||||
for slot in 0..annex.len() {
|
||||
let Some(mask) = annex.get(slot) else { continue };
|
||||
if !mask.is_minorant() {
|
||||
continue;
|
||||
}
|
||||
minorant_total += 1;
|
||||
for b in 0..4u8 {
|
||||
if mask.has(b) {
|
||||
has_base[b as usize] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Cross-check: for a sample of this layer's minorants, is the
|
||||
// slot's *own* central base ever actually 'A' (bit 0)? — reads
|
||||
// the real k-mer via the layer's MPHF, not just the mask.
|
||||
let meta = PartitionMeta::load(layer_dir.parent().unwrap()).unwrap();
|
||||
let mphf = MphfLayer::open(layer_dir, &meta.mode).unwrap();
|
||||
for (order, kmer) in mphf.enumerate_kmers().take(2_000_000) {
|
||||
let Some(mask) = annex.get(order) else { continue };
|
||||
if mask.is_minorant() && central_base(kmer, idx.kmer_size()) == 0 {
|
||||
minorant_central_a += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"minorant_total={minorant_total} has_base(A,C,G,T)={has_base:?} minorant_central_a_sampled={minorant_central_a}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn diag_real_index_sankoff_bundle_base_a() {
|
||||
// Structural check (`diag_real_index_base_a_representation`) shows
|
||||
// base A fully, proportionally represented at the family level (~23M
|
||||
// minorants, the *highest* of the four bases) — so the anomaly must
|
||||
// be downstream, in `sankoff_bundle`'s actual pairwise resolution.
|
||||
// Reproduces through the real cross-partition code path (not the
|
||||
// minimal synthetic fixture, which showed the mechanism working in
|
||||
// isolation), bounded by `--subsample` to stay fast.
|
||||
let idx = KmerIndex::open("/Users/coissac/Sync/travail/__MOI__/obikmer/benchmark/global_index_presence")
|
||||
.expect("open real index");
|
||||
let n_genomes = idx.meta().genomes.len();
|
||||
let exclude_mask = vec![false; n_genomes];
|
||||
let bundle = idx.sankoff_bundle(Some(1_000_000), None, 0.5, &exclude_mask).expect("sankoff_bundle");
|
||||
println!("base_pair_tally.same={:?}", bundle.base_pair_tally.same);
|
||||
println!("base_pair_tally.counts={:?}", bundle.base_pair_tally.counts);
|
||||
let alignment_a_count: usize = bundle.alignment.sequences.iter()
|
||||
.map(|seq| seq.iter().filter(|&&b| b == b'A').count())
|
||||
.sum();
|
||||
println!("alignment raw 'A' byte count={alignment_a_count}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn diag_real_index_genome_mask_for_base_a_families() {
|
||||
// Directly inspects the raw `genome_mask` array `sankoff_bundle`'s
|
||||
// closures see, for the first few variable families where base A is
|
||||
// present — to check whether A ever co-occurs as `single_form` in two
|
||||
// *different* genomes at the same family at all (the precondition for
|
||||
// `bp_same`/`bp_counts` to ever increment for base A), rather than
|
||||
// reasoning about it further.
|
||||
use std::sync::Arc;
|
||||
let idx = KmerIndex::open("/Users/coissac/Sync/travail/__MOI__/obikmer/benchmark/global_index_presence")
|
||||
.expect("open real index");
|
||||
let n_parts = idx.n_partitions();
|
||||
let n_genomes = idx.meta().genomes.len();
|
||||
let with_counts = idx.meta().config.with_counts;
|
||||
let k = idx.kmer_size();
|
||||
let n_bits = n_parts.trailing_zeros() as usize;
|
||||
let partition = obikpartitionner::KmerPartition::open_with_config(
|
||||
idx.root_path(), idx.kmer_size(), idx.minimizer_size(), n_bits,
|
||||
).unwrap();
|
||||
let cache = Arc::new(super::cache::PartitionCache::build(&partition, n_parts, with_counts).unwrap());
|
||||
let layer_dirs = super::family_scan::sibling_layer_dirs(&idx).unwrap();
|
||||
|
||||
let mut printed = 0usize;
|
||||
for layer_dir in &layer_dirs {
|
||||
super::family_scan::scan_layer_families(
|
||||
layer_dir, n_parts, n_genomes, with_counts, k, &cache, &super::family_scan::Selection::All,
|
||||
|_family_idx, mask, genome_mask| {
|
||||
if printed >= 15 || !mask.has(0) || mask.family_size() < 2 {
|
||||
return;
|
||||
}
|
||||
let single_a: Vec<usize> = (0..n_genomes).filter(|&g| genome_mask[g] == 1).collect();
|
||||
let others: Vec<(usize, u8)> = (0..n_genomes)
|
||||
.filter(|&g| genome_mask[g] != 0 && genome_mask[g] != 1)
|
||||
.map(|g| (g, genome_mask[g]))
|
||||
.collect();
|
||||
println!(
|
||||
"family bits={:#06b} size={} single_A_genomes={:?} other_nonzero_genomes={:?}",
|
||||
mask.bits(), mask.family_size(), single_a, others,
|
||||
);
|
||||
printed += 1;
|
||||
},
|
||||
).unwrap();
|
||||
if printed >= 15 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn diag_plant_index_presence_matrix_sparsity() {
|
||||
|
||||
Reference in New Issue
Block a user