From b132f0847fc955d0263062cc476d67b48e461756 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 22 Oct 2026 11:48:04 +0200 Subject: [PATCH 1/4] feat: add VCF export and refine alignment output infrastructure Introduce `--vcf` and `--fasta` options for phylo commands, allowing users to export pseudo-alignment data as VCF or FASTA. Implement comprehensive VCF logic including stable virtual coordinates, canonical ID generation, and flexible genotype representation based on ambiguity settings. Refactor internal alignment structures to track k-mer families and site loci, ensuring consistent data flow from scanning through to output generation. - Add `src/obikmer/src/cmd/phylo/alignment_out.rs` with `create` and `write_alignment_outputs` to handle file creation and dispatching of FASTA/VCF formats based on `PhyloArgs`. - Add `src/obikphylo/src/siblings/algorithms/alignment_vcf.rs` implementing VCF-specific logic: `write_vcf`, `alignment`, `cell_alleles`, `majority_base`, and site construction via `site` using fixed $K=11$. Includes helpers for IUPAC state inversion, genotype ordering, and stable coordinate generation. - Update `src/obikmer/src/cmd/phylo/args.rs` to add `fasta` and `vcf` flags and a new `alignment_output()` method that returns true if any output format is requested. Add `pseudo_alignment` field documentation. - Modify `src/obikmer/src/cmd/phylo/mod.rs` to use `alignment_output()` for determining alignment needs, update error messages for `--n-sites`, and import `write_alignment_outputs` from the new module. - Refactor `src/obikphylo/src/siblings/algorithms/alignment.rs` by adding `SiteLocus` struct (partition, layer, family_idx, family_id) and updating `SnpAlignment` and `AlignmentReducer` to track sites. Update `new`, `finish`, and `reduce` to populate site information from `SurvivingFamily`. - Update `src/obikphylo/src/siblings/algorithms/iqtree.rs`: `alignment` now returns `SnpAlignment` with empty sites; update `drop_ascertainment_noninformative` and `recode_symbols_as_absent` to handle the new `sites` field. - Update `src/obikphylo/src/siblings/algorithms/family_scan.rs`: `GeneratedBatch` now includes a `kmers` field; `scan_layer_families` accepts `CanonicalKmer` and passes it to closures instead of masks. - Update `src/obikphylo/src/siblings/algorithms/subsample.rs`: `SurvivingFamily` gains a `kmer` field; `sample_layer` closure accepts and uses this k-mer. - Update `src/obikphylo/src/siblings/algorithms/snp_distance.rs`: `tally_2genomes` initializes `SurvivingFamily` with explicit `CanonicalKmer` values. - Update `src/obikphylo/src/siblings/algorithms/stats.rs`: `sibling_annex_stats` signature adds `_kmer` parameter. - Update `src/obikphylo/src/siblings/algorithms/family_annexes.rs`: closures in `ensure_layer_family_annexes` and `ensure_layer_pair_weight_annexes` now accept `_kmer`. - Update `src/obikphylo/src/siblings/algorithms/pairwise.rs`: `ambiguous_vs_single` initializes k-mer using `CanonicalKmer::from_raw_unchecked(0)`. - Add `iupac_state` function to `src/obikphylo/src/siblings/algorithms/masking.rs` mapping DNA characters to binary codes. - Update `src/obikphylo/src/siblings/algorithms/mod.rs` to export `SiteLocus` and include new `alignment_vcf` module. - Add `family_id` and `write_family_id_ascii` methods to `src/obikseq/src/kmer.rs` for stable k-mer family identification. - Add tests `family_id_partitions_kmers_into_families` and `family_id_ascii_masks_centre` to `src/obikseq/src/tests/kmer.rs`. - Update `src/obikphylo/src/siblings/mod.rs` imports to include new algorithm exports. --- DevDocMD/theory/evolutionary_distances.md | 51 ++++ UserDocMD/usage/phylo.md | 17 +- src/obikmer/src/cmd/phylo/alignment_out.rs | 51 ++++ src/obikmer/src/cmd/phylo/args.rs | 21 ++ src/obikmer/src/cmd/phylo/iqtree.rs | 13 +- src/obikmer/src/cmd/phylo/mod.rs | 24 +- .../src/siblings/algorithms/alignment.rs | 29 +- .../src/siblings/algorithms/alignment_vcf.rs | 256 ++++++++++++++++++ .../src/siblings/algorithms/entropy_scan.rs | 2 +- .../src/siblings/algorithms/family_annexes.rs | 4 +- .../src/siblings/algorithms/family_scan.rs | 13 +- .../src/siblings/algorithms/masking.rs | 26 ++ src/obikphylo/src/siblings/algorithms/mod.rs | 3 +- .../src/siblings/algorithms/pairwise.rs | 2 +- .../src/siblings/algorithms/snp_distance.rs | 2 + .../src/siblings/algorithms/stats.rs | 2 +- .../src/siblings/algorithms/subsample.rs | 7 +- src/obikphylo/src/siblings/mod.rs | 2 +- src/obikseq/src/kmer.rs | 22 ++ src/obikseq/src/tests/kmer.rs | 48 ++++ 20 files changed, 558 insertions(+), 37 deletions(-) create mode 100644 src/obikmer/src/cmd/phylo/alignment_out.rs create mode 100644 src/obikphylo/src/siblings/algorithms/alignment_vcf.rs diff --git a/DevDocMD/theory/evolutionary_distances.md b/DevDocMD/theory/evolutionary_distances.md index d8d9bf33..ef83f58e 100644 --- a/DevDocMD/theory/evolutionary_distances.md +++ b/DevDocMD/theory/evolutionary_distances.md @@ -2873,6 +2873,57 @@ user, no external consumers to break). Scoped to the distance matrix only: `--shared-kmers` and `--raw-snp-counts` are counts, not distances, and keep their existing CSV-only format. +## VCF export of the pseudo-alignment + +`phylo --vcf` writes the drawn sites of `--pseudo-alignment` as a VCF +(`--fasta`/`--vcf` choose the formats, FASTA is the default, both can be +combined). A family has no genomic coordinate, so coordinates are virtual and +derived from where the scan finds the family: `CHROM = obikmer__`, +`POS = family_idx + 1` (`scan_layer_families`' iteration-order index inside +that layer). Stable for a given index + annex, sorted by construction because +`on_family` is replayed in `family_idx` order. A global dense rank or a bit +packing of (layer, partition, idx) into one `POS` was rejected: it needs +per-layer minorant counts and can exceed VCF's 2^31-1 bound. + +`ID` is `CanonicalKmer::family_id()`: centre forced to `A` on the +k-mer and on its revcomp, each canonicalised, minimum of the two (centre +printed as `N`). Index-independent, hence the key for merging VCFs from +different indexes. No `INFO/LOCUS` duplicate (dropped: same string as `ID`, costly +at 5M loci); `QUAL`/`FILTER`/`INFO` are all `.`. An alternative defined as the minimum of +`central_canonical_neighbors()` was dropped: it canonicalises 4 k-mers where +this one needs 2, with no other benefit. `family_id` idempotence and +family-wide equality are tested exhaustively for k = 3, 5, 7. + +`SurvivingFamily` now carries the minorant k-mer (the scan callback passes it), +and `SnpAlignment` a `SiteLocus` per column. Alleles are the unambiguous cells +of the IUPAC matrix, in the FASTA's convention (central base of the member's +canonical form, not necessarily the `ID` k-mer's strand). + +Absence handling. `masked_state` keeps absence (`m == 0`, `-`) as a real state +unless `--free-loss`, where it becomes missing (`?`). A first version mapped +`-` and `?` both to `.`: on a toy 4-genome index half of the lines then had +`ALT = .` (presence/absence-only sites, selected by `survives_masking`), i.e. +polymorphic sites rendered as monomorphic. Now `-` is the extra allele `*` +(VCF >= 4.2 "allele missing due to upstream deletion"), after the bases; `?` +and IUPAC ambiguity codes stay `.` (haploid GT cannot express a genome +carrying several forms; superseded below). The writer needs no knowledge of +`--free-loss`: `-` simply never occurs with it. A reference-genome option +(`--vcf-ref`) was implemented then dropped: `REF` is always the per-site +majority base (alphabetical on ties). + +Ploidy. VCF allows any ploidy in `GT` (number of alleles written, variable +per sample/site), but an obikmer cell only says *which* forms a genome +carries, not their multiplicity (`A/A/G` and `A/G/G` are both `R`), nor +whether several forms mean heterozygosity or paralogues. Compromise chosen: +`--no-ambiguity` => haploid (ambiguous cells are already `?`); otherwise +diploid, single base `X` -> `X/X`, two-base code -> `X/Y`, `-` -> `*/*`, and +three/four-base codes or `?` -> `./.`. `REF` is the majority allele counting +each copy once (so a het contributes 1+1, a homozygote 2). Counting copies +for a real dosage (polyploids) would need k-mer counts (`--with-counts`), +not used by the pseudo-alignment scan today. Toy index: default mode 187 +lines, no `ALT = .`; `--no-ambiguity` 93 lines (the sites whose polymorphism +was only ambiguity codes disappear, since those cells became `?`). + ## References The Mash mutation-rate model this discussion contrasts with: diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index 80b0f414..ec89eabc 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -55,6 +55,8 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model | `--shannon` | Write `_entropy.csv`: per-family Shannon entropy, one row per family, full unsampled scan | | `--shannon-summary` | Write `_shannon_summary.csv`: the same scan as `--shannon`, reduced to a distribution (see below) — a few hundred thousand lines at most, whatever the size of the index | | `--pseudo-alignment` | Write `_alignment.fasta`: a SNP-only pseudo-alignment. Requires `--n-sites N\|all` | +| `--fasta` | Choose FASTA as the pseudo-alignment format (the default when neither `--fasta` nor `--vcf` is given). Implies `--pseudo-alignment` | +| `--vcf` | Also (or instead) write `_alignment.vcf`, the same sites as a VCF. Implies `--pseudo-alignment`; combinable with `--fasta` | ### `--sibling-stats` @@ -75,6 +77,18 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model `_alignment.fasta` — one record per non-excluded genome, one column per variable family (family size ≥ 2). Each site is IUPAC-coded from the genome's presence mask at that family: a single observed form → the plain base; several forms → the matching IUPAC ambiguity code; no form → `-`. +#### VCF output (`--vcf`) + +The same drawn sites can be written as `_alignment.vcf`, with `--fasta` and `--vcf` freely combined (FASTA alone remains the default). A VCF needs genomic coordinates, which a family of k-mers does not have, so each site gets virtual ones that are stable for a given index: the contig is `obikmer__` (one per scanned layer of each partition) and `POS` is the family's rank inside that layer, starting at 1. The same family therefore always lands at the same position as long as the index and its sibling annex are not rebuilt, and the file is sorted by construction. + +The `ID` column is a label that does not depend on the index at all: the family's k-mer with its central base replaced by `N`, taken in a canonical orientation shared by every member of the family. Two VCF files produced from different indexes can therefore be matched on `ID`. + +`REF` is the most frequent allele among the genomes at that site, each allele copy counted once (alphabetical order on ties); `ALT` lists the other observed bases in the order A, C, G, T. The bases are those of the family member's canonical form, exactly as in the FASTA, so they are not necessarily on the same strand as the `ID` k-mer. + +The ploidy of the genotypes follows `--no-ambiguity`. Without it, a genome can carry several forms of the same family, which the FASTA shows as an IUPAC ambiguity code, and genotypes are diploid: a single base `X` is written `X/X` (for example `0/0`), a code standing for two bases is the heterozygote (`0/1`), with the allele indices always in ascending order. A code standing for three or four bases (or `N`) cannot be expressed as a diploid genotype and is written `./.`. With `--no-ambiguity`, such genomes are already treated as missing, so genotypes are haploid: a single index (`0`, `1`...), and `.` for a missing genome. The diploid choice is a convention, not a measurement: the file records which forms the genome carries, not how many copies of each, and a genome with two forms may equally well come from a heterozygous individual or from two similar regions of the same genome. + +Without `--free-loss`, the absence of a family in a genome is a real state (a loss), written `-` in the FASTA. In the VCF it is the extra allele `*`, listed after the bases, so a genome lacking the family gets the genotype of `*` (`1/1` if `*` is the only alternative, `3/3` if `ALT` is `G,T,*`) and a site that differs only by presence/absence has `ALT` equal to `*`. With `--free-loss`, absence is treated as missing data instead and such genomes get a missing genotype. A site where no genome has a single base or a two-base code has no `REF` and is left out; the log line gives the number of sites written out of the number drawn. + | Option | Default | Description | |---|---|---| | `--n-sites N\|all` | none (mandatory here) | Number of families to draw, or `all` to use every variable family that passes the other filters, with no sampling. A number is a target, not a guarantee. | @@ -222,7 +236,8 @@ CSV matrix layout (`_dist.csv`, `_shared.csv`, `_family_overlap.csv`): header `g | `_shannon_summary.csv` | `--shannon-summary` | CSV table | families per entropy bin, number of carriers and number of single-member carriers | | `_sample_genomes.csv`, `_sample_carriers.csv`, `_sample_overlap.csv` | `--sample-stats` | CSV | what the draw contains: per genome, per number of carriers, per pair of genomes | | `_sample_plan.csv` | `--sample-plan` | CSV table | per layer: eligible families, weight, quota, drawn, kept | -| `_alignment.fasta` | `--pseudo-alignment` | FASTA | SNP-only pseudo-alignment, IUPAC-coded | +| `_alignment.fasta` | `--pseudo-alignment` or `--fasta` | FASTA | SNP-only pseudo-alignment, IUPAC-coded | +| `_alignment.vcf` | `--vcf` | VCF | Same sites as the FASTA, with stable virtual coordinates and a k-mer-based `ID` | ### Sankoff calibration and exports diff --git a/src/obikmer/src/cmd/phylo/alignment_out.rs b/src/obikmer/src/cmd/phylo/alignment_out.rs new file mode 100644 index 00000000..179828b4 --- /dev/null +++ b/src/obikmer/src/cmd/phylo/alignment_out.rs @@ -0,0 +1,51 @@ +//! `--pseudo-alignment` / `--fasta` / `--vcf` file output. + +use std::io::{BufWriter, Write}; + +use obikphylo::siblings::SnpAlignment; +use tracing::info; + +use super::args::PhyloArgs; + +fn create(path: &str) -> BufWriter { + BufWriter::new(std::fs::File::create(path).unwrap_or_else(|e| { + eprintln!("error creating {path}: {e}"); + std::process::exit(1); + })) +} + +/// Writes the drawn alignment as FASTA (default, or `--fasta`) and/or VCF +/// (`--vcf`) next to `-o `. +pub(super) fn write_alignment_outputs( + alignment: &SnpAlignment, + labels: &[String], + args: &PhyloArgs, +) { + let prefix = args.output.as_ref().map(|p| p.display().to_string()); + let path_for = |suffix: &str| match &prefix { + Some(p) => format!("{p}_alignment.{suffix}"), + None => format!("alignment.{suffix}"), + }; + let n_sites = alignment.sequences.first().map_or(0, Vec::len); + let n_genomes = alignment.genome_indices.len(); + + if args.fasta || !args.vcf { + let path = path_for("fasta"); + let mut f = create(&path); + for (&g, seq) in alignment.genome_indices.iter().zip(alignment.sequences.iter()) { + obifastwrite::write_plain_record(seq, &labels[g], &mut f).unwrap(); + } + info!("pseudo-alignment ({n_sites} site(s), {n_genomes} genome(s)) → {path}"); + } + + if args.vcf { + let path = path_for("vcf"); + let mut f = create(&path); + let written = alignment.write_vcf(labels, !args.no_ambiguity, &mut f).unwrap_or_else(|e| { + eprintln!("error writing {path}: {e}"); + std::process::exit(1); + }); + f.flush().unwrap(); + info!("pseudo-alignment VCF ({written}/{n_sites} site(s), {n_genomes} genome(s)) → {path}"); + } +} diff --git a/src/obikmer/src/cmd/phylo/args.rs b/src/obikmer/src/cmd/phylo/args.rs index 2a53aa1d..985add02 100644 --- a/src/obikmer/src/cmd/phylo/args.rs +++ b/src/obikmer/src/cmd/phylo/args.rs @@ -275,6 +275,19 @@ pub struct PhyloArgs { #[arg(long)] pub pseudo_alignment: bool, + /// Write the pseudo-alignment as FASTA (`_alignment.fasta`) — + /// the default format when neither `--fasta` nor `--vcf` is given. Implies + /// `--pseudo-alignment`; combinable with `--vcf`. + #[arg(long)] + pub fasta: bool, + + /// Write the pseudo-alignment as VCF (`_alignment.vcf`): one + /// contig `obikmer__` per scanned layer, `POS` the + /// family's index within it, `ID` the family's masked canonical k-mer. + /// Implies `--pseudo-alignment`; combinable with `--fasta`. + #[arg(long)] + pub vcf: bool, + /// Number of variable sites to draw index-wide, or `all` for every site /// that passes the other filters (no sampling then). Mandatory for /// `--pseudo-alignment`/`--sankoff`; optional for a `snp-*` @@ -504,3 +517,11 @@ pub struct PhyloArgs { #[arg(short, long)] pub output: Option, } + +impl PhyloArgs { + /// Whether a pseudo-alignment must be drawn and written (`--pseudo-alignment`, + /// or any of its output-format flags). + pub fn alignment_output(&self) -> bool { + self.pseudo_alignment || self.fasta || self.vcf + } +} diff --git a/src/obikmer/src/cmd/phylo/iqtree.rs b/src/obikmer/src/cmd/phylo/iqtree.rs index 0a6bcbc9..ef552385 100644 --- a/src/obikmer/src/cmd/phylo/iqtree.rs +++ b/src/obikmer/src/cmd/phylo/iqtree.rs @@ -100,7 +100,14 @@ fn drop_ascertainment_noninformative(alignment: &SnpAlignment) -> SnpAlignment { .collect() }) .collect(); - SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone() } + let sites = alignment + .sites + .iter() + .zip(keep.iter()) + .filter(|&(_, &k)| k) + .map(|(&site, _)| site) + .collect(); + SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone(), sites } } /// Recode every occurrence of a byte in `symbols` to `-` — the same @@ -119,7 +126,7 @@ fn recode_symbols_as_absent(alignment: &SnpAlignment, symbols: &[u8]) -> SnpAlig .collect() }) .collect(); - SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone() } + SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone(), sites: alignment.sites.clone() } } fn compact_alphabet(alignment: &SnpAlignment, free_loss: bool) -> CompactAlphabet { @@ -384,7 +391,7 @@ mod tests { fn alignment(sequences: Vec>) -> SnpAlignment { let genome_indices = (0..sequences.len()).collect(); - SnpAlignment { sequences, genome_indices } + SnpAlignment { sequences, genome_indices, sites: Vec::new() } } #[test] diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 995077a3..d9636162 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -1,3 +1,4 @@ +mod alignment_out; mod args; mod iqtree; mod phyg; @@ -19,6 +20,7 @@ use obikphylo::{Metrics, neighbor_joining, upgma}; use obisys::{Reporter, Stage}; use tracing::info; +use alignment_out::write_alignment_outputs; use args::{DistanceArg, NSitesArg, distance_to_compute}; use iqtree::write_iqtree; use phyg::write_sankoff_phyg; @@ -285,13 +287,13 @@ pub fn run(args: PhyloArgs) { let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree; let distance = distance_to_compute(args.distance, args.nj, args.upgma, args.shared_kmers); let snp_kind = distance.and_then(DistanceArg::as_snp); - let need_alignment = args.pseudo_alignment || sankoff_requested; + let need_alignment = args.alignment_output() || sankoff_requested; let need_pairwise = sankoff_requested || snp_kind.is_some(); // `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional) let target = args.n_sites.and_then(NSitesArg::target); - if args.pseudo_alignment && args.n_sites.is_none() { - eprintln!("error: --pseudo-alignment requires --n-sites "); + if args.alignment_output() && args.n_sites.is_none() { + eprintln!("error: --pseudo-alignment/--fasta/--vcf require --n-sites "); std::process::exit(1); } if sankoff_requested && args.n_sites.is_none() { @@ -357,21 +359,9 @@ pub fn run(args: PhyloArgs) { let alignment = alignment_reducer.map(obikphylo::siblings::AlignmentReducer::finish); // ── SNP pseudo-alignment (`--pseudo-alignment`) ───────────────────────────── - if args.pseudo_alignment { + if args.alignment_output() { let alignment = alignment.as_ref().expect("scanned above: need_alignment"); - - let path = args.output.as_ref() - .map(|p| format!("{}_alignment.fasta", p.display())) - .unwrap_or_else(|| "alignment.fasta".into()); - let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| { - eprintln!("error creating {path}: {e}"); - std::process::exit(1); - })); - let n_sites = alignment.sequences.first().map_or(0, Vec::len); - for (&g, seq) in alignment.genome_indices.iter().zip(alignment.sequences.iter()) { - obifastwrite::write_plain_record(seq, &labels[g], &mut f).unwrap(); - } - info!("pseudo-alignment ({n_sites} site(s), {} genome(s)) → {path}", alignment.genome_indices.len()); + write_alignment_outputs(alignment, &labels, &args); } // ── Sankoff cost-matrix calibration (`--sankoff`, `--tnt`, `--phyg`, `--iqtree`) ── diff --git a/src/obikphylo/src/siblings/algorithms/alignment.rs b/src/obikphylo/src/siblings/algorithms/alignment.rs index fb59a1a4..5a0392a7 100644 --- a/src/obikphylo/src/siblings/algorithms/alignment.rs +++ b/src/obikphylo/src/siblings/algorithms/alignment.rs @@ -5,12 +5,26 @@ //! masking ([`super::masking::survives_masking`]). use obikidxcache::index_cache::IndexCache; +use obikseq::RawKmer; use rkyv::{Archive, Deserialize, Serialize}; use super::masking::iupac_code; use super::scan::LayerReducer; use super::subsample::SurvivingFamily; +/// Where an alignment column comes from: enough to give the same family the +/// same virtual coordinates every time for a given index (VCF `CHROM`/`POS`) +/// and an index-independent label (`family_id`). +#[derive(Clone, Copy, Archive, Serialize, Deserialize)] +pub struct SiteLocus { + pub partition: usize, + pub layer: usize, + /// `scan_layer_families`'s iteration-order index within `(partition, layer)`. + pub family_idx: usize, + /// Raw encoding of [`obikseq::CanonicalKmer::family_id`]. + pub family_id: RawKmer, +} + /// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the /// same length (one byte per sampled family, in the order /// [`super::scan::sampled_scan`] produced them: layer by layer, @@ -27,6 +41,8 @@ pub struct SnpAlignment { /// order — the caller's authoritative mapping back to genome labels, so /// it never has to re-derive which genomes were excluded itself. pub genome_indices: Vec, + /// `sites[j]` describes column `j` of every row. + pub sites: Vec, } /// Accumulates a [`SnpAlignment`] over a [`super::scan::sampled_scan`] @@ -40,6 +56,7 @@ pub struct SnpAlignment { pub struct AlignmentReducer { genome_indices: Vec, sequences: Vec>, + sites: Vec, } impl AlignmentReducer { @@ -49,16 +66,22 @@ impl AlignmentReducer { .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) .collect(); let sequences = vec![Vec::new(); genome_indices.len()]; - Self { genome_indices, sequences } + Self { genome_indices, sequences, sites: Vec::new() } } pub fn finish(self) -> SnpAlignment { - SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices } + SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices, sites: self.sites } } } impl LayerReducer for AlignmentReducer { - fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { + fn reduce(&mut self, partition: usize, layer: usize, survivors: &[SurvivingFamily]) { + self.sites.extend(survivors.iter().map(|f| SiteLocus { + partition, + layer, + family_idx: f.family_idx, + family_id: f.kmer.family_id().raw(), + })); reduce_alignment(survivors, &self.genome_indices, &mut self.sequences); } } diff --git a/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs b/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs new file mode 100644 index 00000000..97666553 --- /dev/null +++ b/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs @@ -0,0 +1,256 @@ +//! VCF rendering of a [`SnpAlignment`]. +//! +//! One contig per `(partition, layer)` (`obikmer__`), `POS` +//! = the family's `family_idx + 1` inside it: for a given index and sibling +//! annex the same family always gets the same virtual coordinates, and +//! sorting by `(partition, layer, family_idx)` is what makes the file a +//! well-formed (sorted) VCF. `ID` is the index-independent +//! [`obikseq::CanonicalKmer::family_id`] label (centre base masked as `N`). +//! +//! `REF`/`ALT` are the unambiguous bases (`A`/`C`/`G`/`T` cells of the +//! alignment) observed at the site, in the alignment's own convention: the +//! central base of the family member's *canonical* form, which is not +//! necessarily the strand of the `ID` k-mer. A `-` cell (family absent from +//! the genome — a real state, unlike `?`, which `--free-loss` produces and +//! which means missing) is the extra allele `*`, listed after the bases. +//! +//! Ploidy of the genotypes is a caller choice. Haploid: one allele per +//! genome; an ambiguous cell is missing. Diploid: a single base `X` is +//! `X/X`, a two-base code is the heterozygote, `*` is `*/*`; codes with +//! three or four bases (and `?`) are missing (`./.`). In both modes a +//! genome gets the allele indices in ascending order. + +use std::io::{self, Write}; + +use obikseq::CanonicalKmer; + +use super::alignment::SnpAlignment; +use super::masking::iupac_state; + +const BASES: [u8; 4] = *b"ACGT"; + +fn base_index(cell: u8) -> Option { + BASES.iter().position(|&b| b == cell) +} + +/// One cell as alleles: `None` = missing; else its (one or, diploid, two) +/// allele characters — a base, or `*` for absence. A cell with more forms +/// than the ploidy allows, or `?`, is missing. +fn cell_alleles(cell: u8, diploid: bool) -> Option<[u8; 2]> { + let state = iupac_state(cell)?; + let allele = |bit: u8| if bit == 0 { b'*' } else { BASES[bit.trailing_zeros() as usize] }; + match state.count_ones() { + 0 => Some([b'*', b'*']), + 1 => Some([allele(state), allele(state)]), + 2 if diploid => { + let low = state & state.wrapping_neg(); + Some([allele(low), allele(state ^ low)]) + } + _ => None, + } +} + +/// `REF` base (index into `ACGT`) of one site: the most frequent unambiguous +/// base, lexicographic minimum on ties. `None` when no cell carries one. +fn majority_base(counts: &[usize; 4]) -> Option { + (0..4) + .filter(|&b| counts[b] > 0) + .max_by_key(|&b| (counts[b], std::cmp::Reverse(b))) +} + +impl SnpAlignment { + /// Writes this alignment as a VCF (haploid `GT` per genome). `labels` is + /// indexed by the index's own genome numbering ([`Self::genome_indices`] + /// values). `diploid` selects the genotype ploidy (see the module docs). + /// `REF` is the per-site majority allele (each allele copy counted once). + /// Returns the number of sites written: a site with no base among its + /// calls has no `REF` to report and is skipped. + pub fn write_vcf(&self, labels: &[String], diploid: bool, out: &mut W) -> io::Result { + let n_sites = self.sequences.first().map_or(0, Vec::len); + if self.sites.len() != n_sites { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("alignment has {n_sites} column(s) but {} site locus record(s)", self.sites.len()), + )); + } + let mut order: Vec = (0..n_sites).collect(); + order.sort_unstable_by_key(|&j| { + let s = &self.sites[j]; + (s.partition, s.layer, s.family_idx) + }); + + writeln!(out, "##fileformat=VCFv4.5")?; + writeln!(out, "##source=obikmer")?; + let mut last = None; + for &j in &order { + let s = &self.sites[j]; + if last != Some((s.partition, s.layer)) { + writeln!(out, "##contig=", s.partition, s.layer)?; + last = Some((s.partition, s.layer)); + } + } + writeln!(out, "##FORMAT=", if diploid { "diploid" } else { "haploid" })?; + write!(out, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT")?; + for &g in &self.genome_indices { + write!(out, "\t{}", labels[g])?; + } + writeln!(out)?; + + let mut written = 0usize; + let mut locus = Vec::new(); + let mut calls: Vec> = Vec::with_capacity(self.sequences.len()); + let copies = if diploid { 2 } else { 1 }; + for &j in &order { + calls.clear(); + calls.extend(self.sequences.iter().map(|seq| cell_alleles(seq[j], diploid))); + let mut counts = [0usize; 4]; + let mut absent = false; + for call in calls.iter().flatten() { + for &a in &call[..copies] { + match base_index(a) { + Some(b) => counts[b] += 1, + None => absent = true, + } + } + } + let Some(ref_base) = majority_base(&counts) else { + continue; + }; + let mut alleles = vec![BASES[ref_base]]; + alleles.extend((0..4).filter(|&b| b != ref_base && counts[b] > 0).map(|b| BASES[b])); + if absent { + alleles.push(b'*'); + } + + let s = &self.sites[j]; + locus.clear(); + CanonicalKmer::from_raw_unchecked(s.family_id).write_family_id_ascii(&mut locus)?; + write!(out, "obikmer_{}_{}\t{}\t", s.partition, s.layer, s.family_idx + 1)?; + out.write_all(&locus)?; + write!(out, "\t{}\t", alleles[0] as char)?; + if alleles.len() == 1 { + out.write_all(b".")?; + } else { + for (i, &a) in alleles[1..].iter().enumerate() { + if i > 0 { + out.write_all(b",")?; + } + out.write_all(&[a])?; + } + } + out.write_all(b"\t.\t.\t.\tGT")?; + for call in &calls { + let idx = |a: u8| alleles.iter().position(|&x| x == a); + match call.map(|c| (idx(c[0]), idx(c[1]))) { + Some((Some(x), Some(y))) if diploid => write!(out, "\t{}/{}", x.min(y), x.max(y))?, + Some((Some(x), _)) => write!(out, "\t{x}")?, + _ => out.write_all(if diploid { b"\t./." } else { b"\t." })?, + } + } + out.write_all(b"\n")?; + written += 1; + } + Ok(written) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::siblings::SiteLocus; + + fn alignment(rows: &[&[u8]], sites: Vec) -> SnpAlignment { + SnpAlignment { + sequences: rows.iter().map(|r| r.to_vec()).collect(), + genome_indices: (0..rows.len()).collect(), + sites, + } + } + + fn site(partition: usize, layer: usize, family_idx: usize) -> SiteLocus { + // Same global k as the crate's other tests (`siblingannex.rs`): the + // global is process-wide here, so it must not conflict. + obikseq::params::set_k(11); + SiteLocus { partition, layer, family_idx, family_id: 0 } + } + + fn first_record(a: &SnpAlignment) -> Vec { + first_record_ploidy(a, false) + } + + fn first_record_ploidy(a: &SnpAlignment, diploid: bool) -> Vec { + let labels: Vec = (0..a.sequences.len()).map(|i| format!("g{i}")).collect(); + let mut out = Vec::new(); + a.write_vcf(&labels, diploid, &mut out).unwrap(); + let text = String::from_utf8(out).unwrap(); + let line = text.lines().find(|l| !l.starts_with('#')).unwrap(); + line.split('\t').map(String::from).collect() + } + + #[test] + fn reference_is_majority_then_lexicographic() { + assert_eq!(majority_base(&[1, 3, 0, 0]), Some(1)); + assert_eq!(majority_base(&[2, 2, 0, 0]), Some(0)); + assert_eq!(majority_base(&[0, 0, 0, 0]), None); + } + + #[test] + fn genotypes_follow_ref_then_alt_order() { + // Column 0: A,G,G,R,? -> REF=G (majority), ALT=A; Column 1: all ambiguous -> skipped. + let a = alignment( + &[b"AY", b"GY", b"GR", b"RR", b"?R"], + vec![site(0, 0, 4), site(0, 0, 9)], + ); + let f = first_record(&a); + assert_eq!(&f[..2], ["obikmer_0_0", "5"]); + assert_eq!(&f[3..5], ["G", "A"]); + assert_eq!(&f[9..], ["1", "0", "0", ".", "."]); + } + + #[test] + fn absence_is_a_star_allele_after_the_bases() { + // A, T, - (family absent), ? (missing): REF=A, ALT=T,*. + let a = alignment(&[b"A", b"T", b"-", b"?"], vec![site(0, 0, 0)]); + let f = first_record(&a); + assert_eq!(&f[3..5], ["A", "T,*"]); + assert_eq!(&f[9..], ["0", "1", "2", "."]); + // Only presence/absence differs: REF=A, ALT=*. + let a = alignment(&[b"A", b"-"], vec![site(0, 0, 0)]); + let f = first_record(&a); + assert_eq!(&f[3..5], ["A", "*"]); + assert_eq!(&f[9..], ["0", "1"]); + } + + #[test] + fn iupac_state_inverts_iupac_code() { + for state in 0u8..16 { + assert_eq!(iupac_state(super::super::masking::iupac_code(state)), Some(state)); + } + assert_eq!(iupac_state(b'?'), None); + } + + #[test] + fn diploid_genotypes() { + // A, R (A/G), G, N (4 forms -> missing), - (absent), ? (missing). + let a = alignment(&[b"A", b"R", b"G", b"N", b"-", b"?"], vec![site(0, 0, 0)]); + let f = first_record_ploidy(&a, true); + // majority: A = 2 + 1 = 3, G = 1 + 2 = 3 -> tie -> A; ALT = G,* + assert_eq!(&f[3..5], ["A", "G,*"]); + assert_eq!(&f[9..], ["0/0", "0/1", "1/1", "./.", "2/2", "./."]); + // Same column, haploid: the ambiguous cells are missing. + let f = first_record_ploidy(&a, false); + assert_eq!(&f[9..], ["0", ".", "1", ".", "2", "."]); + } + + #[test] + fn sites_sorted_by_contig_then_position() { + let a = alignment(&[b"AA", b"GG"], vec![site(1, 0, 0), site(0, 2, 7)]); + let labels = vec!["x".to_string(), "y".to_string()]; + let mut out = Vec::new(); + a.write_vcf(&labels, false, &mut out).unwrap(); + let text = String::from_utf8(out).unwrap(); + let body: Vec<&str> = text.lines().filter(|l| !l.starts_with('#')).collect(); + assert!(body[0].starts_with("obikmer_0_2\t8\t")); + assert!(body[1].starts_with("obikmer_1_0\t1\t")); + } +} diff --git a/src/obikphylo/src/siblings/algorithms/entropy_scan.rs b/src/obikphylo/src/siblings/algorithms/entropy_scan.rs index 6e721573..0f19d8c6 100644 --- a/src/obikphylo/src/siblings/algorithms/entropy_scan.rs +++ b/src/obikphylo/src/siblings/algorithms/entropy_scan.rs @@ -67,7 +67,7 @@ pub(crate) fn scan_family_entropies( n_genomes, fast_mode, &Selection::Some(&eligible), - |family_idx, mask, genome_mask| { + |family_idx, _kmer, mask, genome_mask| { if failure.is_some() { return; } diff --git a/src/obikphylo/src/siblings/algorithms/family_annexes.rs b/src/obikphylo/src/siblings/algorithms/family_annexes.rs index b034734e..bdce53af 100644 --- a/src/obikphylo/src/siblings/algorithms/family_annexes.rs +++ b/src/obikphylo/src/siblings/algorithms/family_annexes.rs @@ -145,7 +145,7 @@ pub(crate) fn ensure_layer_family_annexes( n_genomes, fast_mode, &Selection::Some(&eligible), - |family_idx, mask, genome_mask| { + |family_idx, _kmer, mask, genome_mask| { debug_assert!( mask.family_size() >= 2, "Selection::Some(eligible) must only visit non-monomorphic minorants" @@ -217,7 +217,7 @@ pub(crate) fn ensure_layer_pair_weight_annexes( n_genomes, fast_mode, &Selection::Some(&eligible), - |family_idx, _mask, genome_mask| { + |family_idx, _kmer, _mask, genome_mask| { let compact = reindex[&family_idx]; for (no_ambiguity, inverse, builder) in builders.iter_mut() { builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse)); diff --git a/src/obikphylo/src/siblings/algorithms/family_scan.rs b/src/obikphylo/src/siblings/algorithms/family_scan.rs index c593ce61..79a1dd82 100644 --- a/src/obikphylo/src/siblings/algorithms/family_scan.rs +++ b/src/obikphylo/src/siblings/algorithms/family_scan.rs @@ -125,6 +125,8 @@ struct SourceBatch { struct GeneratedBatch { start_family_idx: usize, masks: Vec, + /// Each family's minorant k-mer, same order as `masks`. + kmers: Vec, /// Flat `slots.len() * n_genomes` — `genome_mask[i * n_genomes + g]`. genome_mask: Vec, /// `outgoing[dest_partition]` = `(variant, family_idx_in_batch, base, @@ -146,8 +148,8 @@ enum FamData { /// generation and resolution use different concurrency. `on_family` is /// called once per /// family, in iteration order, with that family's iteration-order index -/// (the same numbering [`Selection`] indices are drawn from), its own annex -/// mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set +/// (the same numbering [`Selection`] indices are drawn from), its minorant +/// k-mer, its own annex mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set /// iff genome `g` carries the member whose own canonical central base is /// `b`), backed by a scratch buffer reused across every call — callers that /// need to keep data past the call must copy it themselves. @@ -164,7 +166,7 @@ pub(crate) fn scan_layer_families( n_genomes: usize, fast_mode: bool, selection: &Selection, - mut on_family: impl FnMut(usize, FamilyMask, &[u8]), + mut on_family: impl FnMut(usize, CanonicalKmer, FamilyMask, &[u8]), ) -> OKIResult<()> { let n_parts = cache.index().n_partitions(); let k = cache.index().kmer_size(); @@ -225,6 +227,7 @@ pub(crate) fn scan_layer_families( let ctx = &worker_ctx; let n = batch.entries.len(); let mut masks = Vec::with_capacity(n); + let mut kmers = Vec::with_capacity(n); let mut bases = Vec::with_capacity(n); let mut genome_mask = vec![0u8; n * ctx.n_genomes]; let mut outgoing: Vec> = @@ -248,6 +251,7 @@ pub(crate) fn scan_layer_families( for (i, entry) in batch.entries.iter().enumerate() { let (kmer, mask) = (entry.kmer, entry.mask); masks.push(mask); + kmers.push(kmer); let base = central_base(kmer, ctx.k); bases.push(base); let family_idx = batch.start_family_idx + i; @@ -297,6 +301,7 @@ pub(crate) fn scan_layer_families( GeneratedBatch { start_family_idx: batch.start_family_idx, masks, + kmers, genome_mask, outgoing, _permit: batch._permit, @@ -344,7 +349,7 @@ pub(crate) fn scan_layer_families( for (g, dst) in scratch.iter_mut().enumerate() { *dst = genome_mask[i * n_genomes + g].load(Ordering::Relaxed); } - on_family(family_idx, batch.masks[i], &scratch); + on_family(family_idx, batch.kmers[i], batch.masks[i], &scratch); } next_expected += n; } diff --git a/src/obikphylo/src/siblings/algorithms/masking.rs b/src/obikphylo/src/siblings/algorithms/masking.rs index 5838ef41..23b171a0 100644 --- a/src/obikphylo/src/siblings/algorithms/masking.rs +++ b/src/obikphylo/src/siblings/algorithms/masking.rs @@ -188,6 +188,32 @@ pub(crate) fn iupac_code(state: u8) -> u8 { } } +/// Inverse of [`iupac_code`]: the `0..=15` state a cell character encodes +/// (`-` -> 0, a base -> its single bit, an ambiguity code -> its bits), or +/// `None` for anything else — notably `?`, a missing cell. +#[inline] +pub(crate) fn iupac_state(cell: u8) -> Option { + Some(match cell { + b'-' => 0b0000, + b'A' => 0b0001, + b'C' => 0b0010, + b'G' => 0b0100, + b'T' => 0b1000, + b'R' => 0b0101, + b'Y' => 0b1010, + b'S' => 0b0110, + b'W' => 0b1001, + b'K' => 0b1100, + b'M' => 0b0011, + b'B' => 0b1110, + b'D' => 0b1101, + b'H' => 0b1011, + b'V' => 0b0111, + b'N' => 0b1111, + _ => return None, + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 0accf235..7027c413 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -14,6 +14,7 @@ //! encoding. mod alignment; +mod alignment_vcf; mod annex; mod cardcomp; mod entropy; @@ -34,7 +35,7 @@ mod subsample; use obikidxcache::index_cache::IndexCache; -pub use alignment::{AlignmentReducer, SnpAlignment}; +pub use alignment::{AlignmentReducer, SiteLocus, SnpAlignment}; pub use cardcomp::{ cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix, }; diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index c358e2e8..b301d69c 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -510,7 +510,7 @@ mod cardinality_masking_tests { // `SurvivingFamily::masked`'s own docs) — the masking decision is baked // in here, not passed to `reduce_pairwise` itself anymore. fn ambiguous_vs_single(masked: Vec>) -> Vec { - vec![SurvivingFamily { family_idx: 0, mask: FamilyMask::EMPTY.with(0).with(1), masked }] + vec![SurvivingFamily { family_idx: 0, kmer: obikseq::CanonicalKmer::from_raw_unchecked(0), mask: FamilyMask::EMPTY.with(0).with(1), masked }] } #[test] diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 01d62177..dbf2dc74 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -286,6 +286,7 @@ mod tests { for &(a, b) in pairs { families.push(SurvivingFamily { family_idx: families.len(), + kmer: obikseq::CanonicalKmer::from_raw_unchecked(0), mask: FamilyMask::EMPTY.with(a.trailing_zeros() as u8).with(b.trailing_zeros() as u8), masked: vec![Some(a), Some(b)], }); @@ -296,6 +297,7 @@ mod tests { let other = BASES[(i + 1) % 4]; families.push(SurvivingFamily { family_idx: families.len(), + kmer: obikseq::CanonicalKmer::from_raw_unchecked(0), mask: FamilyMask::EMPTY.with(b.trailing_zeros() as u8).with(other.trailing_zeros() as u8), masked: vec![Some(b), Some(b)], }); diff --git a/src/obikphylo/src/siblings/algorithms/stats.rs b/src/obikphylo/src/siblings/algorithms/stats.rs index 354582a4..cbd32bf5 100644 --- a/src/obikphylo/src/siblings/algorithms/stats.rs +++ b/src/obikphylo/src/siblings/algorithms/stats.rs @@ -106,7 +106,7 @@ pub(crate) fn sibling_annex_stats(cache: &IndexCache) -> OKIResult>, } @@ -478,7 +481,7 @@ fn sample_layer( n_genomes, fast_mode, &Selection::Some(&accepted), - |family_idx, mask, genome_mask| { + |family_idx, kmer, mask, genome_mask| { debug_assert!( mask.family_size() >= 2, "eligible bitset must only accept non-monomorphic minorants" @@ -490,7 +493,7 @@ fn sample_layer( if survives_masking(&masked, excluded) && (!spec.quartet || survives_quartet(&masked, excluded)) { - survivors.push(SurvivingFamily { family_idx, mask, masked }); + survivors.push(SurvivingFamily { family_idx, kmer, mask, masked }); kept += 1; } }, diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index 69fde7fa..0361f7db 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -43,7 +43,7 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use algorithms::{ AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer, LayerReport, PairwiseReducer, RawSnpDistanceOutput, SamplePlan, SampleStats, - SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SnpAlignment, + SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SiteLocus, SnpAlignment, SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix, pairwise_cost_matrix, sampled_scan, }; diff --git a/src/obikseq/src/kmer.rs b/src/obikseq/src/kmer.rs index 0fe41101..dd0af8f0 100644 --- a/src/obikseq/src/kmer.rs +++ b/src/obikseq/src/kmer.rs @@ -362,6 +362,28 @@ impl CanonicalKmerOf { ] } + /// Stable identifier of this k-mer's central-base family: the centre + /// forced to `A` on the k-mer and on its reverse complement, each + /// canonicalised, then the lexicographic minimum of the two. Identical + /// for the 4 members of a family whatever the orientation they are seen + /// in, and independent of any index. Well-defined for odd `L::len()`. + pub fn family_id(&self) -> Self { + let m = (L::len() - 1) / 2; + let shift = KMER_BITS - 2 - 2 * m; + let clear = |raw: RawKmer| raw & !((0b11 as RawKmer) << shift); + let fwd = KmerOf::(clear(self.0), PhantomData).canonical(); + let rev = KmerOf::(clear(self.revcomp().raw()), PhantomData).canonical(); + if fwd.0 <= rev.0 { fwd } else { rev } + } + + /// Writes [`family_id`](Self::family_id) as ASCII with the centre base + /// replaced by `N` (the family's textual locus label). + pub fn write_family_id_ascii(&self, writer: &mut W) -> io::Result<()> { + let mut ascii = self.family_id().to_ascii(); + ascii[(L::len() - 1) / 2] = b'N'; + writer.write_all(&ascii) + } + /// Return the inner value as a raw [`KmerOf`]. #[inline] pub fn into_kmer(self) -> KmerOf { diff --git a/src/obikseq/src/tests/kmer.rs b/src/obikseq/src/tests/kmer.rs index 5d9d29c7..503b68ed 100644 --- a/src/obikseq/src/tests/kmer.rs +++ b/src/obikseq/src/tests/kmer.rs @@ -252,4 +252,52 @@ mod tests { check!(5); check!(31); } + + // ── family_id ─────────────────────────────────────────────────────────── + + /// Exhaustive: `family_id` is shared by every member of a family (and by + /// each member's reverse complement) and separates distinct families — + /// the latter checked against `central_canonical_neighbors` as ground truth. + #[test] + fn family_id_partitions_kmers_into_families() { + macro_rules! check { + ($n:expr) => {{ + type K = KmerOf>; + let mut families: std::collections::HashMap<_, Vec<_>> = Default::default(); + for code in 0..4usize.pow($n) { + let ascii: Vec = (0..$n).map(|i| b"ACGT"[(code >> (2 * i)) & 3]).collect(); + let ck = K::from_ascii(&ascii).unwrap().canonical(); + let id = ck.family_id(); + assert_eq!(id, ck.revcomp().canonical().family_id()); + let mut members: Vec<_> = + ck.central_canonical_neighbors().iter().map(|n| n.raw()).collect(); + members.sort_unstable(); + for n in ck.central_canonical_neighbors() { + assert_eq!(n.family_id(), id, "k={}: members disagree", $n); + } + assert_eq!(id, id.family_id(), "k={}: family_id not idempotent", $n); + families.entry(id).or_default().push(members); + } + // One id <=> one family: all members seen under an id share one member set. + for sets in families.values() { + assert!(sets.windows(2).all(|w| w[0] == w[1]), "k={}: id merges families", $n); + } + }}; + } + check!(3); + check!(5); + check!(7); + } + + #[test] + fn family_id_ascii_masks_centre() { + let ck = KmerOf::>::from_ascii(b"ACGTA").unwrap().canonical(); + let mut out = Vec::new(); + ck.write_family_id_ascii(&mut out).unwrap(); + assert_eq!(out.len(), 5); + assert_eq!(out[2], b'N'); + // flanks of the id are the family's two flank pairs, in either orientation + let id_ascii = ck.family_id().to_ascii(); + assert_eq!([out[0], out[1], out[3], out[4]], [id_ascii[0], id_ascii[1], id_ascii[3], id_ascii[4]]); + } } -- 2.54.1 From b55123a376e4e4d55ac883ec5fad01eb48b2bfee Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 22 Oct 2026 11:52:34 +0200 Subject: [PATCH 2/4] feat: add --only-diploid flag to filter for diploid genotypes only Introduce a new `--only-diploid` option that restricts analysis to genomes with one or two family members (base or two-base IUPAC codes), treating others as missing data. This complements the existing `--no-ambiguity` behavior by providing a stricter filter for diploid-compatible sites, while ensuring conflict detection between the two flags. - Add `only_diploid` boolean argument to `PhyloArgs` in `src/obikmer/src/cmd/phylo/args.rs`, documenting its effect on pseudo-alignment and its conflict with `--no-ambiguity`. - Propagate `only_diploid` through `SamplingSpec` in `src/obikmer/src/cmd/phylo/mod.rs` and update the `run` function signature to accept the new argument. - Implement masking logic in `src/obikphylo/src/siblings/algorithms/masking.rs`: modify `masked_state` to accept `only_diploid`, add `only_diploid_keeps_up_to_two_members` helper, and update `genome_mask` documentation. - Update `sample_layer` in `src/obikphylo/src/siblings/algorithms/subsample.rs` to use `spec.only_diploid` for masking decisions and update `SamplingSpec` documentation. - Adjust `tally_2genomes` in `src/obikphylo/src/siblings/algorithms/snp_distance.rs` to pass the new `only_diploid` parameter to `masked_state`. - Update `DevDocMD/theory/evolutionary_distances.md` to describe the new `--only-diploid` option, the `masked_state` rule (`count_ones() > 2`), and clarify toy index behavior. - Update `UserDocMD/usage/phylo.md` to document `--only-diploid`, `--quartet`, and `--pair-balance` behaviors, clarifying that diploid choice is a convention and explaining how three/four-allele genomes are treated as missing. --- DevDocMD/theory/evolutionary_distances.md | 12 +++++++++- UserDocMD/usage/phylo.md | 3 ++- src/obikmer/src/cmd/phylo/args.rs | 7 ++++++ src/obikmer/src/cmd/phylo/mod.rs | 1 + .../src/siblings/algorithms/masking.rs | 23 ++++++++++++++++--- .../src/siblings/algorithms/snp_distance.rs | 2 +- .../src/siblings/algorithms/subsample.rs | 8 ++++++- 7 files changed, 49 insertions(+), 7 deletions(-) diff --git a/DevDocMD/theory/evolutionary_distances.md b/DevDocMD/theory/evolutionary_distances.md index ef83f58e..a3684aa7 100644 --- a/DevDocMD/theory/evolutionary_distances.md +++ b/DevDocMD/theory/evolutionary_distances.md @@ -2920,7 +2920,17 @@ diploid, single base `X` -> `X/X`, two-base code -> `X/Y`, `-` -> `*/*`, and three/four-base codes or `?` -> `./.`. `REF` is the majority allele counting each copy once (so a het contributes 1+1, a homozygote 2). Counting copies for a real dosage (polyploids) would need k-mer counts (`--with-counts`), -not used by the pseudo-alignment scan today. Toy index: default mode 187 +not used by the pseudo-alignment scan today. `--only-diploid` (selection option, `conflicts_with --no-ambiguity`) adds a +third `masked_state` rule: `count_ones() > 2` -> missing. It lives in +`SamplingSpec`/`masked_state`, i.e. the post-hoc masking and +`survives_masking`, so FASTA, Sankoff and `snp-*` see it too. The persisted +entropy/quartet/pair-weight/overlap annexes are keyed by the `no_ambiguity` +bool (two variants on disk); a third variant was not added: they are read +under `no_ambiguity = false`, which for the quartet pre-filter is a superset +of what survives the stricter masking (reasoned, not tested) and for the +entropy/pair-weight draw weights only changes the weighting. Unit-tested +(`masked_state`); the toy index has no 3/4-member cells, so the end-to-end +run is identical to the default mode. Toy index: default mode 187 lines, no `ALT = .`; `--no-ambiguity` 93 lines (the sites whose polymorphism was only ambiguity codes disappear, since those cells became `?`). diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index ec89eabc..6f37dece 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -85,7 +85,7 @@ The `ID` column is a label that does not depend on the index at all: the family' `REF` is the most frequent allele among the genomes at that site, each allele copy counted once (alphabetical order on ties); `ALT` lists the other observed bases in the order A, C, G, T. The bases are those of the family member's canonical form, exactly as in the FASTA, so they are not necessarily on the same strand as the `ID` k-mer. -The ploidy of the genotypes follows `--no-ambiguity`. Without it, a genome can carry several forms of the same family, which the FASTA shows as an IUPAC ambiguity code, and genotypes are diploid: a single base `X` is written `X/X` (for example `0/0`), a code standing for two bases is the heterozygote (`0/1`), with the allele indices always in ascending order. A code standing for three or four bases (or `N`) cannot be expressed as a diploid genotype and is written `./.`. With `--no-ambiguity`, such genomes are already treated as missing, so genotypes are haploid: a single index (`0`, `1`...), and `.` for a missing genome. The diploid choice is a convention, not a measurement: the file records which forms the genome carries, not how many copies of each, and a genome with two forms may equally well come from a heterozygous individual or from two similar regions of the same genome. +The ploidy of the genotypes follows `--no-ambiguity`; `--only-diploid` keeps the diploid genotypes and simply removes, before the sites are selected, the genomes whose cell would have needed three or four alleles (so no `./.` comes from them). Without it, a genome can carry several forms of the same family, which the FASTA shows as an IUPAC ambiguity code, and genotypes are diploid: a single base `X` is written `X/X` (for example `0/0`), a code standing for two bases is the heterozygote (`0/1`), with the allele indices always in ascending order. A code standing for three or four bases (or `N`) cannot be expressed as a diploid genotype and is written `./.`. With `--no-ambiguity`, such genomes are already treated as missing, so genotypes are haploid: a single index (`0`, `1`...), and `.` for a missing genome. The diploid choice is a convention, not a measurement: the file records which forms the genome carries, not how many copies of each, and a genome with two forms may equally well come from a heterozygous individual or from two similar regions of the same genome. Without `--free-loss`, the absence of a family in a genome is a real state (a loss), written `-` in the FASTA. In the VCF it is the extra allele `*`, listed after the bases, so a genome lacking the family gets the genotype of `*` (`1/1` if `*` is the only alternative, `3/3` if `ALT` is `G,T,*`) and a site that differs only by presence/absence has `ALT` equal to `*`. With `--free-loss`, absence is treated as missing data instead and such genomes get a missing genotype. A site where no genome has a single base or a two-base code has no `REF` and is left out; the log line gives the number of sites written out of the number drawn. @@ -94,6 +94,7 @@ Without `--free-loss`, the absence of a family in a genome is a real state (a lo | `--n-sites N\|all` | none (mandatory here) | Number of families to draw, or `all` to use every variable family that passes the other filters, with no sampling. A number is a target, not a guarantee. | | `--free-loss` | off | Treat a genome carrying none of a family's observed members as missing data (`?`) instead of `-` | | `--no-ambiguity` | off | Treat a genome carrying more than one member of a family as missing data (`?`) instead of an IUPAC ambiguity code | +| `--only-diploid` | off | Milder than `--no-ambiguity` (the two cannot be combined): keep the genomes carrying one or two members of a family (a base, or a two-base IUPAC code such as `R`) and treat those carrying three or four as missing data (`?`) | | `--quartet` | off | Keep only the families that can separate two pairs of genomes from each other: at least two distinct called states, each carried by at least two genomes. Which genomes count as carriers follows `--no-ambiguity` | | `--pair-balance` | off | Favour the families covering the pairs of genomes that share the fewest families across the index, so that every pair is better represented in the sample. Which genomes count as carriers follows `--no-ambiguity` | | `--sample-stats` | off | Write three CSV files describing what the draw contains (see "Describing a draw" below) | diff --git a/src/obikmer/src/cmd/phylo/args.rs b/src/obikmer/src/cmd/phylo/args.rs index 985add02..8a9d1d49 100644 --- a/src/obikmer/src/cmd/phylo/args.rs +++ b/src/obikmer/src/cmd/phylo/args.rs @@ -309,6 +309,13 @@ pub struct PhyloArgs { #[arg(long)] pub no_ambiguity: bool, + /// In `--pseudo-alignment`, keep only the genomes carrying one or two + /// members of a family (a base, or a two-base IUPAC code) and treat the + /// others (three or four members) as missing data (`?`). Milder than + /// `--no-ambiguity`, which it excludes. + #[arg(long, conflicts_with = "no_ambiguity")] + pub only_diploid: bool, + /// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, keep only /// the families that can separate two pairs of genomes from each other /// (a bipartition): at least two distinct called states, each carried diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index d9636162..78f43ce9 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -254,6 +254,7 @@ pub fn run(args: PhyloArgs) { let sampling = SamplingSpec { free_loss: args.free_loss, no_ambiguity: args.no_ambiguity, + only_diploid: args.only_diploid, quartet: args.quartet, pair_balance: args.pair_balance, entropy_bias, diff --git a/src/obikphylo/src/siblings/algorithms/masking.rs b/src/obikphylo/src/siblings/algorithms/masking.rs index 23b171a0..7314fbf4 100644 --- a/src/obikphylo/src/siblings/algorithms/masking.rs +++ b/src/obikphylo/src/siblings/algorithms/masking.rs @@ -23,15 +23,20 @@ use crate::siblings::helpers::is_carrier; /// is one `genome_mask` entry (bit `b` set iff this genome carries the /// family member whose own canonical central base is `b`; `0` = no member /// present in this genome, `∅` — a real state on its own unless -/// `free_loss` says otherwise). +/// `free_loss` says otherwise). `only_diploid` is the milder sibling of +/// `no_ambiguity`: it keeps the genomes carrying one or two members (a base +/// or a two-base IUPAC code) and drops those carrying three or four. #[inline] -pub(crate) fn masked_state(m: u8, free_loss: bool, no_ambiguity: bool) -> Option { +pub(crate) fn masked_state(m: u8, free_loss: bool, no_ambiguity: bool, only_diploid: bool) -> Option { if free_loss && m == 0 { return None; // absence -> missing, not a real state } if no_ambiguity && m.count_ones() > 1 { return None; // ambiguous (>1 member in this genome) -> missing } + if only_diploid && m.count_ones() > 2 { + return None; // more than a heterozygote's two members -> missing + } Some(m) } @@ -222,7 +227,19 @@ mod tests { /// `subsample::sample_layer` does once, up front, before calling /// [`survives_masking`]. fn masked(raw: &[u8], free_loss: bool, no_ambiguity: bool) -> Vec> { - raw.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect() + raw.iter().map(|&m| masked_state(m, free_loss, no_ambiguity, false)).collect() + } + + #[test] + fn only_diploid_keeps_up_to_two_members() { + let m = |raw: u8| masked_state(raw, false, false, true); + assert_eq!(m(0b0000), Some(0)); // absence stays a state (no --free-loss) + assert_eq!(m(0b0100), Some(0b0100)); + assert_eq!(m(0b0101), Some(0b0101)); // R + assert_eq!(m(0b0111), None); // V + assert_eq!(m(0b1111), None); // N + // the stricter flag wins when both are set + assert_eq!(masked_state(0b0101, false, true, true), None); } #[test] diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index dbf2dc74..65f52ad9 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -279,7 +279,7 @@ mod tests { /// something these tests need to exercise). fn tally_2genomes(pairs: &[(u8, u8)], n_shared: usize) -> PairwiseTally { // No masking requested (`free_loss = no_ambiguity = false`) — - // `masked_state(m, false, false)` is always `Some(m)`, so the raw + // `masked_state(m, false, false, false)` is always `Some(m)`, so the raw // mask is exactly what `subsample::sample_layer` would have // computed here. let mut families = Vec::new(); diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index 3a32181a..2e477d4d 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -110,6 +110,12 @@ pub struct EntropyBias { pub struct SamplingSpec { pub free_loss: bool, pub no_ambiguity: bool, + /// `--only-diploid`: like `no_ambiguity` but only genomes carrying three + /// or four members of a family become missing. Applied by the post-hoc + /// masking only — the persisted entropy/quartet/pair-weight annexes are + /// read under the `no_ambiguity` convention (`false` here), which is a + /// superset for the quartet pre-filter and only a weight otherwise. + pub only_diploid: bool, pub quartet: bool, /// `--pair-balance`: weight each family by the mean, over the pairs of /// genomes carrying it, of 1 / (number of families the pair shares), so @@ -488,7 +494,7 @@ fn sample_layer( ); let masked: Vec> = genome_mask .iter() - .map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity)) + .map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity, spec.only_diploid)) .collect(); if survives_masking(&masked, excluded) && (!spec.quartet || survives_quartet(&masked, excluded)) -- 2.54.1 From 5368c988752910b3a8e1b0e9ae9a4d4c2e0b3db5 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 22 Oct 2026 16:38:55 +0200 Subject: [PATCH 3/4] refactor: replace no_ambiguity bool with AmbiguityPolicy enum Replace the binary `no_ambiguity` flag with an `AmbiguityPolicy` enum (`Keep`, `Forbid`, `OnlyDiploid`) to support multiple ambiguity conventions. This change updates CLI arguments, internal logic, and persistent storage formats to distinguish between keeping all genomes, forbidding ambiguous ones, or restricting analysis to diploid sites only. - Add `src/obikphylo/src/siblings/ambiguity.rs` defining the `AmbiguityPolicy` enum with variants `Keep`, `Forbid`, and `OnlyDiploid`, including methods for header flags and determining if a genome is dropped. - Update `PhyloArgs` in `args.rs` to derive `AmbiquityPolicy` from CLI flags and construct `AnnexSelection` for sibling annexes. - Modify `SamplingSpec` in `subsample.rs` to store `ambiguity: AmbiguityPolicy` instead of `no_ambiguity`, updating masking and scalar value logic accordingly. - Update `SiblingExt` in `sibling_ext.rs` so `build_sibling_annex` accepts an `AnnexSelection` to build only requested derived annexes, and `family_overlap` uses the specific policy. - Refactor `FamilyOverlap` in `family_overlap.rs` to store and use `AmbiguityPolicy`, replacing boolean checks with policy-based carrier determination. - Update `FamilyScalarAnnex` in `family_scalar_annex.rs` to store `AmbiguityPolicy` in headers, enabling distinct file names for each policy variant. - Modify `quartet_annex.rs` to generate distinct file names based on `AmbiguityPolicy`, ensuring no overlap between Keep, Forbid, and OnlyDiploid annexes. - Update `family_annexes.rs` to use `AnnexSelection` for building entropy, quartet, and pair weight annexes under specific policies instead of a single boolean convention. - Refactor algorithm modules (`entropy.rs`, `masking.rs`, `pair_weight.rs`, `quartet.rs`, `snp_distance.rs`) to accept `AmbiguityPolicy` arguments, replacing boolean checks with policy-specific logic (e.g., `OnlyDiploid` masking). - Update documentation in `phylo.md` and `evolutionary_distances.md` to describe the new `--only-diploid` option, the behavior of `--sibling-annex` with specific policies, and the separate storage of entropies per convention. --- DevDocMD/theory/evolutionary_distances.md | 29 +++-- UserDocMD/usage/phylo.md | 8 +- src/obikmer/src/cmd/phylo/args.rs | 26 +++- src/obikmer/src/cmd/phylo/mod.rs | 9 +- .../src/siblings/algorithms/entropy.rs | 52 +++++--- .../src/siblings/algorithms/family_annexes.rs | 118 +++++++++++------- .../src/siblings/algorithms/masking.rs | 44 ++++--- .../siblings/algorithms/minorant_selection.rs | 12 +- src/obikphylo/src/siblings/algorithms/mod.rs | 1 + .../src/siblings/algorithms/pair_weight.rs | 24 ++-- .../src/siblings/algorithms/quartet.rs | 8 +- .../src/siblings/algorithms/snp_distance.rs | 2 +- .../src/siblings/algorithms/subsample.rs | 39 +++--- src/obikphylo/src/siblings/ambiguity.rs | 96 ++++++++++++++ .../siblings/extensions/sibling_builder.rs | 20 +-- .../src/siblings/extensions/sibling_ext.rs | 47 ++++--- src/obikphylo/src/siblings/family_overlap.rs | 90 +++++++------ .../src/siblings/family_scalar_annex.rs | 80 ++++++------ src/obikphylo/src/siblings/helpers.rs | 6 +- src/obikphylo/src/siblings/mod.rs | 4 +- src/obikphylo/src/siblings/quartet_annex.rs | 40 +++--- 21 files changed, 477 insertions(+), 278 deletions(-) create mode 100644 src/obikphylo/src/siblings/ambiguity.rs diff --git a/DevDocMD/theory/evolutionary_distances.md b/DevDocMD/theory/evolutionary_distances.md index a3684aa7..0f4b34e8 100644 --- a/DevDocMD/theory/evolutionary_distances.md +++ b/DevDocMD/theory/evolutionary_distances.md @@ -2923,14 +2923,29 @@ for a real dosage (polyploids) would need k-mer counts (`--with-counts`), not used by the pseudo-alignment scan today. `--only-diploid` (selection option, `conflicts_with --no-ambiguity`) adds a third `masked_state` rule: `count_ones() > 2` -> missing. It lives in `SamplingSpec`/`masked_state`, i.e. the post-hoc masking and -`survives_masking`, so FASTA, Sankoff and `snp-*` see it too. The persisted -entropy/quartet/pair-weight/overlap annexes are keyed by the `no_ambiguity` -bool (two variants on disk); a third variant was not added: they are read -under `no_ambiguity = false`, which for the quartet pre-filter is a superset -of what survives the stricter masking (reasoned, not tested) and for the -entropy/pair-weight draw weights only changes the weighting. Unit-tested +`survives_masking`, so FASTA, Sankoff and `snp-*` see it too. Unit-tested (`masked_state`); the toy index has no 3/4-member cells, so the end-to-end -run is identical to the default mode. Toy index: default mode 187 +run is identical to the default mode. + +First version of this flag read the persisted annexes under the default +convention (a third variant was judged not worth it). Rejected: the draw +would then describe other genomes than the alignment, and the pair-balance +weights and entropy are computed from the carrier set. Replaced by +`AmbiguityPolicy { Keep, Forbid, OnlyDiploid }` (one type instead of +`free_loss`/`no_ambiguity`/`only_diploid` booleans in `masked_state`, +`is_carrier`, `SamplingSpec` and every annex key). Annex headers store +`AmbiguityPolicy::header_flags` (0 = Keep, 1 = Forbid, as the old bit, 2 = +OnlyDiploid), so annexes written before the change stay valid; each policy +has its own file (`*.diploid.*` for the new one). `--sibling-annex` no longer +precomputes everything (`AnnexRequest::everything` removed, a rebuild only +costs the base annex): it builds the derived annexes named by the other +options of the command (`AnnexSelection`: `--entropy`, `--quartet`, +`--pair-balance`, `--family-overlap`/`--min-shared-family`, under the one +policy given); the rest is built on demand as before. A rebuild still drops +every derived annex (their `family_idx` numbering is tied to the base one). +Checked on the toy index: `--sibling-annex` alone leaves only +`siblings.psib`; `--quartet --only-diploid` adds `quartet.diploid.pbiv` only. +Toy index: default mode 187 lines, no `ALT = .`; `--no-ambiguity` 93 lines (the sites whose polymorphism was only ambiguity codes disappear, since those cells became `?`). diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index 6f37dece..7ecc91f1 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -48,7 +48,7 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model | Option | Description | |---|---| -| `--sibling-annex` | Build (or rebuild) the sibling-count/minorant annex, then every annex derived from it (entropy, quartet, pair weight, family overlap, under both `--no-ambiguity` conventions). Every option in this section needs the sibling annex, and builds it by itself, under an exclusive lock on the index, if it is missing (as it is after `migrate-mphf`); running `--sibling-annex` first does the same work once, which is the safe way to start several `phylo` runs on the same index at the same time | +| `--sibling-annex` | Build (or rebuild) the sibling-count/minorant annex. Rebuilding it makes the annexes derived from it obsolete, so they are dropped; then only the derived annexes that the other options of the same command read are rebuilt, each under the `--no-ambiguity` / `--only-diploid` convention given: `--entropy` the entropy annex, `--quartet` the quartet annex, `--pair-balance` the pair-weight and family-overlap annexes, `--family-overlap` or `--min-shared-family` the family-overlap annex. For example `--sibling-annex --quartet --only-diploid` prepares exactly the quartet annex of that convention, and `--sibling-annex` alone prepares none. An annex that is not prepared is built later, on demand, by the computation that reads it. Every option in this section needs the sibling annex, and builds it by itself, under an exclusive lock on the index, if it is missing (as it is after `migrate-mphf`); running `--sibling-annex` first does the same work once, which is the safe way to start several `phylo` runs on the same index at the same time | | `--sibling-stats` | Write `_siblings.csv`: the family-size distribution, per genome and globally | | `--sibling-hist` | Print the global family-size histogram (1-4 members) only | | `--family-overlap` | Write `_family_overlap.csv`: for every genome pair, how many variable families both genomes carry a call for | @@ -146,7 +146,7 @@ With `--n-sites all` (or, for a `snp-*` `--distance` value, without `--n-sites`) A family carried by fewer than four genomes, or whose genomes all share one state except a single variant, fits every tree topology equally well and so tells a tree nothing: to separate two pairs of genomes from each other (`AB|CD`) it takes at least two distinct states, each carried by at least two genomes. `--quartet` restricts the draw to the families that show this pattern. Absence is never a state here: a genome that does not carry the family does not count, with or without `--free-loss`. A genome carrying several members of the family counts as a carrier of its own combination of bases, unless `--no-ambiguity` is given, in which case it is left out, as in the alignment. -The restriction is applied before the draw, from a persisted annex (one bit per variable family, built the first time `--quartet` is used with a given `--no-ambiguity` convention, in one pass shared with the other annexes, and built for both conventions by `--sibling-annex`), then checked exactly on each family drawn, taking `--exclude-genome` into account. The quota given to `--n-sites` is therefore spent only on usable families. +The restriction is applied before the draw, from a persisted annex (one bit per variable family, built the first time `--quartet` is used with a given `--no-ambiguity` / `--only-diploid` convention, in one pass shared with the other annexes, or ahead of time by `--sibling-annex --quartet`), then checked exactly on each family drawn, taking `--exclude-genome` into account. The quota given to `--n-sites` is therefore spent only on usable families. ### Describing a draw: `--sample-stats`, `--sample-plan` @@ -170,7 +170,7 @@ A call is any state other than the empty one: absence is never a call, with or w Two genomes that are distantly related share few families, so a uniform draw covers such a pair with few sites, far fewer than a close pair: the pairs that matter for the deep branches of a tree are the worst represented. `--pair-balance` compensates. Each family is weighted by the mean, over the pairs of genomes carrying it, of 1 divided by the number of families that pair shares across the whole index (the matrix `--family-overlap` writes): a family carried by a rarely-sharing pair weighs more. It is the mean, not the sum, so a family carried by most genomes — in skimming data often a high-copy region — is not favoured merely for covering many pairs. -Each layer's share of the draw is proportional to the total weight of its families, and a family's chance of being drawn is proportional to its own weight. The weights are persisted (one value per variable family, built the first time `--pair-balance` is used with a given `--no-ambiguity` convention, after the family-overlap matrix it depends on, and by `--sibling-annex` for both conventions). They combine with `--entropy` and `--quartet`: the weight multiplies the entropy kernel, and `--quartet` restricts the families first. +Each layer's share of the draw is proportional to the total weight of its families, and a family's chance of being drawn is proportional to its own weight. The weights are persisted (one value per variable family, built the first time `--pair-balance` is used with a given `--no-ambiguity` / `--only-diploid` convention, after the family-overlap matrix it depends on, or ahead of time by `--sibling-annex --pair-balance`). They combine with `--entropy` and `--quartet`: the weight multiplies the entropy kernel, and `--quartet` restricts the families first. ### `--entropy MU` / `--entropy-sd SIGMA`: biasing the sample toward informative families @@ -178,7 +178,7 @@ By default, `--n-sites N` draws families uniformly. With `--entropy`/`--entropy- The first `phylo` run on a given index that uses `--entropy`/`--entropy-sd` pays a one-time extra cost (every candidate family's entropy is computed once and saved alongside the index); later runs, even with different `MU`/`SIGMA`, reuse that saved data. -The entropy that weights the draw follows `--no-ambiguity`, so that it describes the same genomes as the alignment. Without the flag, every genome carrying the family counts, and a genome carrying several members of it is a state of its own (`entropy15`). With the flag, only the genomes carrying exactly one member count, since the others become missing data (`entropy4`). The two entropies are saved separately: switching `--no-ambiguity` builds the other one once, without discarding the first, and `--sibling-annex` builds both. +The entropy that weights the draw follows `--no-ambiguity` and `--only-diploid`, so that it describes the same genomes as the alignment. By default, every genome carrying the family counts, and a genome carrying several members of it is a state of its own (`entropy15`). With `--no-ambiguity`, only the genomes carrying exactly one member count, since the others become missing data (`entropy4`). With `--only-diploid`, the genomes carrying one or two members count and those carrying three or four become missing data. The three entropies are saved separately: switching convention builds the missing one once, without discarding the others, and `--sibling-annex --entropy` builds the one of the convention given. ## Sankoff calibration and phylogenetic exports diff --git a/src/obikmer/src/cmd/phylo/args.rs b/src/obikmer/src/cmd/phylo/args.rs index 8a9d1d49..11dd2632 100644 --- a/src/obikmer/src/cmd/phylo/args.rs +++ b/src/obikmer/src/cmd/phylo/args.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use clap::Args; +use obikphylo::siblings::{AmbiguityPolicy, AnnexSelection}; use obikphylo::{DistanceMetric, GammaShape, SnpDistanceKind}; /// `--gamma-shape` CLI value: a fixed `alpha`, or `auto`/`estimate` to fit @@ -223,7 +224,13 @@ pub struct PhyloArgs { /// Build (or rebuild) the sibling-count/minorant annex — independent of /// the distance metric below, meant to be run routinely, ahead of any - /// SNP-family distance computation that will later consume it. + /// SNP-family distance computation that will later consume it. Rebuilding + /// it invalidates the annexes derived from it; those are rebuilt here + /// only for the options given alongside (`--entropy` the entropy annex, + /// `--quartet` the quartet one, `--pair-balance` the pair-weight and + /// family-overlap ones, `--family-overlap`/`--min-shared-family` the + /// family-overlap one), each under the `--no-ambiguity`/`--only-diploid` + /// convention given — anything not asked for is built later, on demand. #[arg(long)] pub sibling_annex: bool, @@ -526,6 +533,23 @@ pub struct PhyloArgs { } impl PhyloArgs { + /// How genomes carrying several members of a family are treated. + pub fn ambiguity(&self) -> AmbiguityPolicy { + AmbiguityPolicy::from_flags(self.no_ambiguity, self.only_diploid) + } + + /// The derived annexes the options of this invocation read, all under + /// [`ambiguity`](Self::ambiguity). + pub fn annex_selection(&self) -> AnnexSelection { + AnnexSelection { + policy: self.ambiguity(), + entropy: self.entropy.is_some(), + quartet: self.quartet, + overlap: self.family_overlap || self.min_shared_family.is_some() || self.pair_balance, + pair_weight: self.pair_balance, + } + } + /// Whether a pseudo-alignment must be drawn and written (`--pseudo-alignment`, /// or any of its output-format flags). pub fn alignment_output(&self) -> bool { diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 78f43ce9..1497713c 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -85,7 +85,7 @@ pub fn run(args: PhyloArgs) { // so a concurrent run on the same index can't corrupt these writes. info!("building sibling-count/minorant annex"); let t = Stage::start("sibling_annex"); - cache.build_sibling_annex().unwrap_or_else(|e| { + cache.build_sibling_annex(&args.annex_selection()).unwrap_or_else(|e| { eprintln!("error building sibling annex: {e}"); std::process::exit(1); }); @@ -155,7 +155,7 @@ pub fn run(args: PhyloArgs) { // ── Family Overlap matrix (`--family-overlap`) ────────────────────────────── if args.family_overlap { let t = Stage::start("family_overlap"); - let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| { + let overlap = cache.family_overlap(args.ambiguity()).unwrap_or_else(|e| { eprintln!("error computing family overlap: {e}"); std::process::exit(1); }); @@ -217,7 +217,7 @@ pub fn run(args: PhyloArgs) { // `args::PhyloArgs::min_shared_family`'s own docs on why). let snp_exclude_mask: Vec = match args.min_shared_family { Some(threshold) => { - let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| { + let overlap = cache.family_overlap(args.ambiguity()).unwrap_or_else(|e| { eprintln!("error computing family overlap: {e}"); std::process::exit(1); }); @@ -253,8 +253,7 @@ pub fn run(args: PhyloArgs) { }; let sampling = SamplingSpec { free_loss: args.free_loss, - no_ambiguity: args.no_ambiguity, - only_diploid: args.only_diploid, + ambiguity: args.ambiguity(), quartet: args.quartet, pair_balance: args.pair_balance, entropy_bias, diff --git a/src/obikphylo/src/siblings/algorithms/entropy.rs b/src/obikphylo/src/siblings/algorithms/entropy.rs index 0e88bc6f..52cd09db 100644 --- a/src/obikphylo/src/siblings/algorithms/entropy.rs +++ b/src/obikphylo/src/siblings/algorithms/entropy.rs @@ -16,10 +16,16 @@ /// real minorant, guarded against rather than dividing by zero). Second /// value: the denominator (`n_genomes_present`) used, for traceability. pub(crate) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> { + family_entropy_with(genome_mask, AmbiguityPolicy::Keep) +} + +/// [`family_entropy`] over the genomes `policy` keeps as carriers +/// ([`is_carrier`]). +pub(crate) fn family_entropy_with(genome_mask: &[u8], policy: AmbiguityPolicy) -> Option<(f64, usize)> { let mut freq = [0u32; 16]; // index 0 (∅) never incremented, kept for direct `m as usize` indexing let mut present = 0u32; for &m in genome_mask { - if m != 0 { + if is_carrier(m, policy) { freq[m as usize] += 1; present += 1; } @@ -70,6 +76,9 @@ pub(crate) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> { Some((h, n_single as usize)) } +use crate::siblings::AmbiguityPolicy; +use crate::siblings::helpers::is_carrier; + #[cfg(test)] mod tests { use super::*; @@ -104,10 +113,16 @@ mod tests { #[test] fn entropy_under_selects_the_counting_convention() { let masks = [0b0001u8, 0b0001, 0b0011]; // the last genome carries two members - assert_eq!(family_entropy_under(&masks, true).unwrap().1, 2); - assert_eq!(family_entropy_under(&masks, false).unwrap().1, 3); - assert_eq!(family_entropy_under(&masks, true).unwrap().0, 0.0); - assert!(family_entropy_under(&masks, false).unwrap().0 > 0.0); + use AmbiguityPolicy::{Forbid, Keep, OnlyDiploid}; + assert_eq!(family_entropy_under(&masks, Forbid).unwrap().1, 2); + assert_eq!(family_entropy_under(&masks, Keep).unwrap().1, 3); + assert_eq!(family_entropy_under(&masks, OnlyDiploid).unwrap().1, 3); + assert_eq!(family_entropy_under(&masks, Forbid).unwrap().0, 0.0); + assert!(family_entropy_under(&masks, Keep).unwrap().0 > 0.0); + // three members: kept by Keep, dropped by OnlyDiploid + let masks = [0b0001u8, 0b0001, 0b0111]; + assert_eq!(family_entropy_under(&masks, Keep).unwrap().1, 3); + assert_eq!(family_entropy_under(&masks, OnlyDiploid).unwrap().1, 2); } #[test] @@ -117,21 +132,24 @@ mod tests { } } -/// Shannon entropy (bits) of one family under the counting convention -/// `--no-ambiguity` selects: -/// - `false`: [`family_entropy`] — every genome carrying the family counts, +/// Shannon entropy (bits) of one family under the counting convention an +/// [`AmbiguityPolicy`] selects: +/// - `Keep`: [`family_entropy`] — every genome carrying the family counts, /// a genome with several members being a state of its own; -/// - `true`: [`family_entropy_4`] — only the genomes carrying exactly one -/// member count, the others being missing data, exactly as the alignment -/// itself will see them. Over those genomes the 15-state entropy can only -/// ever take the four single-base states, so the two definitions coincide. +/// - `Forbid` (`--no-ambiguity`): [`family_entropy_4`] — only the genomes +/// carrying exactly one member count, the others being missing data, +/// exactly as the alignment itself will see them. Over those genomes the +/// 15-state entropy can only ever take the four single-base states, so the +/// two definitions coincide; +/// - `OnlyDiploid`: the 15-state entropy over the genomes with one or two +/// members (ten states), the others being missing data. /// /// Second value: the number of genomes the entropy was computed over. -pub(crate) fn family_entropy_under(genome_mask: &[u8], no_ambiguity: bool) -> Option<(f64, usize)> { - if no_ambiguity { - family_entropy_4(genome_mask) - } else { - family_entropy(genome_mask) +pub(crate) fn family_entropy_under(genome_mask: &[u8], policy: AmbiguityPolicy) -> Option<(f64, usize)> { + match policy { + AmbiguityPolicy::Keep => family_entropy(genome_mask), + AmbiguityPolicy::Forbid => family_entropy_4(genome_mask), + AmbiguityPolicy::OnlyDiploid => family_entropy_with(genome_mask, policy), } } diff --git a/src/obikphylo/src/siblings/algorithms/family_annexes.rs b/src/obikphylo/src/siblings/algorithms/family_annexes.rs index bdce53af..56e0454e 100644 --- a/src/obikphylo/src/siblings/algorithms/family_annexes.rs +++ b/src/obikphylo/src/siblings/algorithms/family_annexes.rs @@ -21,34 +21,58 @@ use super::family_scan::{Selection, scan_layer_families}; use super::masking::is_quartet_informative; use super::minorant_selection::non_monomorphic_reindex_layer; use super::pair_weight::pair_balance_weight; +use crate::siblings::AmbiguityPolicy; use crate::siblings::quartet_annex::quartet_annex_file_name; use crate::siblings::{ FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind, family_overlap_file_name, }; -/// Which annexes a caller needs, each listed once per `--no-ambiguity` -/// counting convention (`false` = genomes carrying several members count as a -/// state of their own, `true` = they are left out) — an empty list asks for -/// none. A pair weight is computed from the family-overlap annex of its own -/// convention, so asking for one asks for the other. +/// Which annexes a caller needs, each listed once per ambiguity policy (how +/// genomes carrying several members of a family are counted) — an empty list +/// asks for none. A pair weight is computed from the family-overlap annex of +/// its own policy, so asking for one asks for the other. #[derive(Clone, Debug, Default)] pub(crate) struct AnnexRequest { - pub entropy: Vec, - pub quartet: Vec, - pub overlap: Vec, - pub pair_weight: Vec, + pub entropy: Vec, + pub quartet: Vec, + pub overlap: Vec, + pub pair_weight: Vec, +} + +/// The derived annexes a run asks to have ready, all under the one ambiguity +/// `policy` of that run (`--sibling-annex` together with the options that +/// say which annexes, and under which convention, the following computations +/// will read). Nothing is built that is not asked for. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnnexSelection { + pub policy: AmbiguityPolicy, + pub entropy: bool, + pub quartet: bool, + pub overlap: bool, + pub pair_weight: bool, +} + +impl AnnexSelection { + pub(crate) fn request(&self) -> AnnexRequest { + let one = |wanted: bool| if wanted { vec![self.policy] } else { Vec::new() }; + AnnexRequest { + entropy: one(self.entropy), + quartet: one(self.quartet), + overlap: one(self.overlap), + pair_weight: one(self.pair_weight), + } + } + + /// Whether any derived annex is asked for. + pub fn is_empty(&self) -> bool { + !(self.entropy || self.quartet || self.overlap || self.pair_weight) + } } impl AnnexRequest { - /// Everything under both conventions — what `--sibling-annex` precomputes. - pub(crate) fn everything() -> Self { - let both = vec![false, true]; - Self { entropy: both.clone(), quartet: both.clone(), overlap: both.clone(), pair_weight: both } - } - /// The overlap conventions to ensure: the requested ones plus every one a /// requested pair weight is computed from, each once. - pub(crate) fn overlap_conventions(&self) -> Vec { + pub(crate) fn overlap_conventions(&self) -> Vec { dedup(self.overlap.iter().chain(&self.pair_weight).copied()) } @@ -59,25 +83,25 @@ impl AnnexRequest { let overlap = self .overlap_conventions() .into_iter() - .all(|nb| cache.index().dir().join(family_overlap_file_name(nb)).exists()); + .all(|p| cache.index().dir().join(family_overlap_file_name(p)).exists()); let pair_weight = self.pair_weight_conventions(); overlap && cache.iter_indexed().all(|(_, _, layer)| { let has = |name: &str| layer.dir().join(name).exists(); - self.entropy.iter().all(|&nb| has(ScalarKind::Entropy.file_name(nb))) - && self.quartet.iter().all(|&nb| has(quartet_annex_file_name(nb))) - && pair_weight.iter().all(|&nb| has(ScalarKind::PairWeight.file_name(nb))) + self.entropy.iter().all(|&p| has(ScalarKind::Entropy.file_name(p))) + && self.quartet.iter().all(|&p| has(quartet_annex_file_name(p))) + && pair_weight.iter().all(|&p| has(ScalarKind::PairWeight.file_name(p))) }) } /// The pair-weight conventions requested, each once. - pub(crate) fn pair_weight_conventions(&self) -> Vec { + pub(crate) fn pair_weight_conventions(&self) -> Vec { dedup(self.pair_weight.iter().copied()) } } -fn dedup(variants: impl IntoIterator) -> Vec { - let mut v: Vec = variants.into_iter().collect(); +fn dedup(variants: impl IntoIterator) -> Vec { + let mut v: Vec = variants.into_iter().collect(); v.sort_unstable(); v.dedup(); v @@ -102,19 +126,21 @@ pub(crate) fn ensure_layer_family_annexes( n_genomes: usize, fast_mode: bool, request: &AnnexRequest, - overlap_accs: &mut [(bool, FamilyOverlapAccumulator)], + overlap_accs: &mut [(AmbiguityPolicy, FamilyOverlapAccumulator)], ) -> OKIResult<()> { let layer = cache .get_layer(partition, layer_idx) .expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)"); - let missing = |name: &dyn Fn(bool) -> &'static str, variants: &[bool]| -> Vec<(bool, PathBuf)> { + let missing = |name: &dyn Fn(AmbiguityPolicy) -> &'static str, + variants: &[AmbiguityPolicy]| + -> Vec<(AmbiguityPolicy, PathBuf)> { dedup(variants.iter().copied()) .into_iter() - .map(|no_ambiguity| (no_ambiguity, layer.dir().join(name(no_ambiguity)))) + .map(|policy| (policy, layer.dir().join(name(policy)))) .filter(|(_, path)| !path.exists()) .collect() }; - let missing_entropy = missing(&|nb| ScalarKind::Entropy.file_name(nb), &request.entropy); + let missing_entropy = missing(&|p| ScalarKind::Entropy.file_name(p), &request.entropy); let missing_quartet = missing(&quartet_annex_file_name, &request.quartet); if missing_entropy.is_empty() && missing_quartet.is_empty() && overlap_accs.is_empty() { return Ok(()); @@ -124,16 +150,16 @@ pub(crate) fn ensure_layer_family_annexes( let eligible: HashSet = reindex.keys().copied().collect(); let mut entropy_builders = missing_entropy .iter() - .map(|(no_ambiguity, path)| { - FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *no_ambiguity) - .map(|b| (*no_ambiguity, b)) + .map(|(policy, path)| { + FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *policy) + .map(|b| (*policy, b)) }) .collect::>>() .map_err(OKIError::Io)?; let mut quartet_builders = missing_quartet .iter() - .map(|(no_ambiguity, path)| { - PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*no_ambiguity, b)) + .map(|(policy, path)| { + PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*policy, b)) }) .collect::>>() .map_err(OKIError::Io)?; @@ -151,13 +177,13 @@ pub(crate) fn ensure_layer_family_annexes( "Selection::Some(eligible) must only visit non-monomorphic minorants" ); let compact = reindex[&family_idx]; - for (no_ambiguity, builder) in entropy_builders.iter_mut() { - if let Some((h, _)) = family_entropy_under(genome_mask, *no_ambiguity) { + for (policy, builder) in entropy_builders.iter_mut() { + if let Some((h, _)) = family_entropy_under(genome_mask, *policy) { builder.set(compact, h as f32); } } - for (no_ambiguity, builder) in quartet_builders.iter_mut() { - builder.set(compact, is_quartet_informative(genome_mask, *no_ambiguity)); + for (policy, builder) in quartet_builders.iter_mut() { + builder.set(compact, is_quartet_informative(genome_mask, *policy)); } for (_, acc) in overlap_accs.iter_mut() { acc.add_family(genome_mask); @@ -175,7 +201,7 @@ pub(crate) fn ensure_layer_family_annexes( } /// Builds the pair-weight annexes of one layer that don't exist yet, one per -/// `(no_ambiguity, inverse overlap matrix)` in `matrices` (see +/// `(policy, inverse overlap matrix)` in `matrices` (see /// [`super::pair_weight::inverse_overlap_matrix`]), in one scan. A no-op (no /// scan) when they all exist. Needs the family-overlap annex of each /// convention to be complete: run after the sweep that builds it. @@ -185,14 +211,14 @@ pub(crate) fn ensure_layer_pair_weight_annexes( layer_idx: usize, n_genomes: usize, fast_mode: bool, - matrices: &[(bool, Vec)], + matrices: &[(AmbiguityPolicy, Vec)], ) -> OKIResult<()> { let layer = cache .get_layer(partition, layer_idx) .expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)"); - let pending: Vec<&(bool, Vec)> = matrices + let pending: Vec<&(AmbiguityPolicy, Vec)> = matrices .iter() - .filter(|(no_ambiguity, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)).exists()) + .filter(|(policy, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*policy)).exists()) .collect(); if pending.is_empty() { return Ok(()); @@ -202,10 +228,10 @@ pub(crate) fn ensure_layer_pair_weight_annexes( let eligible: HashSet = reindex.keys().copied().collect(); let mut builders = pending .iter() - .map(|(no_ambiguity, inverse)| { - let path = layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)); - FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *no_ambiguity) - .map(|b| (*no_ambiguity, inverse, b)) + .map(|(policy, inverse)| { + let path = layer.dir().join(ScalarKind::PairWeight.file_name(*policy)); + FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *policy) + .map(|b| (*policy, inverse, b)) }) .collect::>>() .map_err(OKIError::Io)?; @@ -219,8 +245,8 @@ pub(crate) fn ensure_layer_pair_weight_annexes( &Selection::Some(&eligible), |family_idx, _kmer, _mask, genome_mask| { let compact = reindex[&family_idx]; - for (no_ambiguity, inverse, builder) in builders.iter_mut() { - builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse)); + for (policy, inverse, builder) in builders.iter_mut() { + builder.set(compact, pair_balance_weight(genome_mask, *policy, inverse)); } }, )?; diff --git a/src/obikphylo/src/siblings/algorithms/masking.rs b/src/obikphylo/src/siblings/algorithms/masking.rs index 7314fbf4..de5b24e7 100644 --- a/src/obikphylo/src/siblings/algorithms/masking.rs +++ b/src/obikphylo/src/siblings/algorithms/masking.rs @@ -17,25 +17,22 @@ //! waste sample quota on sites that die anyway once the excluded genome's //! data is dropped. +use crate::siblings::AmbiguityPolicy; use crate::siblings::helpers::is_carrier; /// This genome's masked character state, or `None` for "?" (missing). `m` /// is one `genome_mask` entry (bit `b` set iff this genome carries the /// family member whose own canonical central base is `b`; `0` = no member /// present in this genome, `∅` — a real state on its own unless -/// `free_loss` says otherwise). `only_diploid` is the milder sibling of -/// `no_ambiguity`: it keeps the genomes carrying one or two members (a base -/// or a two-base IUPAC code) and drops those carrying three or four. +/// `free_loss` says otherwise). `policy` makes missing the genomes carrying +/// too many members ([`AmbiguityPolicy::drops`]). #[inline] -pub(crate) fn masked_state(m: u8, free_loss: bool, no_ambiguity: bool, only_diploid: bool) -> Option { +pub(crate) fn masked_state(m: u8, free_loss: bool, policy: AmbiguityPolicy) -> Option { if free_loss && m == 0 { return None; // absence -> missing, not a real state } - if no_ambiguity && m.count_ones() > 1 { - return None; // ambiguous (>1 member in this genome) -> missing - } - if only_diploid && m.count_ones() > 2 { - return None; // more than a heterozygote's two members -> missing + if policy.drops(m) { + return None; // ambiguous beyond what the policy keeps -> missing } Some(m) } @@ -90,10 +87,10 @@ pub(crate) fn has_bipartition(counts: &[u32; 16]) -> bool { /// bipartition, never create one — so `false` is final, and `true` is only /// a necessary condition, re-checked exactly by [`survives_quartet`] once /// the family is resolved. This is what the persisted quartet annex stores. -pub(crate) fn is_quartet_informative(genome_mask: &[u8], no_ambiguity: bool) -> bool { +pub(crate) fn is_quartet_informative(genome_mask: &[u8], policy: AmbiguityPolicy) -> bool { let mut counts = [0u32; 16]; for &m in genome_mask { - if !is_carrier(m, no_ambiguity) { + if !is_carrier(m, policy) { continue; } counts[(m & 0b1111) as usize] += 1; @@ -124,31 +121,31 @@ mod quartet_tests { #[test] fn two_pairs_of_distinct_states_is_informative() { - assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], false)); - assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], true)); + assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], AmbiguityPolicy::Keep)); + assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], AmbiguityPolicy::Forbid)); } #[test] fn singleton_variant_is_not_informative() { // three genomes in one state plus one lone variant: fits any topology - assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], false)); + assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], AmbiguityPolicy::Keep)); // fewer than four carriers can never be informative - assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], false)); - assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], false)); + assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], AmbiguityPolicy::Keep)); + assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], AmbiguityPolicy::Keep)); } #[test] fn absent_genomes_are_not_a_state() { // two carriers of one base, two absences: absence must not form the 2nd pair - assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], false)); + assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], AmbiguityPolicy::Keep)); } #[test] fn no_ambiguity_drops_multi_member_genomes_from_the_count() { // the two-member genomes form a pair of their own state without the flag let masks = [0b0001u8, 0b0001, 0b0011, 0b0011]; - assert!(is_quartet_informative(&masks, false)); - assert!(!is_quartet_informative(&masks, true)); + assert!(is_quartet_informative(&masks, AmbiguityPolicy::Keep)); + assert!(!is_quartet_informative(&masks, AmbiguityPolicy::Forbid)); } #[test] @@ -227,19 +224,20 @@ mod tests { /// `subsample::sample_layer` does once, up front, before calling /// [`survives_masking`]. fn masked(raw: &[u8], free_loss: bool, no_ambiguity: bool) -> Vec> { - raw.iter().map(|&m| masked_state(m, free_loss, no_ambiguity, false)).collect() + let policy = AmbiguityPolicy::from_flags(no_ambiguity, false); + raw.iter().map(|&m| masked_state(m, free_loss, policy)).collect() } #[test] fn only_diploid_keeps_up_to_two_members() { - let m = |raw: u8| masked_state(raw, false, false, true); + let m = |raw: u8| masked_state(raw, false, AmbiguityPolicy::OnlyDiploid); assert_eq!(m(0b0000), Some(0)); // absence stays a state (no --free-loss) assert_eq!(m(0b0100), Some(0b0100)); assert_eq!(m(0b0101), Some(0b0101)); // R assert_eq!(m(0b0111), None); // V assert_eq!(m(0b1111), None); // N - // the stricter flag wins when both are set - assert_eq!(masked_state(0b0101, false, true, true), None); + // `Forbid` is stricter: a heterozygote is missing there + assert_eq!(masked_state(0b0101, false, AmbiguityPolicy::Forbid), None); } #[test] diff --git a/src/obikphylo/src/siblings/algorithms/minorant_selection.rs b/src/obikphylo/src/siblings/algorithms/minorant_selection.rs index d1d224b7..79e621c5 100644 --- a/src/obikphylo/src/siblings/algorithms/minorant_selection.rs +++ b/src/obikphylo/src/siblings/algorithms/minorant_selection.rs @@ -11,7 +11,7 @@ use std::path::Path; use obicompactvec::TempBitVecBuilder; use obikindex::{OKIError, OKIResult}; -use crate::siblings::{FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex}; +use crate::siblings::{AmbiguityPolicy, FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex}; pub(crate) fn is_non_monomorphic_minorant(mask: FamilyMask) -> bool { mask.is_minorant() && mask.siblings() >= 1 @@ -92,7 +92,7 @@ pub(crate) fn eligible_bitset_layer(layer_dir: &Path) -> OKIResult OKIResult OKIResult + use<>> { let annex = SiblingAnnex::open_in(layer_dir).map_err(OKIError::Io)?; let scalars = - FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(no_ambiguity)), kind).map_err(OKIError::Io)?; - if scalars.no_ambiguity() != no_ambiguity { + FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(policy)), kind).map_err(OKIError::Io)?; + if scalars.policy() != policy { return Err(OKIError::Io(std::io::Error::new( std::io::ErrorKind::InvalidData, - format!("{kind:?} annex was built under the other --no-ambiguity convention"), + format!("{kind:?} annex was built under {:?}, not {policy:?}", scalars.policy()), ))); } diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 7027c413..197e0f34 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -49,6 +49,7 @@ pub use subsample::{EntropyBias, LayerReport, SamplePlan, SamplingSpec, Survivin pub(crate) use annex::build_layer_sibling_annex; pub(crate) use entropy_scan::{EntropySummary, scan_family_entropies}; +pub use family_annexes::AnnexSelection; pub(crate) use family_annexes::{ AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes, }; diff --git a/src/obikphylo/src/siblings/algorithms/pair_weight.rs b/src/obikphylo/src/siblings/algorithms/pair_weight.rs index 3ba10eba..9c3b7248 100644 --- a/src/obikphylo/src/siblings/algorithms/pair_weight.rs +++ b/src/obikphylo/src/siblings/algorithms/pair_weight.rs @@ -13,7 +13,7 @@ //! Values are only meaningful relative to each other; the sampler rescales //! them per layer (`SamplingSpec::pair_balance`). -use crate::siblings::FamilyOverlap; +use crate::siblings::{AmbiguityPolicy, FamilyOverlap}; use crate::siblings::helpers::is_carrier; /// `1 / O` for every pair of genomes, as a dense symmetric `n × n` matrix @@ -38,16 +38,16 @@ pub(crate) fn inverse_overlap_matrix(overlap: &FamilyOverlap) -> Vec { /// The pair-balance weight of one family from its raw per-genome masks: /// mean of `inv_overlap` over the pairs of carriers (carriers per -/// [`is_carrier`], under `no_ambiguity`). `0.0` for a family with fewer than +/// [`is_carrier`], under `policy`). `0.0` for a family with fewer than /// two carriers — it covers no pair, so it can balance none. pub(crate) fn pair_balance_weight( genome_mask: &[u8], - no_ambiguity: bool, + policy: AmbiguityPolicy, inv_overlap: &[f32], ) -> f32 { let n = genome_mask.len(); let carriers: Vec = - (0..n).filter(|&g| is_carrier(genome_mask[g], no_ambiguity)).collect(); + (0..n).filter(|&g| is_carrier(genome_mask[g], policy)).collect(); let k = carriers.len(); if k < 2 { return 0.0; @@ -83,30 +83,30 @@ mod tests { #[test] fn single_pair_weight_is_the_inverse_overlap() { - let w = pair_balance_weight(&[1, 1, 0, 0], false, &matrix()); + let w = pair_balance_weight(&[1, 1, 0, 0], AmbiguityPolicy::Keep, &matrix()); assert!((w - 0.5).abs() < 1e-6); } #[test] fn weight_is_the_mean_not_the_sum_over_pairs() { // carriers 0, 1, 2: pairs (0,1)=1/2, (0,2)=1/4, (1,2)=1 -> mean 7/12 - let w = pair_balance_weight(&[1, 1, 1, 0], false, &matrix()); + let w = pair_balance_weight(&[1, 1, 1, 0], AmbiguityPolicy::Keep, &matrix()); assert!((w - 7.0 / 12.0).abs() < 1e-6); } #[test] fn fewer_than_two_carriers_weigh_nothing() { - assert_eq!(pair_balance_weight(&[1, 0, 0, 0], false, &matrix()), 0.0); - assert_eq!(pair_balance_weight(&[0, 0, 0, 0], false, &matrix()), 0.0); + assert_eq!(pair_balance_weight(&[1, 0, 0, 0], AmbiguityPolicy::Keep, &matrix()), 0.0); + assert_eq!(pair_balance_weight(&[0, 0, 0, 0], AmbiguityPolicy::Keep, &matrix()), 0.0); } #[test] fn no_ambiguity_leaves_multi_member_genomes_out_of_the_pairs() { let masks = [1u8, 1, 0b0011, 0]; // without the flag genome 2 carries the family: pairs (0,1), (0,2), (1,2) - let with = pair_balance_weight(&masks, false, &matrix()); + let with = pair_balance_weight(&masks, AmbiguityPolicy::Keep, &matrix()); // with it, only the pair (0,1) remains - let without = pair_balance_weight(&masks, true, &matrix()); + let without = pair_balance_weight(&masks, AmbiguityPolicy::Forbid, &matrix()); assert!((with - 7.0 / 12.0).abs() < 1e-6); assert!((without - 0.5).abs() < 1e-6); } @@ -114,8 +114,8 @@ mod tests { #[test] fn a_rare_pair_outweighs_a_common_one() { // genomes (0,2) share many families, (1,3) few - let common = pair_balance_weight(&[1, 0, 1, 0], false, &matrix()); - let rare = pair_balance_weight(&[0, 1, 0, 1], false, &matrix()); + let common = pair_balance_weight(&[1, 0, 1, 0], AmbiguityPolicy::Keep, &matrix()); + let rare = pair_balance_weight(&[0, 1, 0, 1], AmbiguityPolicy::Keep, &matrix()); assert!(rare > common); } } diff --git a/src/obikphylo/src/siblings/algorithms/quartet.rs b/src/obikphylo/src/siblings/algorithms/quartet.rs index b983d153..36e13baf 100644 --- a/src/obikphylo/src/siblings/algorithms/quartet.rs +++ b/src/obikphylo/src/siblings/algorithms/quartet.rs @@ -8,11 +8,11 @@ use obikindex::{OKIError, OKIResult}; use super::minorant_selection::is_non_monomorphic_minorant; use crate::siblings::quartet_annex::quartet_annex_file_name; -use crate::siblings::SiblingAnnex; +use crate::siblings::{AmbiguityPolicy, SiblingAnnex}; /// Clears, in `eligible` (indexed by `family_idx`, monomorphic minorants /// included), every family the quartet annex flags as not quartet-informative -/// under the `no_ambiguity` counting convention. The annex is indexed by the +/// under the `policy` counting convention. The annex is indexed by the /// compact non-monomorphic ordinal, so the sibling annex is walked in the same /// order to translate one numbering into the other — annex-only, no /// cross-partition resolution. The annex must already exist @@ -20,10 +20,10 @@ use crate::siblings::SiblingAnnex; pub(crate) fn restrict_to_quartet( eligible: &mut TempBitVecBuilder, layer_dir: &Path, - no_ambiguity: bool, + policy: AmbiguityPolicy, ) -> OKIResult<()> { let annex = SiblingAnnex::open_in(layer_dir).map_err(OKIError::Io)?; - let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(no_ambiguity))) + let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(policy))) .map_err(OKIError::Io)?; let mut family_idx = 0usize; diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 65f52ad9..3584c09f 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -279,7 +279,7 @@ mod tests { /// something these tests need to exercise). fn tally_2genomes(pairs: &[(u8, u8)], n_shared: usize) -> PairwiseTally { // No masking requested (`free_loss = no_ambiguity = false`) — - // `masked_state(m, false, false, false)` is always `Some(m)`, so the raw + // `masked_state(m, false, AmbiguityPolicy::Keep)` is always `Some(m)`, so the raw // mask is exactly what `subsample::sample_layer` would have // computed here. let mut families = Vec::new(); diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index 2e477d4d..6cafb5e6 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -57,7 +57,7 @@ use super::family_scan::{Selection, scan_layer_families}; use super::masking::{masked_state, survives_masking, survives_quartet}; use super::minorant_selection::{eligible_bitset_layer, iter_full_scalar}; use super::{AnnexRequest, is_fast_mode, restrict_to_quartet}; -use crate::siblings::{FamilyMask, ScalarKind}; +use crate::siblings::{AmbiguityPolicy, FamilyMask, ScalarKind}; use crate::siblings::extensions::SiblingBuilder; /// Resolution rounds a single layer's sampling may take before giving up on @@ -109,13 +109,10 @@ pub struct EntropyBias { #[derive(Clone, Copy)] pub struct SamplingSpec { pub free_loss: bool, - pub no_ambiguity: bool, - /// `--only-diploid`: like `no_ambiguity` but only genomes carrying three - /// or four members of a family become missing. Applied by the post-hoc - /// masking only — the persisted entropy/quartet/pair-weight annexes are - /// read under the `no_ambiguity` convention (`false` here), which is a - /// superset for the quartet pre-filter and only a weight otherwise. - pub only_diploid: bool, + /// How genomes carrying several members of a family are treated + /// (`--no-ambiguity` / `--only-diploid`): by the post-hoc masking, and as + /// the counting convention of the persisted annexes the draw reads. + pub ambiguity: AmbiguityPolicy, pub quartet: bool, /// `--pair-balance`: weight each family by the mean, over the pairs of /// genomes carrying it, of 1 / (number of families the pair shares), so @@ -165,17 +162,17 @@ impl SamplePlan { /// [`sample_layer`]'s own circular index sequence walks, so the streams can be /// `.zip()`-ed directly. Two more streamed passes over the layer's (small) /// annex files, not a materialised `Vec`/`HashMap` — `iter_full_scalar` -/// itself never allocates more than its own iterator state. `no_ambiguity` +/// itself never allocates more than its own iterator state. `policy` /// selects the annex built under the same counting convention as the masking /// the survivors go through afterwards. fn circular_scalar_values( layer_dir: &Path, kind: ScalarKind, start: usize, - no_ambiguity: bool, + policy: AmbiguityPolicy, ) -> OKIResult + use<>> { - let head = iter_full_scalar(layer_dir, kind, no_ambiguity)?.skip(start); - let tail = iter_full_scalar(layer_dir, kind, no_ambiguity)?.take(start); + let head = iter_full_scalar(layer_dir, kind, policy)?.skip(start); + let tail = iter_full_scalar(layer_dir, kind, policy)?.take(start); Ok(head.chain(tail)) } @@ -201,7 +198,7 @@ struct LayerPlan { /// per layer, calling `on_layer(partition, layer, survivors)` once per /// cached layer that contributed anything, with every family from that /// layer that was both drawn *and* still carries real polymorphism after -/// `free_loss`/`no_ambiguity`/`excluded` masking +/// `free_loss`/`ambiguity`/`excluded` masking /// (`algorithms::masking::survives_masking`) — i.e. every /// [`SurvivingFamily`] this produces is already a usable alignment site, no /// second resolution pass needed by the caller. `excluded[g]` (see @@ -262,10 +259,10 @@ pub(crate) fn sample_index( // layer, below, each under this run's `--no-ambiguity` counting // convention) — build any missing one now, once, rather than mid-walk. let request = AnnexRequest { - entropy: spec.entropy_bias.map(|_| vec![spec.no_ambiguity]).unwrap_or_default(), - quartet: if spec.quartet { vec![spec.no_ambiguity] } else { Vec::new() }, + entropy: spec.entropy_bias.map(|_| vec![spec.ambiguity]).unwrap_or_default(), + quartet: if spec.quartet { vec![spec.ambiguity] } else { Vec::new() }, overlap: Vec::new(), - pair_weight: if spec.pair_balance { vec![spec.no_ambiguity] } else { Vec::new() }, + pair_weight: if spec.pair_balance { vec![spec.ambiguity] } else { Vec::new() }, }; if !request.entropy.is_empty() || !request.quartet.is_empty() || !request.pair_weight.is_empty() { cache.ensure_family_annexes(&request)?; @@ -285,12 +282,12 @@ pub(crate) fn sample_index( .expect("cache-consistent: n_layer(partition) already bounds layer_idx"); let mut eligible = eligible_bitset_layer(layer.dir())?; if spec.quartet { - restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?; + restrict_to_quartet(&mut eligible, layer.dir(), spec.ambiguity)?; } let n_minorants = eligible.view().len(); let eligible_count = eligible.view().count_ones(); let mass = if spec.pair_balance { - iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)? + iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.ambiguity)? .enumerate() .filter(|&(family_idx, _)| eligible.view().get(family_idx)) .map(|(_, weight)| f64::from(weight)) @@ -435,7 +432,7 @@ fn sample_layer( &layer_dir, ScalarKind::Entropy, round_start, - spec.no_ambiguity, + spec.ambiguity, )?), None => Box::new(std::iter::repeat(0.0f32)), // never read: kernel forced to 1.0 below }; @@ -444,7 +441,7 @@ fn sample_layer( &layer_dir, ScalarKind::PairWeight, round_start, - spec.no_ambiguity, + spec.ambiguity, )?) } else { Box::new(std::iter::repeat(1.0f32)) @@ -494,7 +491,7 @@ fn sample_layer( ); let masked: Vec> = genome_mask .iter() - .map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity, spec.only_diploid)) + .map(|&m| masked_state(m, spec.free_loss, spec.ambiguity)) .collect(); if survives_masking(&masked, excluded) && (!spec.quartet || survives_quartet(&masked, excluded)) diff --git a/src/obikphylo/src/siblings/ambiguity.rs b/src/obikphylo/src/siblings/ambiguity.rs new file mode 100644 index 00000000..12f94527 --- /dev/null +++ b/src/obikphylo/src/siblings/ambiguity.rs @@ -0,0 +1,96 @@ +//! How a genome carrying several members of a family is treated — the one +//! parameter behind `--no-ambiguity` / `--only-diploid`, and the key of every +//! persisted annex whose values depend on which genomes count as carriers +//! (entropy, quartet, pair weight, family overlap). + +/// What becomes of a genome carrying more than one member of a family. +/// +/// - [`Keep`](Self::Keep): nothing — its set of members is a state of its +/// own (an IUPAC ambiguity code in the alignment). +/// - [`OnlyDiploid`](Self::OnlyDiploid): kept up to two members (a +/// heterozygote); three or four members make it missing data. +/// - [`Forbid`](Self::Forbid) (`--no-ambiguity`): any genome with several +/// members is missing data. +/// +/// The order is the on-disk header code (see [`header_flags`](Self::header_flags)), +/// kept so that annexes written when this was a `bool` stay valid. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum AmbiguityPolicy { + #[default] + Keep, + Forbid, + OnlyDiploid, +} + +impl AmbiguityPolicy { + /// Every policy, in header-code order. + pub const ALL: [AmbiguityPolicy; 3] = [Self::Keep, Self::Forbid, Self::OnlyDiploid]; + + /// From the two (mutually exclusive) CLI flags. + pub fn from_flags(no_ambiguity: bool, only_diploid: bool) -> Self { + debug_assert!(!(no_ambiguity && only_diploid), "--no-ambiguity and --only-diploid are exclusive"); + if no_ambiguity { + Self::Forbid + } else if only_diploid { + Self::OnlyDiploid + } else { + Self::Keep + } + } + + /// Whether a genome whose family presence mask is `mask` becomes missing + /// data under this policy. + #[inline] + pub fn drops(self, mask: u8) -> bool { + match self { + Self::Keep => false, + Self::OnlyDiploid => mask.count_ones() > 2, + Self::Forbid => mask.count_ones() > 1, + } + } + + /// The `flags` value persisted in an annex header: `0` for `Keep` and `1` + /// for `Forbid` are what the `no_ambiguity` bit used to be. + pub(crate) fn header_flags(self) -> u32 { + match self { + Self::Keep => 0, + Self::Forbid => 1, + Self::OnlyDiploid => 2, + } + } + + /// Inverse of [`header_flags`](Self::header_flags); unknown codes read as + /// `Keep`, as a file from before the flag existed. + pub(crate) fn from_header_flags(flags: u32) -> Self { + match flags { + 1 => Self::Forbid, + 2 => Self::OnlyDiploid, + _ => Self::Keep, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn drops_follows_the_member_count() { + for (mask, keep, diploid, forbid) in + [(0b0000u8, false, false, false), (0b0100, false, false, false), (0b0101, false, false, true), (0b0111, false, true, true), (0b1111, false, true, true)] + { + assert_eq!(AmbiguityPolicy::Keep.drops(mask), keep); + assert_eq!(AmbiguityPolicy::OnlyDiploid.drops(mask), diploid); + assert_eq!(AmbiguityPolicy::Forbid.drops(mask), forbid); + } + } + + #[test] + fn header_flags_roundtrip_and_stay_compatible_with_the_old_bool() { + for p in AmbiguityPolicy::ALL { + assert_eq!(AmbiguityPolicy::from_header_flags(p.header_flags()), p); + } + assert_eq!(AmbiguityPolicy::Keep.header_flags(), 0); + assert_eq!(AmbiguityPolicy::Forbid.header_flags(), 1); + } +} diff --git a/src/obikphylo/src/siblings/extensions/sibling_builder.rs b/src/obikphylo/src/siblings/extensions/sibling_builder.rs index cf57eeae..c1bb0ea5 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_builder.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_builder.rs @@ -16,7 +16,7 @@ use crate::siblings::algorithms::{ ensure_layer_pair_weight_annexes, inverse_overlap_matrix, is_fast_mode, }; use crate::siblings::{ - ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, FamilyOverlapAccumulator, + ANNEX_FILE_NAME, AmbiguityPolicy, FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, FamilyOverlapAccumulator, QUARTET_ANNEX_FILE_NAMES, ScalarKind, family_overlap_file_name, }; @@ -154,14 +154,14 @@ impl SiblingBuilder for IndexCache { fn build_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> { let n_genomes = self.meta().genomes().len(); let fast_mode = is_fast_mode(self); - let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity)); + let overlap_path = |policy: AmbiguityPolicy| self.index().dir().join(family_overlap_file_name(policy)); // Sweep 1: entropy, quartet, and every family-overlap annex still missing. - let mut overlap_accs: Vec<(bool, FamilyOverlapAccumulator)> = request + let mut overlap_accs: Vec<(AmbiguityPolicy, FamilyOverlapAccumulator)> = request .overlap_conventions() .into_iter() - .filter(|&no_ambiguity| !overlap_path(no_ambiguity).exists()) - .map(|no_ambiguity| (no_ambiguity, FamilyOverlapAccumulator::new(n_genomes, no_ambiguity))) + .filter(|&policy| !overlap_path(policy).exists()) + .map(|policy| (policy, FamilyOverlapAccumulator::new(n_genomes, policy))) .collect(); let pb = progress_bar("family_annexes", self.n_partition() as u64, "partitions"); for part in self.partitions() { @@ -172,8 +172,8 @@ impl SiblingBuilder for IndexCache { pb.inc(1); } pb.finish_and_clear(); - for (no_ambiguity, acc) in overlap_accs { - acc.write(&overlap_path(no_ambiguity)).map_err(OKIError::Io)?; + for (policy, acc) in overlap_accs { + acc.write(&overlap_path(policy)).map_err(OKIError::Io)?; } // Sweep 2: pair weights, from the now-complete overlap annexes. @@ -183,9 +183,9 @@ impl SiblingBuilder for IndexCache { let matrices = request .pair_weight_conventions() .into_iter() - .map(|no_ambiguity| { - FamilyOverlap::open(&overlap_path(no_ambiguity)) - .map(|overlap| (no_ambiguity, inverse_overlap_matrix(&overlap))) + .map(|policy| { + FamilyOverlap::open(&overlap_path(policy)) + .map(|overlap| (policy, inverse_overlap_matrix(&overlap))) .map_err(OKIError::Io) }) .collect::>>()?; diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index e38a467b..9f7a68e3 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -13,10 +13,10 @@ use obikidxcache::index_cache::IndexCache; use obikindex::{OKIError, OKIResult}; use crate::siblings::algorithms::{ - AnnexRequest, EntropySummary, SiblingAnnexStats, scan_family_entropies, sibling_annex_stats, sibling_family_size_histogram, + AnnexRequest, AnnexSelection, EntropySummary, SiblingAnnexStats, scan_family_entropies, sibling_annex_stats, sibling_family_size_histogram, }; use crate::siblings::extensions::SiblingBuilder; -use crate::siblings::{FamilyOverlap, family_overlap_file_name}; +use crate::siblings::{AmbiguityPolicy, FamilyOverlap, family_overlap_file_name}; pub trait SiblingExt { /// Build the sibling-count/minorant annex for every layer of every @@ -39,7 +39,14 @@ pub trait SiblingExt { /// Cross-partition/cross-layer lookups are required (a k-mer's siblings /// can live in any partition), but the layer loop itself — and thus the /// annex file this produces — stays local to one layer at a time. - fn build_sibling_annex(&self) -> OKIResult<()>; + /// + /// `derived` names the annexes derived from the sibling annex (entropy, + /// quartet, pair weight, family overlap) to have ready afterwards, all + /// under its one ambiguity policy — only those, so a routine rebuild + /// pays for nothing the following computations will not read. Derived + /// annexes that already existed are invalidated by the rebuild and are + /// rebuilt on demand, by the computation that reads them, if not listed. + fn build_sibling_annex(&self, derived: &AnnexSelection) -> OKIResult<()>; /// Write a CSV (`partition,layer,family_idx,entropy15,entropy4, /// family_size,n_genomes_present,n_genomes_unambiguous`) of per-family @@ -91,9 +98,9 @@ pub trait SiblingExt { /// The Family Overlap annex — number of shared *variable* families per /// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs), - /// under the `--no-ambiguity` counting convention `no_ambiguity` selects - /// (genomes carrying several members of a family do not count as - /// carrying it when `true`). Built if missing, same as every other + /// under the counting convention `policy` selects (which genomes + /// carrying several members of a family still count as carrying it, see + /// [`AmbiguityPolicy`]). Built if missing, same as every other /// annex-backed method here. Feeds `--family-overlap`'s own matrix /// output directly, and `--min-shared-family`'s auto-exclusion /// (`FamilyOverlap::mean_row(g) < threshold` — same statistic @@ -101,22 +108,22 @@ pub trait SiblingExt { /// triggers adding `g` to the excluded-genome set consumed by /// `algorithms::scan::sampled_scan`'s callers; this needs no /// computation of its own beyond that check). - fn family_overlap(&self, no_ambiguity: bool) -> OKIResult; + fn family_overlap(&self, policy: AmbiguityPolicy) -> OKIResult; } impl SiblingExt for IndexCache { - fn build_sibling_annex(&self) -> OKIResult<()> { + fn build_sibling_annex(&self, derived: &AnnexSelection) -> OKIResult<()> { self.index().with_lock(|| { self.build_base_sibling_annex()?; - // Precompute the entropy, quartet, pair-weight and family-overlap - // annexes too, now that every layer's fresh sibling annex is on - // disk (`non_monomorphic_reindex_layer`, which they depend on, - // reads it) — same routine-rebuild footing as the sibling annex - // itself, rather than leaving them to be built lazily on first - // `--shannon`/`--entropy`/`--quartet`/`--pair-balance`/ - // `--min-shared-family`/`--family-overlap` use. - tracing::info!("building entropy, quartet, pair-weight (both --no-ambiguity conventions) and family-overlap annexes"); - self.ensure_family_annexes(&AnnexRequest::everything()) + // Then exactly the derived annexes asked for, now that every + // layer's fresh sibling annex is on disk + // (`non_monomorphic_reindex_layer`, which they depend on, reads + // it). + if derived.is_empty() { + return Ok(()); + } + tracing::info!("building the requested derived annexes ({:?})", derived.policy); + self.ensure_family_annexes(&derived.request()) }) } @@ -160,9 +167,9 @@ impl SiblingExt for IndexCache { sibling_annex_stats(self) } - fn family_overlap(&self, no_ambiguity: bool) -> OKIResult { - self.ensure_family_annexes(&AnnexRequest { overlap: vec![no_ambiguity], ..Default::default() })?; - let path = self.index().dir().join(family_overlap_file_name(no_ambiguity)); + fn family_overlap(&self, policy: AmbiguityPolicy) -> OKIResult { + self.ensure_family_annexes(&AnnexRequest { overlap: vec![policy], ..Default::default() })?; + let path = self.index().dir().join(family_overlap_file_name(policy)); FamilyOverlap::open(&path).map_err(OKIError::Io) } } diff --git a/src/obikphylo/src/siblings/family_overlap.rs b/src/obikphylo/src/siblings/family_overlap.rs index f371bc92..90fd8a2b 100644 --- a/src/obikphylo/src/siblings/family_overlap.rs +++ b/src/obikphylo/src/siblings/family_overlap.rs @@ -25,35 +25,41 @@ use std::path::Path; use memmap2::Mmap; use obicompactvec::{building_path, publish_built}; +use crate::siblings::AmbiguityPolicy; use crate::siblings::helpers::{is_carrier, triangle_index}; const MAGIC: [u8; 4] = *b"POVL"; // Header: magic(4) + flags(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair, -// upper triangle) follows. `flags` bit 0 = `no_ambiguity` (see +// upper triangle) follows. `flags` = `AmbiguityPolicy::header_flags` (see // `FamilyOverlapAccumulator::new`); older files have `flags == 0`, i.e. -// `no_ambiguity == false`, which is what they were computed under. +// `Keep`, which is what they were computed under. const HEADER_SIZE: usize = 16; -const FLAG_NO_AMBIGUITY: u32 = 1; pub(crate) const FAMILY_OVERLAP_FILE_NAME: &str = "family_overlap.povl"; -/// Same annex, built with `no_ambiguity == true`. +/// Same annex, built under [`AmbiguityPolicy::Forbid`]. pub(crate) const FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME: &str = "family_overlap.noamb.povl"; +/// Same annex, built under [`AmbiguityPolicy::OnlyDiploid`]. +pub(crate) const FAMILY_OVERLAP_ONLY_DIPLOID_FILE_NAME: &str = "family_overlap.diploid.povl"; + /// Every family-overlap file name, whatever its counting convention — for /// invalidating them all together when the sibling annex they derive from is /// rebuilt. -pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 2] = - [FAMILY_OVERLAP_FILE_NAME, FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME]; +pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 3] = [ + FAMILY_OVERLAP_FILE_NAME, + FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME, + FAMILY_OVERLAP_ONLY_DIPLOID_FILE_NAME, +]; -/// File name of the family-overlap annex built under the given -/// `--no-ambiguity` counting convention. The two never share a file. -pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str { - if no_ambiguity { - FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME - } else { - FAMILY_OVERLAP_FILE_NAME +/// File name of the family-overlap annex built under the given ambiguity +/// policy. The policies never share a file. +pub(crate) fn family_overlap_file_name(policy: AmbiguityPolicy) -> &'static str { + match policy { + AmbiguityPolicy::Keep => FAMILY_OVERLAP_FILE_NAME, + AmbiguityPolicy::Forbid => FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME, + AmbiguityPolicy::OnlyDiploid => FAMILY_OVERLAP_ONLY_DIPLOID_FILE_NAME, } } @@ -61,7 +67,7 @@ pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str { pub struct FamilyOverlap { mmap: Mmap, n_genomes: usize, - no_ambiguity: bool, + policy: AmbiguityPolicy, } impl FamilyOverlap { @@ -79,19 +85,17 @@ impl FamilyOverlap { return Err(io::Error::new(io::ErrorKind::InvalidData, "POVL file truncated")); } let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap()); - Ok(Self { mmap, n_genomes, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 }) + Ok(Self { mmap, n_genomes, policy: AmbiguityPolicy::from_header_flags(flags) }) } pub fn n_genomes(&self) -> usize { self.n_genomes } - /// The counting convention these counts were computed under: `true` - /// when a genome carrying several members of a family was *not* counted - /// as carrying it (`--no-ambiguity`), `false` when any non-empty - /// state counts as carrying it. - pub fn no_ambiguity(&self) -> bool { - self.no_ambiguity + /// The counting convention these counts were computed under (see + /// [`AmbiguityPolicy`]): which genomes count as carrying a family. + pub fn policy(&self) -> AmbiguityPolicy { + self.policy } /// Number of variable families shared by genomes `i` and `j` — `0` for @@ -128,18 +132,17 @@ impl FamilyOverlap { /// `EntropyAnnexBuilder` do for their own, much larger, per-k-mer data. pub(crate) struct FamilyOverlapAccumulator { n_genomes: usize, - no_ambiguity: bool, + policy: AmbiguityPolicy, counts: Vec, } impl FamilyOverlapAccumulator { - /// `no_ambiguity` is the counting convention: `false` counts every - /// genome with a non-empty state as carrying the family, `true` only the - /// genomes carrying exactly one member of it — the others are missing - /// data in the alignment (`--no-ambiguity`), so they share nothing there. - pub(crate) fn new(n_genomes: usize, no_ambiguity: bool) -> Self { + /// `policy` is the counting convention: a genome counts as carrying the + /// family unless the policy makes it missing data in the alignment, in + /// which case it shares nothing there (see [`AmbiguityPolicy::drops`]). + pub(crate) fn new(n_genomes: usize, policy: AmbiguityPolicy) -> Self { let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2; - Self { n_genomes, no_ambiguity, counts: vec![0u64; n_pairs] } + Self { n_genomes, policy, counts: vec![0u64; n_pairs] } } /// One variable family's contribution: every pair of genomes both @@ -147,11 +150,11 @@ impl FamilyOverlapAccumulator { /// once. pub(crate) fn add_family(&mut self, genome_mask: &[u8]) { for i in 0..self.n_genomes { - if !is_carrier(genome_mask[i], self.no_ambiguity) { + if !is_carrier(genome_mask[i], self.policy) { continue; } for j in (i + 1)..self.n_genomes { - if is_carrier(genome_mask[j], self.no_ambiguity) { + if is_carrier(genome_mask[j], self.policy) { self.counts[triangle_index(self.n_genomes, i, j)] += 1; } } @@ -161,7 +164,7 @@ impl FamilyOverlapAccumulator { pub(crate) fn write(&self, path: &Path) -> io::Result<()> { let mut buf = Vec::with_capacity(HEADER_SIZE + self.counts.len() * 8); buf.extend_from_slice(&MAGIC); - let flags = if self.no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 }; + let flags = self.policy.header_flags(); buf.extend_from_slice(&flags.to_le_bytes()); buf.extend_from_slice(&(self.n_genomes as u64).to_le_bytes()); for &c in &self.counts { @@ -181,7 +184,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test.povl"); - let mut acc = FamilyOverlapAccumulator::new(3, false); + let mut acc = FamilyOverlapAccumulator::new(3, AmbiguityPolicy::Keep); acc.add_family(&[1, 1, 0]); // genomes 0,1 share this family acc.add_family(&[1, 1, 1]); // all three share this one acc.add_family(&[0, 1, 1]); // genomes 1,2 share this family @@ -201,26 +204,31 @@ mod tests { fn single_genome_mean_row_is_zero() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test.povl"); - FamilyOverlapAccumulator::new(1, false).write(&path).unwrap(); + FamilyOverlapAccumulator::new(1, AmbiguityPolicy::Keep).write(&path).unwrap(); let overlap = FamilyOverlap::open(&path).unwrap(); assert_eq!(overlap.mean_row(0), 0.0); } #[test] - fn no_ambiguity_does_not_count_multi_member_genomes_as_carriers() { + fn the_policy_decides_which_multi_member_genomes_are_carriers() { let dir = tempfile::tempdir().unwrap(); - let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0100]]; - for no_ambiguity in [false, true] { - let path = dir.path().join(family_overlap_file_name(no_ambiguity)); - let mut acc = FamilyOverlapAccumulator::new(3, no_ambiguity); + // genome 2: two members in the first family, three in the second + let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0111]]; + for policy in AmbiguityPolicy::ALL { + let path = dir.path().join(family_overlap_file_name(policy)); + let mut acc = FamilyOverlapAccumulator::new(3, policy); for m in masks { acc.add_family(m); } acc.write(&path).unwrap(); let overlap = FamilyOverlap::open(&path).unwrap(); - assert_eq!(overlap.no_ambiguity(), no_ambiguity); - // genome 2 carries two members in the first family: only counted without the flag - assert_eq!(overlap.get(0, 2), if no_ambiguity { 1 } else { 2 }); + assert_eq!(overlap.policy(), policy); + let shared_with_2 = match policy { + AmbiguityPolicy::Keep => 2, + AmbiguityPolicy::OnlyDiploid => 1, + AmbiguityPolicy::Forbid => 0, + }; + assert_eq!(overlap.get(0, 2), shared_with_2); assert_eq!(overlap.get(0, 1), 1 + 1); } } diff --git a/src/obikphylo/src/siblings/family_scalar_annex.rs b/src/obikphylo/src/siblings/family_scalar_annex.rs index a61fe30b..d96f1514 100644 --- a/src/obikphylo/src/siblings/family_scalar_annex.rs +++ b/src/obikphylo/src/siblings/family_scalar_annex.rs @@ -25,12 +25,13 @@ use std::path::{Path, PathBuf}; use memmap2::{Mmap, MmapMut}; use obicompactvec::{building_path, publish_built}; +use crate::siblings::AmbiguityPolicy; + // Header: magic(4) + flags(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows. -// `flags` bit 0 = `no_ambiguity` (the counting convention the values were -// computed under, see `FamilyScalarAnnexBuilder::new`); every file written -// before the flag existed has `flags == 0`, i.e. `no_ambiguity == false`, -// which is what it was computed under. -const FLAG_NO_AMBIGUITY: u32 = 1; +// `flags` = `AmbiguityPolicy::header_flags` (the counting convention the +// values were computed under, see `FamilyScalarAnnexBuilder::new`); every file +// written before the flag existed has `flags == 0`, i.e. `Keep`, which is what +// it was computed under. const HEADER_SIZE: usize = 16; /// Sentinel: not yet computed. Every stored value is `>= 0.0`, so any @@ -56,31 +57,32 @@ impl ScalarKind { } } - /// File name of this kind's annex built under the given `--no-ambiguity` - /// counting convention. The two conventions never share a file: - /// switching between them builds the missing annex without discarding the - /// other. - pub(crate) fn file_name(self, no_ambiguity: bool) -> &'static str { - match (self, no_ambiguity) { - (ScalarKind::Entropy, false) => "entropy.pent", - (ScalarKind::Entropy, true) => "entropy.noamb.pent", - (ScalarKind::PairWeight, false) => "pairweight.ppwt", - (ScalarKind::PairWeight, true) => "pairweight.noamb.ppwt", + /// File name of this kind's annex built under the given ambiguity + /// policy. The policies never share a file: switching between them builds + /// the missing annex without discarding the others. + pub(crate) fn file_name(self, policy: AmbiguityPolicy) -> &'static str { + match (self, policy) { + (ScalarKind::Entropy, AmbiguityPolicy::Keep) => "entropy.pent", + (ScalarKind::Entropy, AmbiguityPolicy::Forbid) => "entropy.noamb.pent", + (ScalarKind::Entropy, AmbiguityPolicy::OnlyDiploid) => "entropy.diploid.pent", + (ScalarKind::PairWeight, AmbiguityPolicy::Keep) => "pairweight.ppwt", + (ScalarKind::PairWeight, AmbiguityPolicy::Forbid) => "pairweight.noamb.ppwt", + (ScalarKind::PairWeight, AmbiguityPolicy::OnlyDiploid) => "pairweight.diploid.ppwt", } } /// Every file name of this kind, whatever its counting convention — for /// invalidating them all together (e.g. when the sibling annex they were /// numbered against is rebuilt). - pub(crate) fn file_names(self) -> [&'static str; 2] { - [self.file_name(false), self.file_name(true)] + pub(crate) fn file_names(self) -> [&'static str; 3] { + AmbiguityPolicy::ALL.map(|p| self.file_name(p)) } } pub struct FamilyScalarAnnex { mmap: Mmap, n: usize, - no_ambiguity: bool, + policy: AmbiguityPolicy, } impl FamilyScalarAnnex { @@ -100,19 +102,16 @@ impl FamilyScalarAnnex { return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file truncated")); } let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap()); - Ok(Self { mmap, n, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 }) + Ok(Self { mmap, n, policy: AmbiguityPolicy::from_header_flags(flags) }) } pub fn len(&self) -> usize { self.n } - /// The counting convention these values were computed under: `true` - /// when genomes carrying several members of a family were left out of - /// the count (`--no-ambiguity`), `false` when they were counted as a - /// state of their own. - pub fn no_ambiguity(&self) -> bool { - self.no_ambiguity + /// The counting convention these values were computed under. + pub fn policy(&self) -> AmbiguityPolicy { + self.policy } /// `None` for the sentinel (not yet computed — shouldn't happen against @@ -132,9 +131,9 @@ pub struct FamilyScalarAnnexBuilder { impl FamilyScalarAnnexBuilder { /// Create a new annex of `n` entries (a layer's non-monomorphic /// minorant count) at `path`, pre-initialised to the sentinel. - /// `no_ambiguity` records, in the header, the counting convention the - /// values will be computed under (see [`FamilyScalarAnnex::no_ambiguity`]). - pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, no_ambiguity: bool) -> io::Result { + /// `policy` records, in the header, the counting convention the values + /// will be computed under (see [`FamilyScalarAnnex::policy`]). + pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, policy: AmbiguityPolicy) -> io::Result { let file_size = HEADER_SIZE + n * 4; let file = OpenOptions::new() .read(true) @@ -145,7 +144,7 @@ impl FamilyScalarAnnexBuilder { file.set_len(file_size as u64)?; let mut mmap = unsafe { MmapMut::map_mut(&file)? }; mmap[0..4].copy_from_slice(&kind.magic()); - let flags = if no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 }; + let flags = policy.header_flags(); mmap[4..8].copy_from_slice(&flags.to_le_bytes()); mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); for i in 0..n { @@ -181,7 +180,7 @@ mod tests { fn sentinel_is_negative_and_unset_entries_read_as_none() { let dir = tempdir().unwrap(); let path = dir.path().join("test.pent"); - FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap().close().unwrap(); + FamilyScalarAnnexBuilder::new(4, &path, E, AmbiguityPolicy::Keep).unwrap().close().unwrap(); let annex = FamilyScalarAnnex::open(&path, E).unwrap(); for i in 0..4 { assert_eq!(annex.get(i), None); @@ -192,7 +191,7 @@ mod tests { fn roundtrip_including_zero_value() { let dir = tempdir().unwrap(); let path = dir.path().join("test.pent"); - let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap(); + let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, AmbiguityPolicy::Keep).unwrap(); let values = [0.0f32, 1.5, 3.9, 0.0001]; for (i, &v) in values.iter().enumerate() { builder.set(i, v); @@ -208,7 +207,7 @@ mod tests { fn mixed_sentinel_and_real_values() { let dir = tempdir().unwrap(); let path = dir.path().join("test.pent"); - let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, false).unwrap(); + let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, AmbiguityPolicy::Keep).unwrap(); builder.set(1, 2.0); builder.close().unwrap(); let annex = FamilyScalarAnnex::open(&path, E).unwrap(); @@ -222,12 +221,13 @@ mod tests { fn counting_convention_roundtrips_through_the_header() { let dir = tempdir().unwrap(); for kind in [ScalarKind::Entropy, ScalarKind::PairWeight] { - for no_ambiguity in [false, true] { - let path = dir.path().join(kind.file_name(no_ambiguity)); - FamilyScalarAnnexBuilder::new(2, &path, kind, no_ambiguity).unwrap().close().unwrap(); - assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().no_ambiguity(), no_ambiguity); + for policy in AmbiguityPolicy::ALL { + let path = dir.path().join(kind.file_name(policy)); + FamilyScalarAnnexBuilder::new(2, &path, kind, policy).unwrap().close().unwrap(); + assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().policy(), policy); } - assert_ne!(kind.file_name(false), kind.file_name(true)); + let names = kind.file_names(); + assert!(names[0] != names[1] && names[1] != names[2] && names[0] != names[2]); } } @@ -235,7 +235,7 @@ mod tests { fn one_kind_cannot_be_read_as_the_other() { let dir = tempdir().unwrap(); let path = dir.path().join("w.ppwt"); - FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, false).unwrap().close().unwrap(); + FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, AmbiguityPolicy::Keep).unwrap().close().unwrap(); assert!(FamilyScalarAnnex::open(&path, ScalarKind::Entropy).is_err()); assert!(FamilyScalarAnnex::open(&path, ScalarKind::PairWeight).is_ok()); } @@ -245,9 +245,9 @@ mod tests { // files written before the flag existed carried four zero bytes there let dir = tempdir().unwrap(); let path = dir.path().join("old.pent"); - FamilyScalarAnnexBuilder::new(1, &path, E, false).unwrap().close().unwrap(); + FamilyScalarAnnexBuilder::new(1, &path, E, AmbiguityPolicy::Keep).unwrap().close().unwrap(); let bytes = std::fs::read(&path).unwrap(); assert_eq!(&bytes[4..8], &[0u8; 4]); - assert!(!FamilyScalarAnnex::open(&path, E).unwrap().no_ambiguity()); + assert_eq!(FamilyScalarAnnex::open(&path, E).unwrap().policy(), AmbiguityPolicy::Keep); } } diff --git a/src/obikphylo/src/siblings/helpers.rs b/src/obikphylo/src/siblings/helpers.rs index d924ceb5..a213f8bd 100644 --- a/src/obikphylo/src/siblings/helpers.rs +++ b/src/obikphylo/src/siblings/helpers.rs @@ -1,6 +1,6 @@ use obikseq::CanonicalKmer; -use super::FamilyMask; +use super::{AmbiguityPolicy, FamilyMask}; /// Central-position base of a canonical k-mer, in the fixed 0=A/1=C/2=G/3=T /// encoding — the mask's bit index. `k` must be odd (project invariant). @@ -43,6 +43,6 @@ pub(super) fn triangle_index(n: usize, i: usize, j: usize) -> usize { /// for every annex that counts carriers (family overlap, pair weight, /// quartet), so they can never disagree. #[inline] -pub(crate) fn is_carrier(mask: u8, no_ambiguity: bool) -> bool { - mask != 0 && !(no_ambiguity && mask.count_ones() > 1) +pub(crate) fn is_carrier(mask: u8, policy: AmbiguityPolicy) -> bool { + mask != 0 && !policy.drops(mask) } diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index 0361f7db..fc65f51b 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -20,6 +20,7 @@ //! reverse. pub mod algorithms; +mod ambiguity; pub mod extensions; mod family_overlap; mod family_scalar_annex; @@ -35,13 +36,14 @@ pub(crate) use family_scalar_annex::ScalarKind; pub use family_scalar_annex::{FamilyScalarAnnex, FamilyScalarAnnexBuilder}; pub use family_overlap::FamilyOverlap; pub(crate) use quartet_annex::QUARTET_ANNEX_FILE_NAMES; +pub use ambiguity::AmbiguityPolicy; pub use iter::{ MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt, }; pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use algorithms::{ - AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer, + AlignmentReducer, AnnexSelection, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer, LayerReport, PairwiseReducer, RawSnpDistanceOutput, SamplePlan, SampleStats, SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SiteLocus, SnpAlignment, SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix, diff --git a/src/obikphylo/src/siblings/quartet_annex.rs b/src/obikphylo/src/siblings/quartet_annex.rs index 6eb0d590..00ea6064 100644 --- a/src/obikphylo/src/siblings/quartet_annex.rs +++ b/src/obikphylo/src/siblings/quartet_annex.rs @@ -12,8 +12,8 @@ //! fewer than four genomes — fits every topology and tells a tree nothing. //! See `algorithms::masking::is_quartet_informative` for the exact rule. //! -//! Which genomes count as carriers depends on `--no-ambiguity`, so each -//! convention has its own file, and **the file name is the parameter**: a +//! Which genomes count as carriers depends on the ambiguity policy +//! (`--no-ambiguity`, `--only-diploid`), so each policy has its own file, and **the file name is the parameter**: a //! `PersistentBitVec` header has no room for a flag, and changing it would //! change a shared structure for the sake of one annex. //! @@ -24,24 +24,31 @@ //! (`algorithms::masking::survives_quartet`) still runs on the resolved //! family. +use crate::siblings::AmbiguityPolicy; + pub(crate) const QUARTET_ANNEX_FILE_NAME: &str = "quartet.pbiv"; -/// Same annex, built with `no_ambiguity == true`. +/// Same annex, built under [`AmbiguityPolicy::Forbid`]. pub(crate) const QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME: &str = "quartet.noamb.pbiv"; +/// Same annex, built under [`AmbiguityPolicy::OnlyDiploid`]. +pub(crate) const QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME: &str = "quartet.diploid.pbiv"; + /// Every quartet-annex file name, whatever its counting convention — for /// invalidating them all together when the sibling annex they were numbered /// against is rebuilt. -pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 2] = - [QUARTET_ANNEX_FILE_NAME, QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME]; +pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 3] = [ + QUARTET_ANNEX_FILE_NAME, + QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME, + QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME, +]; -/// File name of the quartet annex built under the given `--no-ambiguity` -/// convention. -pub(crate) fn quartet_annex_file_name(no_ambiguity: bool) -> &'static str { - if no_ambiguity { - QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME - } else { - QUARTET_ANNEX_FILE_NAME +/// File name of the quartet annex built under the given ambiguity policy. +pub(crate) fn quartet_annex_file_name(policy: AmbiguityPolicy) -> &'static str { + match policy { + AmbiguityPolicy::Keep => QUARTET_ANNEX_FILE_NAME, + AmbiguityPolicy::Forbid => QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME, + AmbiguityPolicy::OnlyDiploid => QUARTET_ANNEX_ONLY_DIPLOID_FILE_NAME, } } @@ -50,10 +57,11 @@ mod tests { use super::*; #[test] - fn the_two_conventions_never_share_a_file() { - assert_ne!(quartet_annex_file_name(false), quartet_annex_file_name(true)); - for no_ambiguity in [false, true] { - assert!(QUARTET_ANNEX_FILE_NAMES.contains(&quartet_annex_file_name(no_ambiguity))); + fn the_policies_never_share_a_file() { + let names = AmbiguityPolicy::ALL.map(quartet_annex_file_name); + assert!(names[0] != names[1] && names[1] != names[2] && names[0] != names[2]); + for name in names { + assert!(QUARTET_ANNEX_FILE_NAMES.contains(&name)); } } } -- 2.54.1 From 0134c69cf4268d10ec7fd0100a4817e5e109ae52 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Mon, 9 Nov 2026 20:57:11 +1100 Subject: [PATCH 4/4] docs: clarify VCF pseudo-alignment format and update obikmer dependency Update the VCF export documentation to reflect that `-` represents an overlapping deletion (extra allele `*`) and introduce the `` symbolic allele for absent families. Add a new dedicated documentation file for the VCF pseudo-alignment format, remove obsolete descriptions from phylo.md, and fix benchmark scripts to handle errors and directory creation correctly. - Update `obikmer` dependency from 1.4.1 to 1.4.2 in Cargo.toml and Cargo.lock - Add `UserDocMD/formats/vcf.md` describing VCF pseudo-alignment structure, alleles, and genotypes - Remove obsolete VCF descriptions from `UserDocMD/usage/phylo.md` and update ploidy/free-loss notes - Update `DevDocMD/theory/evolutionary_distances.md` to clarify `-` as overlapping deletion and `` for others - Fix `benchmark/index_one_count.sh` to create index directory and check read command errors - Fix `benchmark/index_one_presence.sh` to create index directory and check simulation errors - Update `src/obikphylo/src/siblings/algorithms/alignment_vcf.rs` to use `` for absent families instead of `*` --- DevDocMD/theory/evolutionary_distances.md | 18 ++- UserDocMD/formats/vcf.md | 107 ++++++++++++++++++ UserDocMD/usage/phylo.md | 10 +- benchmark/index_one_count.sh | 4 +- benchmark/index_one_presence.sh | 4 +- mkdocs-user.yml | 1 + src/Cargo.lock | 2 +- src/obikmer/Cargo.toml | 2 +- .../src/siblings/algorithms/alignment_vcf.rs | 42 ++++--- 9 files changed, 156 insertions(+), 34 deletions(-) create mode 100644 UserDocMD/formats/vcf.md diff --git a/DevDocMD/theory/evolutionary_distances.md b/DevDocMD/theory/evolutionary_distances.md index 0f4b34e8..bba375ce 100644 --- a/DevDocMD/theory/evolutionary_distances.md +++ b/DevDocMD/theory/evolutionary_distances.md @@ -2903,10 +2903,18 @@ Absence handling. `masked_state` keeps absence (`m == 0`, `-`) as a real state unless `--free-loss`, where it becomes missing (`?`). A first version mapped `-` and `?` both to `.`: on a toy 4-genome index half of the lines then had `ALT = .` (presence/absence-only sites, selected by `survives_masking`), i.e. -polymorphic sites rendered as monomorphic. Now `-` is the extra allele `*` -(VCF >= 4.2 "allele missing due to upstream deletion"), after the bases; `?` -and IUPAC ambiguity codes stay `.` (haploid GT cannot express a genome -carrying several forms; superseded below). The writer needs no knowledge of +polymorphic sites rendered as monomorphic. Absence became an extra allele, +first written `*`; that was a misuse: in VCF 4.5 (section on the ALT column) +`*` is "allele missing due to overlapping deletion", and `<*>` is the gVCF +unspecified allele, `DEL`/`INS`/`DUP`/`INV`/`CNV` are structural-variant +types. Now the symbolic allele `` (free-form ID, declared by a +`##ALT=` line, always written in the header, listed after the +bases); `?` and IUPAC ambiguity codes stay `.` (haploid GT cannot express a +genome carrying several forms; superseded below). A symbolic ALT formally +asks for a padding base before `POS`; moot here, positions being virtual. +Checked with `bcftools view`/`stats` and `tabix`: no warning, the sites with +`` are classed "others" by `bcftools stats`, and +`-e 'ALT=""'` drops exactly them. The writer needs no knowledge of `--free-loss`: `-` simply never occurs with it. A reference-genome option (`--vcf-ref`) was implemented then dropped: `REF` is always the per-site majority base (alphabetical on ties). @@ -2916,7 +2924,7 @@ per sample/site), but an obikmer cell only says *which* forms a genome carries, not their multiplicity (`A/A/G` and `A/G/G` are both `R`), nor whether several forms mean heterozygosity or paralogues. Compromise chosen: `--no-ambiguity` => haploid (ambiguous cells are already `?`); otherwise -diploid, single base `X` -> `X/X`, two-base code -> `X/Y`, `-` -> `*/*`, and +diploid, single base `X` -> `X/X`, two-base code -> `X/Y`, `-` -> `/`, and three/four-base codes or `?` -> `./.`. `REF` is the majority allele counting each copy once (so a het contributes 1+1, a homozygote 2). Counting copies for a real dosage (polyploids) would need k-mer counts (`--with-counts`), diff --git a/UserDocMD/formats/vcf.md b/UserDocMD/formats/vcf.md new file mode 100644 index 00000000..b9aa3a98 --- /dev/null +++ b/UserDocMD/formats/vcf.md @@ -0,0 +1,107 @@ +# VCF pseudo-alignment + +[`phylo --vcf`](../usage/phylo.md) writes the sites of a SNP pseudo-alignment as a VCF file, `_alignment.vcf`. This page describes exactly what that file contains, because the k-mer families it is built from are not ordinary genomic variants and a few conventions follow from that. + +## Why a VCF, and what a site is + +The pseudo-alignment (see [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md)) has one column per *family*: the up to four k-mers that share all their bases except the central one. A genome that carries one of these k-mers shows a particular base at the centre, and the columns of the alignment are those central bases, genome by genome. The FASTA output keeps this as a matrix of letters. The VCF output lists the same columns as sites with alleles and genotypes, which is the form that variant-oriented tools (`bcftools`, population-genetics software, genotype viewers) expect. + +Nothing here comes from a reference genome. The file has no real coordinates, and the sites are not located on a chromosome: the positions below are labels that make the file valid and reproducible, not map positions. + +## Producing the file + +``` +obikmer phylo --vcf --n-sites 5000000 -o run1 my_index +``` + +`--vcf` implies `--pseudo-alignment`, so `--n-sites N|all` is required, together with an already-built sibling annex (see [`phylo`](../usage/phylo.md)). `--fasta` and `--vcf` can be given together to get both files from the same draw; with neither, only the FASTA is written. Every option that shapes the draw (`--free-loss`, `--no-ambiguity`, `--only-diploid`, `--quartet`, `--exclude-genome`, ...) applies to the VCF exactly as to the FASTA, since both describe the same sites. + +## Layout + +An illustrative excerpt (columns are tab-separated in the file): + +``` +##fileformat=VCFv4.5 +##source=obikmer +##contig= +##contig= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT idx0 idx1 idx2 idx3 +obikmer_0_0 440 CGATTNGTTAA A G . . . GT 0/0 1/1 0/0 0/0 +obikmer_0_0 1511 CACGANTGTAG A G,T . . . GT 0/0 0/0 1/1 2/2 +obikmer_0_1 9 TTACGNAGCTA C * . . . GT 0/0 0/0 1/1 0/0 +``` + +There is one sample column per genome that takes part in the draw, named after the genome label, in the index's own order. A genome removed with `--exclude-genome` or `--min-shared-family` has no column at all. `QUAL`, `FILTER` and `INFO` are always `.`: obikmer applies no variant filter and attaches no annotation, and leaving them empty keeps a file of millions of sites compact. + +## Coordinates: `CHROM` and `POS` + +An index is organised in partitions, and each partition holds one or more layers of k-mers (see [Index construction and on-disk layout](index_layout.md)). A family always belongs to exactly one layer of one partition, and the families of a layer are visited in a fixed order. The VCF uses that structure as coordinates: + +- `CHROM` is `obikmer__`, one virtual contig per layer of each partition in which at least one site was drawn; +- `POS` is the rank of the family inside that layer, starting at 1. + +For a given index and a given sibling annex, a family therefore always receives the same `CHROM` and `POS`, whatever the draw, the options or the number of sites requested. Two runs on the same index can be compared position by position. This stability is lost if the sibling annex is rebuilt, or if the index is rebuilt, merged or filtered, because the families are then enumerated again. Positions are never comparable between two different indexes; use `ID` for that. Even rebuilding an index from the very same genomes may enumerate the families in another order: on a small test index built twice from identical input, the family `CGATTNGTTAA` had `POS` 440 the first time and 371 the second, with the same `ID`. + +The file is sorted by partition, then layer, then position, which is also the order of the `##contig` lines, so tools that require a sorted file accept it directly. Positions are sparse: only the drawn families are written. + +## Identifier: `ID` + +The `ID` column holds a label for the family that does not depend on the index. It is built from one of the family's k-mers by replacing its central base with `N` (for example `CGATTNGTTAA` for k = 11), after choosing, among the family's possible orientations, the one that sorts first. All the members of a family, whichever strand they were read on, give the same label. + +This is what makes two files comparable even when they come from different indexes: the same family has the same `ID` in both, while its `CHROM` and `POS` will generally differ. It is also unique within a file, since two different families never share a label. + +## Alleles: `REF` and `ALT` + +The alleles are the bases carried at the centre of the family's k-mers, as in the FASTA. Two points are worth knowing. + +First, the base is the central base of the member's *canonical* form (the smaller of the k-mer and its reverse complement). That is not necessarily the strand of the `ID` label, so the letters should not be matched to the `ID` sequence by position; they are labels for the allele, not a reading of that sequence. + +Second, `REF` carries no meaning of reference: it is simply the most frequent allele among the genomes of the draw, each allele copy counted once, with the alphabetical order A < C < G < T deciding ties. `ALT` lists the remaining observed bases in the order A, C, G, T, separated by commas. There is no option to choose a reference genome. + +### Absence: the `` allele + +A genome may carry none of the family's k-mers. Without `--free-loss` this absence is treated as a real state (the family was lost, or is absent from that genome), which the FASTA shows as `-`. In the VCF it becomes one more allele, the symbolic allele ``, listed last in `ALT` and declared in the header by `##ALT=`. A genome without the family receives the genotype of ``; for a site with `ALT` equal to `G,T,` that is `3/3`. A site that differs between genomes only by presence versus absence has `ALT` equal to ``. + +The VCF specification gives the plain character `*` a meaning of its own (an allele missing because of an overlapping deletion described by another record), which is not the situation here, so it is deliberately not used. `` is a free-form symbolic identifier, outside the reserved structural-variant names (`DEL`, `INS`, `DUP`, `INV`, `CNV`) and outside `<*>`, which belongs to reference blocks. Tools that only look at sequence alleles treat the site as having an alternative that is not a base. + +With `--free-loss`, absence is declared missing data instead: `` never appears, and the genomes concerned get a missing genotype. + +## Genotypes + +Every genome gets one `GT` value per site. The ploidy of these values is a convention chosen from the options, and the header says which (`Genotype (diploid)` or `Genotype (haploid)`). + +| Cell in the alignment | Diploid (default, `--only-diploid`) | Haploid (`--no-ambiguity`) | +|---|---|---| +| one base `X` | `X/X`, for example `0/0` or `1/1` | `X`, for example `0` | +| two bases (IUPAC `R Y S W K M`) | the heterozygote, for example `0/1` | does not occur | +| three or four bases (`B D H V N`) | `./.` (not expressible) | does not occur | +| absent family (``) | `/`, for example `3/3` | the index of `` | +| missing (`?`) | `./.` | `.` | + +The two allele indices of a diploid genotype are always written in increasing order (`0/1`, never `1/0`), and the separator is `/` because no phasing information exists. + +Why diploid by default? A genome can carry several forms of the same family, which the alignment records as an IUPAC ambiguity code. A diploid genotype can express up to two forms, and `--only-diploid` removes the genomes that would need more (three or four forms) before the sites are selected, so that no `./.` comes from them. `--no-ambiguity` goes further and treats every such genome as missing, which makes all genotypes haploid. + +This is a convention, not a measurement. The file records which forms a genome carries, not how many copies of each: `A/A/G` and `A/G/G` both give the code `R` and the genotype `0/1`. Likewise, a genome with two forms may be heterozygous at one locus, or it may carry two similar regions of its genome (paralogues, repeats); the file cannot tell these apart. Do not read the genotypes as calls from sequencing reads. + +## Which sites are written + +Only the families that survive the draw and its masking appear. Within those: + +- a site needs at least one genome with a base or (in diploid mode) a two-base code, because `REF` is a base; a site where every genome is missing, absent or has an ambiguity code too wide for the mode is left out of the VCF while remaining in the FASTA; +- the log line at the end of the run gives the number of sites written out of the number drawn, so the difference is visible. + +A site is selected because its genomes show at least two distinct states, and absence counts as a state. A site that differs between genomes only by presence versus absence is therefore a legitimate site, written with `ALT` equal to ``. + +## Using the file with other tools + +- The contigs are virtual and have no declared length, and the file refers to no reference. Tools that need one (annotation, liftover) cannot use it; tools that only read sites and genotypes (`bcftools view`, `query`, `stats`, population-genetics software, PLINK-style converters) can. +- If a tool rejects multiallelic sites or the `` allele, `bcftools view -m2 -M2` keeps biallelic sites only, and `bcftools view -e 'ALT=""'` removes the sites involving absence. The file as written was read by `bcftools` and indexed by `tabix` without complaint on a small test index, in both diploid and haploid modes. `bcftools stats` counts the sites with `` among the "others" (symbolic alleles) rather than as SNPs. +- If a tool expects haploid data, run `phylo` with `--no-ambiguity` rather than converting the genotypes afterwards. + +## See also + +- [`phylo`](../usage/phylo.md): all the options that shape the draw. +- [Sampling design](../theory/phylogeny/snp_based/sampling_design.md): how the sites are drawn. +- [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md): what a family is. diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index 7ecc91f1..9a57d01c 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -79,15 +79,7 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model #### VCF output (`--vcf`) -The same drawn sites can be written as `_alignment.vcf`, with `--fasta` and `--vcf` freely combined (FASTA alone remains the default). A VCF needs genomic coordinates, which a family of k-mers does not have, so each site gets virtual ones that are stable for a given index: the contig is `obikmer__` (one per scanned layer of each partition) and `POS` is the family's rank inside that layer, starting at 1. The same family therefore always lands at the same position as long as the index and its sibling annex are not rebuilt, and the file is sorted by construction. - -The `ID` column is a label that does not depend on the index at all: the family's k-mer with its central base replaced by `N`, taken in a canonical orientation shared by every member of the family. Two VCF files produced from different indexes can therefore be matched on `ID`. - -`REF` is the most frequent allele among the genomes at that site, each allele copy counted once (alphabetical order on ties); `ALT` lists the other observed bases in the order A, C, G, T. The bases are those of the family member's canonical form, exactly as in the FASTA, so they are not necessarily on the same strand as the `ID` k-mer. - -The ploidy of the genotypes follows `--no-ambiguity`; `--only-diploid` keeps the diploid genotypes and simply removes, before the sites are selected, the genomes whose cell would have needed three or four alleles (so no `./.` comes from them). Without it, a genome can carry several forms of the same family, which the FASTA shows as an IUPAC ambiguity code, and genotypes are diploid: a single base `X` is written `X/X` (for example `0/0`), a code standing for two bases is the heterozygote (`0/1`), with the allele indices always in ascending order. A code standing for three or four bases (or `N`) cannot be expressed as a diploid genotype and is written `./.`. With `--no-ambiguity`, such genomes are already treated as missing, so genotypes are haploid: a single index (`0`, `1`...), and `.` for a missing genome. The diploid choice is a convention, not a measurement: the file records which forms the genome carries, not how many copies of each, and a genome with two forms may equally well come from a heterozygous individual or from two similar regions of the same genome. - -Without `--free-loss`, the absence of a family in a genome is a real state (a loss), written `-` in the FASTA. In the VCF it is the extra allele `*`, listed after the bases, so a genome lacking the family gets the genotype of `*` (`1/1` if `*` is the only alternative, `3/3` if `ALT` is `G,T,*`) and a site that differs only by presence/absence has `ALT` equal to `*`. With `--free-loss`, absence is treated as missing data instead and such genomes get a missing genotype. A site where no genome has a single base or a two-base code has no `REF` and is left out; the log line gives the number of sites written out of the number drawn. +The same drawn sites can be written as `_alignment.vcf`, with `--fasta` and `--vcf` freely combined (FASTA alone remains the default). The coordinates, identifiers, alleles and genotypes of this file follow conventions of their own, described on the [VCF pseudo-alignment](../formats/vcf.md) page. The genotype ploidy follows `--no-ambiguity` (haploid) and `--only-diploid` (diploid, with the genomes carrying three or four forms of a family removed beforehand). | Option | Default | Description | |---|---|---| diff --git a/benchmark/index_one_count.sh b/benchmark/index_one_count.sh index 473d9ff9..98b2040e 100755 --- a/benchmark/index_one_count.sh +++ b/benchmark/index_one_count.sh @@ -19,7 +19,7 @@ INDEX_PATH="${RUN_DIR}/specimen_index_count/${SPECIMEN}" STATS_DIR="${RUN_DIR}/stats/indexing_count" STATS_FILE="${STATS_DIR}/${SPECIMEN}.stats" -mkdir -p "${STATS_DIR}" +mkdir -p "${STATS_DIR}" "$(dirname "${INDEX_PATH}")" r1="${READS_DIR}/reads_R1.fastq.gz" r2="${READS_DIR}/reads_R2.fastq.gz" @@ -41,7 +41,7 @@ trap 'rm -f "${STDERR_LOG}"' EXIT --label "${SPECIMEN}" \ --meta "species=${species}" \ "${r1}" "${r2}" \ - 2>"${STDERR_LOG}" + 2>"${STDERR_LOG}" || { cat "${STDERR_LOG}" >&2; exit 1; } cat "${STDERR_LOG}" >&2 diff --git a/benchmark/index_one_presence.sh b/benchmark/index_one_presence.sh index 7b51887c..2a5e2c89 100755 --- a/benchmark/index_one_presence.sh +++ b/benchmark/index_one_presence.sh @@ -19,7 +19,7 @@ INDEX_PATH="${RUN_DIR}/specimen_index_presence/${SPECIMEN}" STATS_DIR="${RUN_DIR}/stats/indexing_presence" STATS_FILE="${STATS_DIR}/${SPECIMEN}.stats" -mkdir -p "${STATS_DIR}" +mkdir -p "${STATS_DIR}" "$(dirname "${INDEX_PATH}")" r1="${READS_DIR}/reads_R1.fastq.gz" r2="${READS_DIR}/reads_R2.fastq.gz" @@ -40,7 +40,7 @@ trap 'rm -f "${STDERR_LOG}"' EXIT --label "${SPECIMEN}" \ --meta "species=${species}" \ "${r1}" "${r2}" \ - 2>"${STDERR_LOG}" + 2>"${STDERR_LOG}" || { cat "${STDERR_LOG}" >&2; exit 1; } cat "${STDERR_LOG}" >&2 diff --git a/mkdocs-user.yml b/mkdocs-user.yml index cf38dc92..6063896e 100644 --- a/mkdocs-user.yml +++ b/mkdocs-user.yml @@ -64,6 +64,7 @@ nav: - Predicates and taxonomy paths: usage/predicates.md - Formats: - Index construction and on-disk layout: formats/index_layout.md + - VCF pseudo-alignment: formats/vcf.md - Architecture notes: architecture.md watch: diff --git a/src/Cargo.lock b/src/Cargo.lock index 73ec958b..42d39a03 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1537,7 +1537,7 @@ dependencies = [ [[package]] name = "obikmer" -version = "1.4.1" +version = "1.4.2" dependencies = [ "clap", "csv", diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index af73517f..0674acaf 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "obikmer" -version = "1.4.1" +version = "1.4.2" edition = "2024" [[bin]] diff --git a/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs b/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs index 97666553..66ac58fe 100644 --- a/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs +++ b/src/obikphylo/src/siblings/algorithms/alignment_vcf.rs @@ -12,11 +12,13 @@ //! central base of the family member's *canonical* form, which is not //! necessarily the strand of the `ID` k-mer. A `-` cell (family absent from //! the genome — a real state, unlike `?`, which `--free-loss` produces and -//! which means missing) is the extra allele `*`, listed after the bases. +//! which means missing) is the symbolic allele ``, declared by a `##ALT` +//! line and listed after the bases. (`*` is not used: in VCF it means an +//! allele missing because of an overlapping deletion.) //! //! Ploidy of the genotypes is a caller choice. Haploid: one allele per //! genome; an ambiguous cell is missing. Diploid: a single base `X` is -//! `X/X`, a two-base code is the heterozygote, `*` is `*/*`; codes with +//! `X/X`, a two-base code is the heterozygote, `` is `/`; codes with //! three or four bases (and `?`) are missing (`./.`). In both modes a //! genome gets the allele indices in ascending order. @@ -29,18 +31,25 @@ use super::masking::iupac_state; const BASES: [u8; 4] = *b"ACGT"; +/// Internal marker of the absence allele (the alignment's own `-`), written +/// as [`ABSENT_ALT`]. +const ABSENT: u8 = b'-'; + +/// How the absence allele appears in the `ALT` column. +const ABSENT_ALT: &[u8] = b""; + fn base_index(cell: u8) -> Option { BASES.iter().position(|&b| b == cell) } /// One cell as alleles: `None` = missing; else its (one or, diploid, two) -/// allele characters — a base, or `*` for absence. A cell with more forms +/// allele characters — a base, or [`ABSENT`] for absence. A cell with more forms /// than the ploidy allows, or `?`, is missing. fn cell_alleles(cell: u8, diploid: bool) -> Option<[u8; 2]> { let state = iupac_state(cell)?; - let allele = |bit: u8| if bit == 0 { b'*' } else { BASES[bit.trailing_zeros() as usize] }; + let allele = |bit: u8| if bit == 0 { ABSENT } else { BASES[bit.trailing_zeros() as usize] }; match state.count_ones() { - 0 => Some([b'*', b'*']), + 0 => Some([ABSENT, ABSENT]), 1 => Some([allele(state), allele(state)]), 2 if diploid => { let low = state & state.wrapping_neg(); @@ -89,6 +98,7 @@ impl SnpAlignment { last = Some((s.partition, s.layer)); } } + writeln!(out, "##ALT=")?; writeln!(out, "##FORMAT=", if diploid { "diploid" } else { "haploid" })?; write!(out, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT")?; for &g in &self.genome_indices { @@ -119,7 +129,7 @@ impl SnpAlignment { let mut alleles = vec![BASES[ref_base]]; alleles.extend((0..4).filter(|&b| b != ref_base && counts[b] > 0).map(|b| BASES[b])); if absent { - alleles.push(b'*'); + alleles.push(ABSENT); } let s = &self.sites[j]; @@ -135,7 +145,11 @@ impl SnpAlignment { if i > 0 { out.write_all(b",")?; } - out.write_all(&[a])?; + if a == ABSENT { + out.write_all(ABSENT_ALT)?; + } else { + out.write_all(&[a])?; + } } } out.write_all(b"\t.\t.\t.\tGT")?; @@ -208,16 +222,16 @@ mod tests { } #[test] - fn absence_is_a_star_allele_after_the_bases() { - // A, T, - (family absent), ? (missing): REF=A, ALT=T,*. + fn absence_is_the_abs_allele_after_the_bases() { + // A, T, - (family absent), ? (missing): REF=A, ALT=T,. let a = alignment(&[b"A", b"T", b"-", b"?"], vec![site(0, 0, 0)]); let f = first_record(&a); - assert_eq!(&f[3..5], ["A", "T,*"]); + assert_eq!(&f[3..5], ["A", "T,"]); assert_eq!(&f[9..], ["0", "1", "2", "."]); - // Only presence/absence differs: REF=A, ALT=*. + // Only presence/absence differs: REF=A, ALT=. let a = alignment(&[b"A", b"-"], vec![site(0, 0, 0)]); let f = first_record(&a); - assert_eq!(&f[3..5], ["A", "*"]); + assert_eq!(&f[3..5], ["A", ""]); assert_eq!(&f[9..], ["0", "1"]); } @@ -234,8 +248,8 @@ mod tests { // A, R (A/G), G, N (4 forms -> missing), - (absent), ? (missing). let a = alignment(&[b"A", b"R", b"G", b"N", b"-", b"?"], vec![site(0, 0, 0)]); let f = first_record_ploidy(&a, true); - // majority: A = 2 + 1 = 3, G = 1 + 2 = 3 -> tie -> A; ALT = G,* - assert_eq!(&f[3..5], ["A", "G,*"]); + // majority: A = 2 + 1 = 3, G = 1 + 2 = 3 -> tie -> A; ALT = G, + assert_eq!(&f[3..5], ["A", "G,"]); assert_eq!(&f[9..], ["0/0", "0/1", "1/1", "./.", "2/2", "./."]); // Same column, haploid: the ambiguous cells are missing. let f = first_record_ploidy(&a, false); -- 2.54.1