feat: add VCF export and refine alignment output infrastructure

Introduce `--vcf` and `--fasta` options for phylo commands, allowing users to
export pseudo-alignment data as VCF or FASTA. Implement comprehensive VCF logic
including stable virtual coordinates, canonical ID generation, and flexible
genotype representation based on ambiguity settings. Refactor internal alignment
structures to track k-mer families and site loci, ensuring consistent data flow
from scanning through to output generation.

- Add `src/obikmer/src/cmd/phylo/alignment_out.rs` with `create` and
  `write_alignment_outputs` to handle file creation and dispatching of FASTA/VCF
  formats based on `PhyloArgs`.
- Add `src/obikphylo/src/siblings/algorithms/alignment_vcf.rs` implementing
  VCF-specific logic: `write_vcf`, `alignment`, `cell_alleles`, `majority_base`,
  and site construction via `site` using fixed $K=11$. Includes helpers for
  IUPAC state inversion, genotype ordering, and stable coordinate generation.
- Update `src/obikmer/src/cmd/phylo/args.rs` to add `fasta` and `vcf` flags
  and a new `alignment_output()` method that returns true if any output format
  is requested. Add `pseudo_alignment` field documentation.
- Modify `src/obikmer/src/cmd/phylo/mod.rs` to use `alignment_output()` for
  determining alignment needs, update error messages for `--n-sites`, and import
  `write_alignment_outputs` from the new module.
- Refactor `src/obikphylo/src/siblings/algorithms/alignment.rs` by adding
  `SiteLocus` struct (partition, layer, family_idx, family_id) and updating
  `SnpAlignment` and `AlignmentReducer` to track sites. Update `new`, `finish`,
  and `reduce` to populate site information from `SurvivingFamily`.
- Update `src/obikphylo/src/siblings/algorithms/iqtree.rs`: `alignment` now
  returns `SnpAlignment` with empty sites; update `drop_ascertainment_noninformative`
  and `recode_symbols_as_absent` to handle the new `sites` field.
- Update `src/obikphylo/src/siblings/algorithms/family_scan.rs`: `GeneratedBatch`
  now includes a `kmers` field; `scan_layer_families` accepts `CanonicalKmer`
  and passes it to closures instead of masks.
- Update `src/obikphylo/src/siblings/algorithms/subsample.rs`: `SurvivingFamily`
  gains a `kmer` field; `sample_layer` closure accepts and uses this k-mer.
- Update `src/obikphylo/src/siblings/algorithms/snp_distance.rs`: `tally_2genomes`
  initializes `SurvivingFamily` with explicit `CanonicalKmer` values.
- Update `src/obikphylo/src/siblings/algorithms/stats.rs`: `sibling_annex_stats`
  signature adds `_kmer` parameter.
- Update `src/obikphylo/src/siblings/algorithms/family_annexes.rs`: closures in
  `ensure_layer_family_annexes` and `ensure_layer_pair_weight_annexes` now accept
  `_kmer`.
- Update `src/obikphylo/src/siblings/algorithms/pairwise.rs`: `ambiguous_vs_single`
  initializes k-mer using `CanonicalKmer::from_raw_unchecked(0)`.
- Add `iupac_state` function to `src/obikphylo/src/siblings/algorithms/masking.rs`
  mapping DNA characters to binary codes.
- Update `src/obikphylo/src/siblings/algorithms/mod.rs` to export `SiteLocus`
  and include new `alignment_vcf` module.
- Add `family_id` and `write_family_id_ascii` methods to
  `src/obikseq/src/kmer.rs` for stable k-mer family identification.
- Add tests `family_id_partitions_kmers_into_families` and
  `family_id_ascii_masks_centre` to `src/obikseq/src/tests/kmer.rs`.
- Update `src/obikphylo/src/siblings/mod.rs` imports to include new algorithm
  exports.
This commit is contained in:
Eric Coissac committed 2026-11-09 20:58:05 +11:00
1 parent ee62b131f9
commit b132f0847f
20 files changed
+558 -37

No files matched your search

@@ -0,0 +1,51 @@
//! `--pseudo-alignment` / `--fasta` / `--vcf` file output.
use std::io::{BufWriter, Write};
use obikphylo::siblings::SnpAlignment;
use tracing::info;
use super::args::PhyloArgs;
fn create(path: &str) -> BufWriter<std::fs::File> {
BufWriter::new(std::fs::File::create(path).unwrap_or_else(|e| {
eprintln!("error creating {path}: {e}");
std::process::exit(1);
}))
}
/// Writes the drawn alignment as FASTA (default, or `--fasta`) and/or VCF
/// (`--vcf`) next to `-o <prefix>`.
pub(super) fn write_alignment_outputs(
alignment: &SnpAlignment,
labels: &[String],
args: &PhyloArgs,
) {
let prefix = args.output.as_ref().map(|p| p.display().to_string());
let path_for = |suffix: &str| match &prefix {
Some(p) => format!("{p}_alignment.{suffix}"),
None => format!("alignment.{suffix}"),
};
let n_sites = alignment.sequences.first().map_or(0, Vec::len);
let n_genomes = alignment.genome_indices.len();
if args.fasta || !args.vcf {
let path = path_for("fasta");
let mut f = create(&path);
for (&g, seq) in alignment.genome_indices.iter().zip(alignment.sequences.iter()) {
obifastwrite::write_plain_record(seq, &labels[g], &mut f).unwrap();
}
info!("pseudo-alignment ({n_sites} site(s), {n_genomes} genome(s)) → {path}");
}
if args.vcf {
let path = path_for("vcf");
let mut f = create(&path);
let written = alignment.write_vcf(labels, !args.no_ambiguity, &mut f).unwrap_or_else(|e| {
eprintln!("error writing {path}: {e}");
std::process::exit(1);
});
f.flush().unwrap();
info!("pseudo-alignment VCF ({written}/{n_sites} site(s), {n_genomes} genome(s)) → {path}");
}
}
+21
View File
@@ -275,6 +275,19 @@ pub struct PhyloArgs {
#[arg(long)]
pub pseudo_alignment: bool,
/// Write the pseudo-alignment as FASTA (`<prefix>_alignment.fasta`) —
/// the default format when neither `--fasta` nor `--vcf` is given. Implies
/// `--pseudo-alignment`; combinable with `--vcf`.
#[arg(long)]
pub fasta: bool,
/// Write the pseudo-alignment as VCF (`<prefix>_alignment.vcf`): one
/// contig `obikmer_<partition>_<layer>` per scanned layer, `POS` the
/// family's index within it, `ID` the family's masked canonical k-mer.
/// Implies `--pseudo-alignment`; combinable with `--fasta`.
#[arg(long)]
pub vcf: bool,
/// Number of variable sites to draw index-wide, or `all` for every site
/// that passes the other filters (no sampling then). Mandatory for
/// `--pseudo-alignment`/`--sankoff`; optional for a `snp-*`
@@ -504,3 +517,11 @@ pub struct PhyloArgs {
#[arg(short, long)]
pub output: Option<PathBuf>,
}
impl PhyloArgs {
/// Whether a pseudo-alignment must be drawn and written (`--pseudo-alignment`,
/// or any of its output-format flags).
pub fn alignment_output(&self) -> bool {
self.pseudo_alignment || self.fasta || self.vcf
}
}
+10 -3
View File
@@ -100,7 +100,14 @@ fn drop_ascertainment_noninformative(alignment: &SnpAlignment) -> SnpAlignment {
.collect()
})
.collect();
SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone() }
let sites = alignment
.sites
.iter()
.zip(keep.iter())
.filter(|&(_, &k)| k)
.map(|(&site, _)| site)
.collect();
SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone(), sites }
}
/// Recode every occurrence of a byte in `symbols` to `-` — the same
@@ -119,7 +126,7 @@ fn recode_symbols_as_absent(alignment: &SnpAlignment, symbols: &[u8]) -> SnpAlig
.collect()
})
.collect();
SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone() }
SnpAlignment { sequences, genome_indices: alignment.genome_indices.clone(), sites: alignment.sites.clone() }
}
fn compact_alphabet(alignment: &SnpAlignment, free_loss: bool) -> CompactAlphabet {
@@ -384,7 +391,7 @@ mod tests {
fn alignment(sequences: Vec<Vec<u8>>) -> SnpAlignment {
let genome_indices = (0..sequences.len()).collect();
SnpAlignment { sequences, genome_indices }
SnpAlignment { sequences, genome_indices, sites: Vec::new() }
}
#[test]
+7 -17
View File
@@ -1,3 +1,4 @@
mod alignment_out;
mod args;
mod iqtree;
mod phyg;
@@ -19,6 +20,7 @@ use obikphylo::{Metrics, neighbor_joining, upgma};
use obisys::{Reporter, Stage};
use tracing::info;
use alignment_out::write_alignment_outputs;
use args::{DistanceArg, NSitesArg, distance_to_compute};
use iqtree::write_iqtree;
use phyg::write_sankoff_phyg;
@@ -285,13 +287,13 @@ pub fn run(args: PhyloArgs) {
let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree;
let distance = distance_to_compute(args.distance, args.nj, args.upgma, args.shared_kmers);
let snp_kind = distance.and_then(DistanceArg::as_snp);
let need_alignment = args.pseudo_alignment || sankoff_requested;
let need_alignment = args.alignment_output() || sankoff_requested;
let need_pairwise = sankoff_requested || snp_kind.is_some();
// `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional)
let target = args.n_sites.and_then(NSitesArg::target);
if args.pseudo_alignment && args.n_sites.is_none() {
eprintln!("error: --pseudo-alignment requires --n-sites <N|all>");
if args.alignment_output() && args.n_sites.is_none() {
eprintln!("error: --pseudo-alignment/--fasta/--vcf require --n-sites <N|all>");
std::process::exit(1);
}
if sankoff_requested && args.n_sites.is_none() {
@@ -357,21 +359,9 @@ pub fn run(args: PhyloArgs) {
let alignment = alignment_reducer.map(obikphylo::siblings::AlignmentReducer::finish);
// ── SNP pseudo-alignment (`--pseudo-alignment`) ─────────────────────────────
if args.pseudo_alignment {
if args.alignment_output() {
let alignment = alignment.as_ref().expect("scanned above: need_alignment");
let path = args.output.as_ref()
.map(|p| format!("{}_alignment.fasta", p.display()))
.unwrap_or_else(|| "alignment.fasta".into());
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
eprintln!("error creating {path}: {e}");
std::process::exit(1);
}));
let n_sites = alignment.sequences.first().map_or(0, Vec::len);
for (&g, seq) in alignment.genome_indices.iter().zip(alignment.sequences.iter()) {
obifastwrite::write_plain_record(seq, &labels[g], &mut f).unwrap();
}
info!("pseudo-alignment ({n_sites} site(s), {} genome(s)) → {path}", alignment.genome_indices.len());
write_alignment_outputs(alignment, &labels, &args);
}
// ── Sankoff cost-matrix calibration (`--sankoff`, `--tnt`, `--phyg`, `--iqtree`) ──
@@ -5,12 +5,26 @@
//! masking ([`super::masking::survives_masking`]).
use obikidxcache::index_cache::IndexCache;
use obikseq::RawKmer;
use rkyv::{Archive, Deserialize, Serialize};
use super::masking::iupac_code;
use super::scan::LayerReducer;
use super::subsample::SurvivingFamily;
/// Where an alignment column comes from: enough to give the same family the
/// same virtual coordinates every time for a given index (VCF `CHROM`/`POS`)
/// and an index-independent label (`family_id`).
#[derive(Clone, Copy, Archive, Serialize, Deserialize)]
pub struct SiteLocus {
pub partition: usize,
pub layer: usize,
/// `scan_layer_families`'s iteration-order index within `(partition, layer)`.
pub family_idx: usize,
/// Raw encoding of [`obikseq::CanonicalKmer::family_id`].
pub family_id: RawKmer,
}
/// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the
/// same length (one byte per sampled family, in the order
/// [`super::scan::sampled_scan`] produced them: layer by layer,
@@ -27,6 +41,8 @@ pub struct SnpAlignment {
/// order — the caller's authoritative mapping back to genome labels, so
/// it never has to re-derive which genomes were excluded itself.
pub genome_indices: Vec<usize>,
/// `sites[j]` describes column `j` of every row.
pub sites: Vec<SiteLocus>,
}
/// Accumulates a [`SnpAlignment`] over a [`super::scan::sampled_scan`]
@@ -40,6 +56,7 @@ pub struct SnpAlignment {
pub struct AlignmentReducer {
genome_indices: Vec<usize>,
sequences: Vec<Vec<u8>>,
sites: Vec<SiteLocus>,
}
impl AlignmentReducer {
@@ -49,16 +66,22 @@ impl AlignmentReducer {
.filter(|&g| !excluded.get(g).copied().unwrap_or(false))
.collect();
let sequences = vec![Vec::new(); genome_indices.len()];
Self { genome_indices, sequences }
Self { genome_indices, sequences, sites: Vec::new() }
}
pub fn finish(self) -> SnpAlignment {
SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices }
SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices, sites: self.sites }
}
}
impl LayerReducer for AlignmentReducer {
fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) {
fn reduce(&mut self, partition: usize, layer: usize, survivors: &[SurvivingFamily]) {
self.sites.extend(survivors.iter().map(|f| SiteLocus {
partition,
layer,
family_idx: f.family_idx,
family_id: f.kmer.family_id().raw(),
}));
reduce_alignment(survivors, &self.genome_indices, &mut self.sequences);
}
}
@@ -0,0 +1,256 @@
//! VCF rendering of a [`SnpAlignment`].
//!
//! One contig per `(partition, layer)` (`obikmer_<partition>_<layer>`), `POS`
//! = the family's `family_idx + 1` inside it: for a given index and sibling
//! annex the same family always gets the same virtual coordinates, and
//! sorting by `(partition, layer, family_idx)` is what makes the file a
//! well-formed (sorted) VCF. `ID` is the index-independent
//! [`obikseq::CanonicalKmer::family_id`] label (centre base masked as `N`).
//!
//! `REF`/`ALT` are the unambiguous bases (`A`/`C`/`G`/`T` cells of the
//! alignment) observed at the site, in the alignment's own convention: the
//! central base of the family member's *canonical* form, which is not
//! necessarily the strand of the `ID` k-mer. A `-` cell (family absent from
//! the genome — a real state, unlike `?`, which `--free-loss` produces and
//! which means missing) is the extra allele `*`, listed after the bases.
//!
//! Ploidy of the genotypes is a caller choice. Haploid: one allele per
//! genome; an ambiguous cell is missing. Diploid: a single base `X` is
//! `X/X`, a two-base code is the heterozygote, `*` is `*/*`; codes with
//! three or four bases (and `?`) are missing (`./.`). In both modes a
//! genome gets the allele indices in ascending order.
use std::io::{self, Write};
use obikseq::CanonicalKmer;
use super::alignment::SnpAlignment;
use super::masking::iupac_state;
const BASES: [u8; 4] = *b"ACGT";
fn base_index(cell: u8) -> Option<usize> {
BASES.iter().position(|&b| b == cell)
}
/// One cell as alleles: `None` = missing; else its (one or, diploid, two)
/// allele characters — a base, or `*` for absence. A cell with more forms
/// than the ploidy allows, or `?`, is missing.
fn cell_alleles(cell: u8, diploid: bool) -> Option<[u8; 2]> {
let state = iupac_state(cell)?;
let allele = |bit: u8| if bit == 0 { b'*' } else { BASES[bit.trailing_zeros() as usize] };
match state.count_ones() {
0 => Some([b'*', b'*']),
1 => Some([allele(state), allele(state)]),
2 if diploid => {
let low = state & state.wrapping_neg();
Some([allele(low), allele(state ^ low)])
}
_ => None,
}
}
/// `REF` base (index into `ACGT`) of one site: the most frequent unambiguous
/// base, lexicographic minimum on ties. `None` when no cell carries one.
fn majority_base(counts: &[usize; 4]) -> Option<usize> {
(0..4)
.filter(|&b| counts[b] > 0)
.max_by_key(|&b| (counts[b], std::cmp::Reverse(b)))
}
impl SnpAlignment {
/// Writes this alignment as a VCF (haploid `GT` per genome). `labels` is
/// indexed by the index's own genome numbering ([`Self::genome_indices`]
/// values). `diploid` selects the genotype ploidy (see the module docs).
/// `REF` is the per-site majority allele (each allele copy counted once).
/// Returns the number of sites written: a site with no base among its
/// calls has no `REF` to report and is skipped.
pub fn write_vcf<W: Write>(&self, labels: &[String], diploid: bool, out: &mut W) -> io::Result<usize> {
let n_sites = self.sequences.first().map_or(0, Vec::len);
if self.sites.len() != n_sites {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("alignment has {n_sites} column(s) but {} site locus record(s)", self.sites.len()),
));
}
let mut order: Vec<usize> = (0..n_sites).collect();
order.sort_unstable_by_key(|&j| {
let s = &self.sites[j];
(s.partition, s.layer, s.family_idx)
});
writeln!(out, "##fileformat=VCFv4.5")?;
writeln!(out, "##source=obikmer")?;
let mut last = None;
for &j in &order {
let s = &self.sites[j];
if last != Some((s.partition, s.layer)) {
writeln!(out, "##contig=<ID=obikmer_{}_{}>", s.partition, s.layer)?;
last = Some((s.partition, s.layer));
}
}
writeln!(out, "##FORMAT=<ID=GT,Number=1,Type=String,Description=\"Genotype ({})\">", if diploid { "diploid" } else { "haploid" })?;
write!(out, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT")?;
for &g in &self.genome_indices {
write!(out, "\t{}", labels[g])?;
}
writeln!(out)?;
let mut written = 0usize;
let mut locus = Vec::new();
let mut calls: Vec<Option<[u8; 2]>> = Vec::with_capacity(self.sequences.len());
let copies = if diploid { 2 } else { 1 };
for &j in &order {
calls.clear();
calls.extend(self.sequences.iter().map(|seq| cell_alleles(seq[j], diploid)));
let mut counts = [0usize; 4];
let mut absent = false;
for call in calls.iter().flatten() {
for &a in &call[..copies] {
match base_index(a) {
Some(b) => counts[b] += 1,
None => absent = true,
}
}
}
let Some(ref_base) = majority_base(&counts) else {
continue;
};
let mut alleles = vec![BASES[ref_base]];
alleles.extend((0..4).filter(|&b| b != ref_base && counts[b] > 0).map(|b| BASES[b]));
if absent {
alleles.push(b'*');
}
let s = &self.sites[j];
locus.clear();
CanonicalKmer::from_raw_unchecked(s.family_id).write_family_id_ascii(&mut locus)?;
write!(out, "obikmer_{}_{}\t{}\t", s.partition, s.layer, s.family_idx + 1)?;
out.write_all(&locus)?;
write!(out, "\t{}\t", alleles[0] as char)?;
if alleles.len() == 1 {
out.write_all(b".")?;
} else {
for (i, &a) in alleles[1..].iter().enumerate() {
if i > 0 {
out.write_all(b",")?;
}
out.write_all(&[a])?;
}
}
out.write_all(b"\t.\t.\t.\tGT")?;
for call in &calls {
let idx = |a: u8| alleles.iter().position(|&x| x == a);
match call.map(|c| (idx(c[0]), idx(c[1]))) {
Some((Some(x), Some(y))) if diploid => write!(out, "\t{}/{}", x.min(y), x.max(y))?,
Some((Some(x), _)) => write!(out, "\t{x}")?,
_ => out.write_all(if diploid { b"\t./." } else { b"\t." })?,
}
}
out.write_all(b"\n")?;
written += 1;
}
Ok(written)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::siblings::SiteLocus;
fn alignment(rows: &[&[u8]], sites: Vec<SiteLocus>) -> SnpAlignment {
SnpAlignment {
sequences: rows.iter().map(|r| r.to_vec()).collect(),
genome_indices: (0..rows.len()).collect(),
sites,
}
}
fn site(partition: usize, layer: usize, family_idx: usize) -> SiteLocus {
// Same global k as the crate's other tests (`siblingannex.rs`): the
// global is process-wide here, so it must not conflict.
obikseq::params::set_k(11);
SiteLocus { partition, layer, family_idx, family_id: 0 }
}
fn first_record(a: &SnpAlignment) -> Vec<String> {
first_record_ploidy(a, false)
}
fn first_record_ploidy(a: &SnpAlignment, diploid: bool) -> Vec<String> {
let labels: Vec<String> = (0..a.sequences.len()).map(|i| format!("g{i}")).collect();
let mut out = Vec::new();
a.write_vcf(&labels, diploid, &mut out).unwrap();
let text = String::from_utf8(out).unwrap();
let line = text.lines().find(|l| !l.starts_with('#')).unwrap();
line.split('\t').map(String::from).collect()
}
#[test]
fn reference_is_majority_then_lexicographic() {
assert_eq!(majority_base(&[1, 3, 0, 0]), Some(1));
assert_eq!(majority_base(&[2, 2, 0, 0]), Some(0));
assert_eq!(majority_base(&[0, 0, 0, 0]), None);
}
#[test]
fn genotypes_follow_ref_then_alt_order() {
// Column 0: A,G,G,R,? -> REF=G (majority), ALT=A; Column 1: all ambiguous -> skipped.
let a = alignment(
&[b"AY", b"GY", b"GR", b"RR", b"?R"],
vec![site(0, 0, 4), site(0, 0, 9)],
);
let f = first_record(&a);
assert_eq!(&f[..2], ["obikmer_0_0", "5"]);
assert_eq!(&f[3..5], ["G", "A"]);
assert_eq!(&f[9..], ["1", "0", "0", ".", "."]);
}
#[test]
fn absence_is_a_star_allele_after_the_bases() {
// A, T, - (family absent), ? (missing): REF=A, ALT=T,*.
let a = alignment(&[b"A", b"T", b"-", b"?"], vec![site(0, 0, 0)]);
let f = first_record(&a);
assert_eq!(&f[3..5], ["A", "T,*"]);
assert_eq!(&f[9..], ["0", "1", "2", "."]);
// Only presence/absence differs: REF=A, ALT=*.
let a = alignment(&[b"A", b"-"], vec![site(0, 0, 0)]);
let f = first_record(&a);
assert_eq!(&f[3..5], ["A", "*"]);
assert_eq!(&f[9..], ["0", "1"]);
}
#[test]
fn iupac_state_inverts_iupac_code() {
for state in 0u8..16 {
assert_eq!(iupac_state(super::super::masking::iupac_code(state)), Some(state));
}
assert_eq!(iupac_state(b'?'), None);
}
#[test]
fn diploid_genotypes() {
// A, R (A/G), G, N (4 forms -> missing), - (absent), ? (missing).
let a = alignment(&[b"A", b"R", b"G", b"N", b"-", b"?"], vec![site(0, 0, 0)]);
let f = first_record_ploidy(&a, true);
// majority: A = 2 + 1 = 3, G = 1 + 2 = 3 -> tie -> A; ALT = G,*
assert_eq!(&f[3..5], ["A", "G,*"]);
assert_eq!(&f[9..], ["0/0", "0/1", "1/1", "./.", "2/2", "./."]);
// Same column, haploid: the ambiguous cells are missing.
let f = first_record_ploidy(&a, false);
assert_eq!(&f[9..], ["0", ".", "1", ".", "2", "."]);
}
#[test]
fn sites_sorted_by_contig_then_position() {
let a = alignment(&[b"AA", b"GG"], vec![site(1, 0, 0), site(0, 2, 7)]);
let labels = vec!["x".to_string(), "y".to_string()];
let mut out = Vec::new();
a.write_vcf(&labels, false, &mut out).unwrap();
let text = String::from_utf8(out).unwrap();
let body: Vec<&str> = text.lines().filter(|l| !l.starts_with('#')).collect();
assert!(body[0].starts_with("obikmer_0_2\t8\t"));
assert!(body[1].starts_with("obikmer_1_0\t1\t"));
}
}
@@ -67,7 +67,7 @@ pub(crate) fn scan_family_entropies(
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, mask, genome_mask| {
|family_idx, _kmer, mask, genome_mask| {
if failure.is_some() {
return;
}
@@ -145,7 +145,7 @@ pub(crate) fn ensure_layer_family_annexes(
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, mask, genome_mask| {
|family_idx, _kmer, mask, genome_mask| {
debug_assert!(
mask.family_size() >= 2,
"Selection::Some(eligible) must only visit non-monomorphic minorants"
@@ -217,7 +217,7 @@ pub(crate) fn ensure_layer_pair_weight_annexes(
n_genomes,
fast_mode,
&Selection::Some(&eligible),
|family_idx, _mask, genome_mask| {
|family_idx, _kmer, _mask, genome_mask| {
let compact = reindex[&family_idx];
for (no_ambiguity, inverse, builder) in builders.iter_mut() {
builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse));
@@ -125,6 +125,8 @@ struct SourceBatch {
struct GeneratedBatch {
start_family_idx: usize,
masks: Vec<FamilyMask>,
/// Each family's minorant k-mer, same order as `masks`.
kmers: Vec<CanonicalKmer>,
/// Flat `slots.len() * n_genomes` — `genome_mask[i * n_genomes + g]`.
genome_mask: Vec<u8>,
/// `outgoing[dest_partition]` = `(variant, family_idx_in_batch, base,
@@ -146,8 +148,8 @@ enum FamData {
/// generation and resolution use different concurrency. `on_family` is
/// called once per
/// family, in iteration order, with that family's iteration-order index
/// (the same numbering [`Selection`] indices are drawn from), its own annex
/// mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set
/// (the same numbering [`Selection`] indices are drawn from), its minorant
/// k-mer, its own annex mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set
/// iff genome `g` carries the member whose own canonical central base is
/// `b`), backed by a scratch buffer reused across every call — callers that
/// need to keep data past the call must copy it themselves.
@@ -164,7 +166,7 @@ pub(crate) fn scan_layer_families(
n_genomes: usize,
fast_mode: bool,
selection: &Selection,
mut on_family: impl FnMut(usize, FamilyMask, &[u8]),
mut on_family: impl FnMut(usize, CanonicalKmer, FamilyMask, &[u8]),
) -> OKIResult<()> {
let n_parts = cache.index().n_partitions();
let k = cache.index().kmer_size();
@@ -225,6 +227,7 @@ pub(crate) fn scan_layer_families(
let ctx = &worker_ctx;
let n = batch.entries.len();
let mut masks = Vec::with_capacity(n);
let mut kmers = Vec::with_capacity(n);
let mut bases = Vec::with_capacity(n);
let mut genome_mask = vec![0u8; n * ctx.n_genomes];
let mut outgoing: Vec<Vec<(CanonicalKmer, usize, u8, u8)>> =
@@ -248,6 +251,7 @@ pub(crate) fn scan_layer_families(
for (i, entry) in batch.entries.iter().enumerate() {
let (kmer, mask) = (entry.kmer, entry.mask);
masks.push(mask);
kmers.push(kmer);
let base = central_base(kmer, ctx.k);
bases.push(base);
let family_idx = batch.start_family_idx + i;
@@ -297,6 +301,7 @@ pub(crate) fn scan_layer_families(
GeneratedBatch {
start_family_idx: batch.start_family_idx,
masks,
kmers,
genome_mask,
outgoing,
_permit: batch._permit,
@@ -344,7 +349,7 @@ pub(crate) fn scan_layer_families(
for (g, dst) in scratch.iter_mut().enumerate() {
*dst = genome_mask[i * n_genomes + g].load(Ordering::Relaxed);
}
on_family(family_idx, batch.masks[i], &scratch);
on_family(family_idx, batch.kmers[i], batch.masks[i], &scratch);
}
next_expected += n;
}
@@ -188,6 +188,32 @@ pub(crate) fn iupac_code(state: u8) -> u8 {
}
}
/// Inverse of [`iupac_code`]: the `0..=15` state a cell character encodes
/// (`-` -> 0, a base -> its single bit, an ambiguity code -> its bits), or
/// `None` for anything else — notably `?`, a missing cell.
#[inline]
pub(crate) fn iupac_state(cell: u8) -> Option<u8> {
Some(match cell {
b'-' => 0b0000,
b'A' => 0b0001,
b'C' => 0b0010,
b'G' => 0b0100,
b'T' => 0b1000,
b'R' => 0b0101,
b'Y' => 0b1010,
b'S' => 0b0110,
b'W' => 0b1001,
b'K' => 0b1100,
b'M' => 0b0011,
b'B' => 0b1110,
b'D' => 0b1101,
b'H' => 0b1011,
b'V' => 0b0111,
b'N' => 0b1111,
_ => return None,
})
}
#[cfg(test)]
mod tests {
use super::*;
+2 -1
View File
@@ -14,6 +14,7 @@
//! encoding.
mod alignment;
mod alignment_vcf;
mod annex;
mod cardcomp;
mod entropy;
@@ -34,7 +35,7 @@ mod subsample;
use obikidxcache::index_cache::IndexCache;
pub use alignment::{AlignmentReducer, SnpAlignment};
pub use alignment::{AlignmentReducer, SiteLocus, SnpAlignment};
pub use cardcomp::{
cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix,
};
@@ -510,7 +510,7 @@ mod cardinality_masking_tests {
// `SurvivingFamily::masked`'s own docs) — the masking decision is baked
// in here, not passed to `reduce_pairwise` itself anymore.
fn ambiguous_vs_single(masked: Vec<Option<u8>>) -> Vec<SurvivingFamily> {
vec![SurvivingFamily { family_idx: 0, mask: FamilyMask::EMPTY.with(0).with(1), masked }]
vec![SurvivingFamily { family_idx: 0, kmer: obikseq::CanonicalKmer::from_raw_unchecked(0), mask: FamilyMask::EMPTY.with(0).with(1), masked }]
}
#[test]
@@ -286,6 +286,7 @@ mod tests {
for &(a, b) in pairs {
families.push(SurvivingFamily {
family_idx: families.len(),
kmer: obikseq::CanonicalKmer::from_raw_unchecked(0),
mask: FamilyMask::EMPTY.with(a.trailing_zeros() as u8).with(b.trailing_zeros() as u8),
masked: vec![Some(a), Some(b)],
});
@@ -296,6 +297,7 @@ mod tests {
let other = BASES[(i + 1) % 4];
families.push(SurvivingFamily {
family_idx: families.len(),
kmer: obikseq::CanonicalKmer::from_raw_unchecked(0),
mask: FamilyMask::EMPTY.with(b.trailing_zeros() as u8).with(other.trailing_zeros() as u8),
masked: vec![Some(b), Some(b)],
});
@@ -106,7 +106,7 @@ pub(crate) fn sibling_annex_stats(cache: &IndexCache) -> OKIResult<SiblingAnnexS
n_genomes,
fast_mode,
&Selection::All,
|_family_idx, mask, genome_mask| {
|_family_idx, _kmer, mask, genome_mask| {
// "Genome g represents this family" means g carries
// *any* of its members, not just the minorant's own —
// `genome_mask[g] != 0` is exactly that.
@@ -49,6 +49,7 @@ use std::sync::Arc;
use obicompactvec::TempBitVecBuilder;
use obikidxcache::index_cache::IndexCache;
use obikindex::OKIResult;
use obikseq::CanonicalKmer;
use obisys::progress_bar;
use rand::RngExt;
@@ -82,6 +83,8 @@ const MAX_TOPUP_ROUNDS: usize = 5;
/// batch, well after that callback returns.
pub struct SurvivingFamily {
pub family_idx: usize,
/// The family's minorant k-mer (feeds `CanonicalKmer::family_id`).
pub kmer: CanonicalKmer,
pub mask: FamilyMask,
pub masked: Vec<Option<u8>>,
}
@@ -478,7 +481,7 @@ fn sample_layer(
n_genomes,
fast_mode,
&Selection::Some(&accepted),
|family_idx, mask, genome_mask| {
|family_idx, kmer, mask, genome_mask| {
debug_assert!(
mask.family_size() >= 2,
"eligible bitset must only accept non-monomorphic minorants"
@@ -490,7 +493,7 @@ fn sample_layer(
if survives_masking(&masked, excluded)
&& (!spec.quartet || survives_quartet(&masked, excluded))
{
survivors.push(SurvivingFamily { family_idx, mask, masked });
survivors.push(SurvivingFamily { family_idx, kmer, mask, masked });
kept += 1;
}
},
+1 -1
View File
@@ -43,7 +43,7 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
pub use algorithms::{
AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer,
LayerReport, PairwiseReducer, RawSnpDistanceOutput, SamplePlan, SampleStats,
SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SnpAlignment,
SampleStatsReducer, SamplingSpec, SankoffBundle, SiblingAnnexStats, SiteLocus, SnpAlignment,
SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix,
pairwise_cost_matrix, sampled_scan,
};
+22
View File
@@ -362,6 +362,28 @@ impl<L: KmerLength> CanonicalKmerOf<L> {
]
}
/// Stable identifier of this k-mer's central-base family: the centre
/// forced to `A` on the k-mer and on its reverse complement, each
/// canonicalised, then the lexicographic minimum of the two. Identical
/// for the 4 members of a family whatever the orientation they are seen
/// in, and independent of any index. Well-defined for odd `L::len()`.
pub fn family_id(&self) -> Self {
let m = (L::len() - 1) / 2;
let shift = KMER_BITS - 2 - 2 * m;
let clear = |raw: RawKmer| raw & !((0b11 as RawKmer) << shift);
let fwd = KmerOf::<L>(clear(self.0), PhantomData).canonical();
let rev = KmerOf::<L>(clear(self.revcomp().raw()), PhantomData).canonical();
if fwd.0 <= rev.0 { fwd } else { rev }
}
/// Writes [`family_id`](Self::family_id) as ASCII with the centre base
/// replaced by `N` (the family's textual locus label).
pub fn write_family_id_ascii<W: Write>(&self, writer: &mut W) -> io::Result<()> {
let mut ascii = self.family_id().to_ascii();
ascii[(L::len() - 1) / 2] = b'N';
writer.write_all(&ascii)
}
/// Return the inner value as a raw [`KmerOf<L>`].
#[inline]
pub fn into_kmer(self) -> KmerOf<L> {
+48
View File
@@ -252,4 +252,52 @@ mod tests {
check!(5);
check!(31);
}
// ── family_id ───────────────────────────────────────────────────────────
/// Exhaustive: `family_id` is shared by every member of a family (and by
/// each member's reverse complement) and separates distinct families —
/// the latter checked against `central_canonical_neighbors` as ground truth.
#[test]
fn family_id_partitions_kmers_into_families() {
macro_rules! check {
($n:expr) => {{
type K = KmerOf<ConstLen<$n>>;
let mut families: std::collections::HashMap<_, Vec<_>> = Default::default();
for code in 0..4usize.pow($n) {
let ascii: Vec<u8> = (0..$n).map(|i| b"ACGT"[(code >> (2 * i)) & 3]).collect();
let ck = K::from_ascii(&ascii).unwrap().canonical();
let id = ck.family_id();
assert_eq!(id, ck.revcomp().canonical().family_id());
let mut members: Vec<_> =
ck.central_canonical_neighbors().iter().map(|n| n.raw()).collect();
members.sort_unstable();
for n in ck.central_canonical_neighbors() {
assert_eq!(n.family_id(), id, "k={}: members disagree", $n);
}
assert_eq!(id, id.family_id(), "k={}: family_id not idempotent", $n);
families.entry(id).or_default().push(members);
}
// One id <=> one family: all members seen under an id share one member set.
for sets in families.values() {
assert!(sets.windows(2).all(|w| w[0] == w[1]), "k={}: id merges families", $n);
}
}};
}
check!(3);
check!(5);
check!(7);
}
#[test]
fn family_id_ascii_masks_centre() {
let ck = KmerOf::<ConstLen<5>>::from_ascii(b"ACGTA").unwrap().canonical();
let mut out = Vec::new();
ck.write_family_id_ascii(&mut out).unwrap();
assert_eq!(out.len(), 5);
assert_eq!(out[2], b'N');
// flanks of the id are the family's two flank pairs, in either orientation
let id_ascii = ck.family_id().to_ascii();
assert_eq!([out[0], out[1], out[3], out[4]], [id_ascii[0], id_ascii[1], id_ascii[3], id_ascii[4]]);
}
}