diff --git a/DevDocMD/index.md b/DevDocMD/index.md index 4c879040..797c04f1 100644 --- a/DevDocMD/index.md +++ b/DevDocMD/index.md @@ -13,11 +13,11 @@ | `query` | Query an index with sequences and annotate matches | | `dump` | Dump all indexed k-mers as CSV (kmer + per-genome counts or presence); supports the shared [kmer filtering](implementation/filtering.md) system; `--head N` limits output to the first N k-mers | | `annotate` | Add or update genome metadata from a CSV file; or dump metadata as CSV | -| `phylo` | Compute pairwise evolutionary-distance proxies between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1); optionally a central-position SNP/Sankoff calibration with exports for TNT/PhyG/IQ-TREE (see [evolutionary distances](theory/evolutionary_distances.md)) | +| `phylo` | Genome-vs-genome distances, trees and SNP-based pseudo-alignments. `--distance` selects either a whole-genome distance computed from k-mer content (`jaccard`, `mash`, `hamming`, `bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean`; `--presence-threshold N` converts counts to presence for Jaccard/Mash on count indexes; `hamming` is currently refused on count indexes) or a `snp-*` distance (`snp-raw`, `snp-jc`, `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv`; optional `--gamma-shape`) computed from central-position SNPs of k-mer families. `--nj`/`--upgma` build trees (Jaccard if no `--distance`); `--shared-kmers` writes the shared-k-mer count matrix on its own. SNP analyses read the sibling annex, built on demand (`--sibling-annex` forces it), with site selection/sampling options (`--n-sites`, `--quartet`, `--pair-balance`, `--entropy`, `--free-loss`, `--no-ambiguity`/`--only-diploid`, `--session`), a pseudo-alignment export (`--pseudo-alignment`, `--fasta`, `--vcf`) and Sankoff cost calibration with TNT/PhyG/IQ-TREE exports. See [evolutionary distances](theory/evolutionary_distances.md) | | `unitig` | Build a global de Bruijn graph across all partitions and enumerate its unitigs as FASTA; supports the shared [kmer filtering](implementation/filtering.md) system | | `select` | Project and/or aggregate genome columns into a new or in-place index; the column-axis counterpart of `filter` (see [select](implementation/select.md)) | | `estimate` | Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing | -| `reindex` | Convert an index's evidence in-place: exact ↔ approx | +| `convert` | Convert an index's evidence representation (exact / approximate / hybrid), in place | | `utils` | Miscellaneous index utilities: `--new-label NEW=OLD` renames a genome label; `--upgrade-index` adds missing `layer_meta.json` to old indexes | | `pack` | Pack per-column matrix files into single-file format to reduce query I/O | diff --git a/src/obikindex/src/index/kmer_index.rs b/src/obikindex/src/index/kmer_index.rs index 9fe2b112..a0d2b92c 100644 --- a/src/obikindex/src/index/kmer_index.rs +++ b/src/obikindex/src/index/kmer_index.rs @@ -280,7 +280,7 @@ impl KmerIndex { /// on-disk formats (`PersistentSparseBitMatrix` for presence, /// `PersistentSparseCompactIntMatrix` for counts) — smaller and faster /// for single-row access on real, sparse data; column-oriented access - /// (`--metric` distance matrices) is much slower on the sparse format. + /// (`phylo --distance` matrices) is much slower on the sparse format. /// /// Public — and not moved out of this crate, despite covering the same /// "maintenance operation on an already-built index" territory as diff --git a/src/obikmer/src/cmd/pack/mod.rs b/src/obikmer/src/cmd/pack/mod.rs index c3073acc..0e00048e 100644 --- a/src/obikmer/src/cmd/pack/mod.rs +++ b/src/obikmer/src/cmd/pack/mod.rs @@ -19,7 +19,7 @@ pub struct PackArgs { /// Pack presence and count matrices into the dense on-disk format instead /// of the default sparse, deduplicated one. Dense is faster for - /// column-oriented access (`--metric` distance matrices); sparse is + /// column-oriented access (`phylo --distance` matrices); sparse is /// smaller and faster for single-row access on real, sparse data. #[arg(long, default_value_t = false)] pub dense: bool, diff --git a/src/obikmer/src/cmd/phylo/args/mod.rs b/src/obikmer/src/cmd/phylo/args/mod.rs index cdc693d7..9b53660e 100644 --- a/src/obikmer/src/cmd/phylo/args/mod.rs +++ b/src/obikmer/src/cmd/phylo/args/mod.rs @@ -40,7 +40,7 @@ Option dependencies --fasta, --vcf imply --pseudo-alignment --tnt, --phyg, --iqtree imply --sankoff - --nj, --upgma, --shared-kmers use the Jaccard distance when --distance + --nj, --upgma use the Jaccard distance when --distance is not given --pseudo-alignment, --sankoff require --n-sites --session-force requires --session"; diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 66944e89..20a49868 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -1,5 +1,6 @@ mod alignment_out; mod args; +mod shared; mod values; mod iqtree; mod phyg; @@ -22,6 +23,7 @@ use obisys::{Reporter, Stage}; use tracing::info; use alignment_out::write_alignment_outputs; +use shared::write_shared_kmers; use values::{DistanceArg, NSitesArg, distance_to_compute}; use iqtree::write_iqtree; use phyg::write_sankoff_phyg; @@ -286,7 +288,7 @@ pub fn run(args: PhyloArgs) { // different sample per flag — see `obikphylo::siblings::LayerReducer`'s // own module docs). let sankoff_requested = args.costs.sankoff || args.costs.tnt || args.costs.phyg || args.costs.iqtree; - let distance = distance_to_compute(args.dist.distance, args.dist.nj, args.dist.upgma, args.dist.shared_kmers); + let distance = distance_to_compute(args.dist.distance, args.dist.nj, args.dist.upgma); let snp_kind = distance.and_then(DistanceArg::as_snp); let need_alignment = args.alignment_output() || sankoff_requested; let need_pairwise = sankoff_requested || snp_kind.is_some(); @@ -424,11 +426,22 @@ pub fn run(args: PhyloArgs) { } // ── Distance computation: classic whole-index metric vs. `snp-*` ─────────── - // Nothing below (matrix, shared-kmer matrix, NJ/UPGMA trees) is wanted - // unless a distance was asked for (`--distance`) or implied - // (`--nj`/`--upgma`/`--shared-kmers`): stop here rather than compute and - // write a matrix nobody requested. + // Nothing below (distance matrix, NJ/UPGMA trees) is wanted unless a + // distance was asked for (`--distance`) or implied (`--nj`/`--upgma`): stop + // here rather than compute and write a matrix nobody requested. + // `--shared-kmers` alone only asks for its own matrix, not for a distance. let Some(distance) = distance else { + if args.dist.shared_kmers { + info!("computing shared k-mer counts for {n} genome(s)"); + let t = Stage::start("shared_kmers"); + let shared = cache.shared_kmers(args.dist.presence_threshold).unwrap_or_else(|e| { + eprintln!("error computing shared k-mer counts: {e}"); + std::process::exit(1); + }); + rep.push(t.stop()); + let kept: Vec = (0..n).filter(|&i| !exclude_mask[i]).collect(); + write_shared_kmers(&shared, &labels, &kept, &args.output); + } rep.print(); return; }; @@ -437,7 +450,7 @@ pub fn run(args: PhyloArgs) { let (matrix, shared_kmers) = match distance.as_classic() { Some(metric) => { info!("computing {metric:?} distances for {n} genome(s)"); - let need_shared = args.dist.shared_kmers || args.dist.nj || args.dist.upgma; + let need_shared = args.dist.shared_kmers; let t = Stage::start("distance"); let result = cache .distance(metric, need_shared, args.dist.presence_threshold) @@ -521,25 +534,8 @@ pub fn run(args: PhyloArgs) { } // ── Shared-kmer matrix → CSV ────────────────────────────────────────────── - if args.dist.shared_kmers { - if let Some(shared) = &shared_kmers { - let path = args.output.as_ref() - .map(|p| format!("{}_shared.csv", p.display())) - .unwrap_or_else(|| "shared.csv".into()); - let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| { - eprintln!("error creating {path}: {e}"); - std::process::exit(1); - })); - write!(f, "genome").unwrap(); - for &j in &kept { write!(f, ",{}", labels[j]).unwrap(); } - writeln!(f).unwrap(); - for &i in &kept { - write!(f, "{}", labels[i]).unwrap(); - for &j in &kept { write!(f, ",{}", shared[[i, j]]).unwrap(); } - writeln!(f).unwrap(); - } - info!("shared-kmer matrix → {path}"); - } + if let Some(shared) = shared_kmers.as_ref().filter(|_| args.dist.shared_kmers) { + write_shared_kmers(shared, &labels, &kept, &args.output); } // ── NJ tree ──────────────────────────────────────────────────────────────── diff --git a/src/obikmer/src/cmd/phylo/shared.rs b/src/obikmer/src/cmd/phylo/shared.rs new file mode 100644 index 00000000..21d014dc --- /dev/null +++ b/src/obikmer/src/cmd/phylo/shared.rs @@ -0,0 +1,39 @@ +//! `--shared-kmers`: the genome-by-genome matrix of shared k-mer counts. + +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::PathBuf; + +use ndarray::Array2; +use tracing::info; + +/// Write `shared` (rows and columns of the genomes in `kept` only) as CSV to +/// `PREFIX_shared.csv`, or `shared.csv` without a prefix. +pub(super) fn write_shared_kmers( + shared: &Array2, + labels: &[String], + kept: &[usize], + output: &Option, +) { + let path = output + .as_ref() + .map(|p| format!("{}_shared.csv", p.display())) + .unwrap_or_else(|| "shared.csv".into()); + let mut f = BufWriter::new(File::create(&path).unwrap_or_else(|e| { + eprintln!("error creating {path}: {e}"); + std::process::exit(1); + })); + write!(f, "genome").unwrap(); + for &j in kept { + write!(f, ",{}", labels[j]).unwrap(); + } + writeln!(f).unwrap(); + for &i in kept { + write!(f, "{}", labels[i]).unwrap(); + for &j in kept { + write!(f, ",{}", shared[[i, j]]).unwrap(); + } + writeln!(f).unwrap(); + } + info!("shared-kmer matrix → {path}"); +} diff --git a/src/obikmer/src/cmd/phylo/values.rs b/src/obikmer/src/cmd/phylo/values.rs index b420bf62..45727a22 100644 --- a/src/obikmer/src/cmd/phylo/values.rs +++ b/src/obikmer/src/cmd/phylo/values.rs @@ -99,16 +99,12 @@ pub enum DistanceArg { } /// The distance matrix this invocation must compute, if any: the one asked for -/// with `--distance`, else `jaccard` when `--nj`, `--upgma` or -/// `--shared-kmers` needs a matrix to work from, else none — a matrix is never -/// computed (nor written) just because nothing said otherwise. -pub fn distance_to_compute( - distance: Option, - nj: bool, - upgma: bool, - shared_kmers: bool, -) -> Option { - distance.or_else(|| (nj || upgma || shared_kmers).then_some(DistanceArg::Jaccard)) +/// with `--distance`, else `jaccard` when `--nj` or `--upgma` needs a matrix to +/// work from, else none — a matrix is never computed (nor written) just +/// because nothing said otherwise. `--shared-kmers` asks for its own matrix, +/// not for a distance. +pub fn distance_to_compute(distance: Option, nj: bool, upgma: bool) -> Option { + distance.or_else(|| (nj || upgma).then_some(DistanceArg::Jaccard)) } #[cfg(test)] @@ -117,27 +113,24 @@ mod distance_tests { #[test] fn nothing_asked_nothing_computed() { - assert!(distance_to_compute(None, false, false, false).is_none()); + assert!(distance_to_compute(None, false, false).is_none()); } #[test] - fn tree_and_shared_kmer_outputs_imply_jaccard() { - for (nj, upgma, shared) in [(true, false, false), (false, true, false), (false, false, true)] { - assert!(matches!( - distance_to_compute(None, nj, upgma, shared), - Some(DistanceArg::Jaccard) - )); + fn tree_outputs_imply_jaccard() { + for (nj, upgma) in [(true, false), (false, true)] { + assert!(matches!(distance_to_compute(None, nj, upgma), Some(DistanceArg::Jaccard))); } } #[test] fn an_explicit_distance_wins() { assert!(matches!( - distance_to_compute(Some(DistanceArg::Mash), true, true, true), + distance_to_compute(Some(DistanceArg::Mash), true, true), Some(DistanceArg::Mash) )); assert!(matches!( - distance_to_compute(Some(DistanceArg::SnpJc), false, false, false), + distance_to_compute(Some(DistanceArg::SnpJc), false, false), Some(DistanceArg::SnpJc) )); } diff --git a/src/obikphylo/src/distance.rs b/src/obikphylo/src/distance.rs index 39731ee6..1d5a723d 100644 --- a/src/obikphylo/src/distance.rs +++ b/src/obikphylo/src/distance.rs @@ -51,20 +51,6 @@ pub struct DistanceOutput { pub shared_kmers: Option>, } -impl DistanceMetric { - pub fn requires_counts(self) -> bool { - matches!( - self, - DistanceMetric::BrayCurtis - | DistanceMetric::RelfreqBrayCurtis - | DistanceMetric::Euclidean - | DistanceMetric::RelfreqEuclidean - | DistanceMetric::Hellinger - | DistanceMetric::HellingerEuclidean - ) - } -} - // ── Aggregators — bounded parallel reduction of every cached layer's own // partial (of the wanted content kind) into one global partial, then handed // off to `CountPartials`/`BitPartials`'s own provided finalisation methods. @@ -188,10 +174,33 @@ impl BitPartials for AggregatedBit<'_> { /// extension trait in this codebase. pub trait Metrics { fn distance(&self, metric: DistanceMetric, shared_kmers: bool, presence_threshold: u32) -> OKIResult; + + /// Genome-by-genome matrix of the number of k-mers shared by each pair of + /// genomes — on a count index a k-mer is present when its count reaches + /// `presence_threshold`. Independent of any distance: nothing else is + /// computed. + fn shared_kmers(&self, presence_threshold: u32) -> OKIResult>; } impl Metrics for IndexCache { + fn shared_kmers(&self, presence_threshold: u32) -> OKIResult> { + let n_genomes = self.meta().genomes().len(); + if n_genomes < 2 { + return Err(OKIError::InvalidInput( + "shared k-mer counts require at least 2 genomes in the index".into(), + )); + } + let (inter, _union) = if self.meta().config().with_counts { + AggregatedCount { cache: self, n_genomes }.partial_threshold_jaccard(presence_threshold) + } else { + AggregatedBit { cache: self, n_genomes }.partial_jaccard() + }; + Ok(inter) + } + fn distance(&self, metric: DistanceMetric, shared_kmers: bool, presence_threshold: u32) -> OKIResult { + use DistanceMetric::*; + let n_genomes = self.meta().genomes().len(); if n_genomes < 2 { return Err(OKIError::InvalidInput( @@ -199,64 +208,44 @@ impl Metrics for IndexCache { )); } - let use_counts = self.meta().config().with_counts; - if metric.requires_counts() && !use_counts { - return Err(OKIError::InvalidInput(format!( - "{metric:?} requires a count index (with_counts = true)" - ))); - } - let kmer_size = self.meta().config().kmer_size; + let use_counts = self.meta().config().with_counts; + let count = AggregatedCount { cache: self, n_genomes }; + let bit = AggregatedBit { cache: self, n_genomes }; - if use_counts { - let agg = AggregatedCount { cache: self, n_genomes }; + // One construction decides both what the index can serve and how: + // the six count-based distances need a count index, Hamming needs a + // presence/absence one, Jaccard and Mash work on either. + let matrix = match (use_counts, metric) { + (true, BrayCurtis) => count.bray_dist_matrix(), + (true, RelfreqBrayCurtis) => count.relfreq_bray_dist_matrix(), + (true, Euclidean) => count.euclidean_dist_matrix(), + (true, RelfreqEuclidean) => count.relfreq_euclidean_dist_matrix(), + (true, Hellinger) => count.hellinger_dist_matrix(), + (true, HellingerEuclidean) => count.hellinger_euclidean_dist_matrix(), + (true, Jaccard) => count.threshold_jaccard_dist_matrix(presence_threshold), + (true, Mash) => count.threshold_mash_dist_matrix(kmer_size, presence_threshold), + (true, Hamming) => { + return Err(OKIError::InvalidInput( + "Hamming is only available for presence/absence indexes".into(), + )); + } + (false, Jaccard) => bit.jaccard_dist_matrix(), + (false, Mash) => bit.mash_dist_matrix(kmer_size), + (false, Hamming) => bit.hamming_dist_matrix().mapv(|v| v as f64), + ( + false, + BrayCurtis | RelfreqBrayCurtis | Euclidean | RelfreqEuclidean | Hellinger | HellingerEuclidean, + ) => { + return Err(OKIError::InvalidInput(format!( + "{metric:?} requires a count index (with_counts = true); \ + on an index without counts use --distance jaccard, mash or hamming" + ))); + } + }; - let matrix = match metric { - DistanceMetric::BrayCurtis => agg.bray_dist_matrix(), - DistanceMetric::RelfreqBrayCurtis => agg.relfreq_bray_dist_matrix(), - DistanceMetric::Euclidean => agg.euclidean_dist_matrix(), - DistanceMetric::RelfreqEuclidean => agg.relfreq_euclidean_dist_matrix(), - DistanceMetric::Hellinger => agg.hellinger_dist_matrix(), - DistanceMetric::HellingerEuclidean => agg.hellinger_euclidean_dist_matrix(), - DistanceMetric::Jaccard => agg.threshold_jaccard_dist_matrix(presence_threshold), - DistanceMetric::Mash => agg.threshold_mash_dist_matrix(kmer_size, presence_threshold), - DistanceMetric::Hamming => { - return Err(OKIError::InvalidInput( - "Hamming is only available for presence/absence indexes".into(), - )); - } - }; + let shared = shared_kmers.then(|| self.shared_kmers(presence_threshold)).transpose()?; - let shared = if shared_kmers { - let (inter, _) = agg.partial_threshold_jaccard(presence_threshold); - Some(inter) - } else { - None - }; - - Ok(DistanceOutput { matrix, shared_kmers: shared }) - } else { - let agg = AggregatedBit { cache: self, n_genomes }; - - let matrix = match metric { - DistanceMetric::Jaccard => agg.jaccard_dist_matrix(), - DistanceMetric::Mash => agg.mash_dist_matrix(kmer_size), - DistanceMetric::Hamming => agg.hamming_dist_matrix().mapv(|v| v as f64), - other => { - return Err(OKIError::InvalidInput(format!( - "{other:?} requires a count index; use --metric jaccard or --metric hamming" - ))); - } - }; - - let shared = if shared_kmers { - let (inter, _) = agg.partial_jaccard(); - Some(inter) - } else { - None - }; - - Ok(DistanceOutput { matrix, shared_kmers: shared }) - } + Ok(DistanceOutput { matrix, shared_kmers: shared }) } }