feat(phylo): expand genome distance metrics and refactor CLI arguments

Refactor the `phylo` command to support explicit distance metrics (Jaccard, Hamming) and SNP-based pseudo-alignments. Replace implicit metric selection with explicit requirements, such as requiring `--n-sites` for SNP analysis. Add a new `shared-kmers` output option and update documentation for performance characteristics of sparse vs dense formats.

- Update `phylo` CLI to support genome-vs-genome distances using k-mer content or SNPs, including options for NJ/UPGMA tree building and data export
- Replace `reindex` command with `convert` for in-place index evidence conversion (exact/approx/hybrid) and add `--upgrade-index` to `utils` for missing layer metadata
- Refactor `distance_to_compute` logic to remove implicit Jaccard implications for `--nj`/`--upgma` unless `--distance` is specified, and enforce `--n-sites` for `--pseudo-alignment`/`--sankoff`
- Add `shared_kmers` method to `Metrics` trait in `src/obikphylo/src/distance.rs` and new `write_shared_kmers` function in `src/obikmer/src/cmd/phylo/shared.rs` for CSV export
- Update `PackArgs::dense` documentation to clarify that dense format is faster for column-oriented distance matrices while sparse is better for single-row access
- Remove `requires_counts` definition and update assertions in `values.rs` to reflect new distance computation signatures and metric requirements
This commit is contained in:
Eric Coissac committed 2026-11-19 07:11:44 +01:00
1 parent 14433e80a5
commit c4eacb3f72
8 files changed
+135 -118

No files matched your search

+1 -1
View File
@@ -280,7 +280,7 @@ impl KmerIndex {
/// on-disk formats (`PersistentSparseBitMatrix` for presence,
/// `PersistentSparseCompactIntMatrix` for counts) — smaller and faster
/// for single-row access on real, sparse data; column-oriented access
/// (`--metric` distance matrices) is much slower on the sparse format.
/// (`phylo --distance` matrices) is much slower on the sparse format.
///
/// Public — and not moved out of this crate, despite covering the same
/// "maintenance operation on an already-built index" territory as
+1 -1
View File
@@ -19,7 +19,7 @@ pub struct PackArgs {
/// Pack presence and count matrices into the dense on-disk format instead
/// of the default sparse, deduplicated one. Dense is faster for
/// column-oriented access (`--metric` distance matrices); sparse is
/// column-oriented access (`phylo --distance` matrices); sparse is
/// smaller and faster for single-row access on real, sparse data.
#[arg(long, default_value_t = false)]
pub dense: bool,
+1 -1
View File
@@ -40,7 +40,7 @@ Option dependencies
--fasta, --vcf imply --pseudo-alignment
--tnt, --phyg, --iqtree imply --sankoff
--nj, --upgma, --shared-kmers use the Jaccard distance when --distance
--nj, --upgma use the Jaccard distance when --distance
is not given
--pseudo-alignment, --sankoff require --n-sites
--session-force requires --session";
+21 -25
View File
@@ -1,5 +1,6 @@
mod alignment_out;
mod args;
mod shared;
mod values;
mod iqtree;
mod phyg;
@@ -22,6 +23,7 @@ use obisys::{Reporter, Stage};
use tracing::info;
use alignment_out::write_alignment_outputs;
use shared::write_shared_kmers;
use values::{DistanceArg, NSitesArg, distance_to_compute};
use iqtree::write_iqtree;
use phyg::write_sankoff_phyg;
@@ -286,7 +288,7 @@ pub fn run(args: PhyloArgs) {
// different sample per flag — see `obikphylo::siblings::LayerReducer`'s
// own module docs).
let sankoff_requested = args.costs.sankoff || args.costs.tnt || args.costs.phyg || args.costs.iqtree;
let distance = distance_to_compute(args.dist.distance, args.dist.nj, args.dist.upgma, args.dist.shared_kmers);
let distance = distance_to_compute(args.dist.distance, args.dist.nj, args.dist.upgma);
let snp_kind = distance.and_then(DistanceArg::as_snp);
let need_alignment = args.alignment_output() || sankoff_requested;
let need_pairwise = sankoff_requested || snp_kind.is_some();
@@ -424,11 +426,22 @@ pub fn run(args: PhyloArgs) {
}
// ── Distance computation: classic whole-index metric vs. `snp-*` ───────────
// Nothing below (matrix, shared-kmer matrix, NJ/UPGMA trees) is wanted
// unless a distance was asked for (`--distance`) or implied
// (`--nj`/`--upgma`/`--shared-kmers`): stop here rather than compute and
// write a matrix nobody requested.
// Nothing below (distance matrix, NJ/UPGMA trees) is wanted unless a
// distance was asked for (`--distance`) or implied (`--nj`/`--upgma`): stop
// here rather than compute and write a matrix nobody requested.
// `--shared-kmers` alone only asks for its own matrix, not for a distance.
let Some(distance) = distance else {
if args.dist.shared_kmers {
info!("computing shared k-mer counts for {n} genome(s)");
let t = Stage::start("shared_kmers");
let shared = cache.shared_kmers(args.dist.presence_threshold).unwrap_or_else(|e| {
eprintln!("error computing shared k-mer counts: {e}");
std::process::exit(1);
});
rep.push(t.stop());
let kept: Vec<usize> = (0..n).filter(|&i| !exclude_mask[i]).collect();
write_shared_kmers(&shared, &labels, &kept, &args.output);
}
rep.print();
return;
};
@@ -437,7 +450,7 @@ pub fn run(args: PhyloArgs) {
let (matrix, shared_kmers) = match distance.as_classic() {
Some(metric) => {
info!("computing {metric:?} distances for {n} genome(s)");
let need_shared = args.dist.shared_kmers || args.dist.nj || args.dist.upgma;
let need_shared = args.dist.shared_kmers;
let t = Stage::start("distance");
let result = cache
.distance(metric, need_shared, args.dist.presence_threshold)
@@ -521,25 +534,8 @@ pub fn run(args: PhyloArgs) {
}
// ── Shared-kmer matrix → CSV ──────────────────────────────────────────────
if args.dist.shared_kmers {
if let Some(shared) = &shared_kmers {
let path = args.output.as_ref()
.map(|p| format!("{}_shared.csv", p.display()))
.unwrap_or_else(|| "shared.csv".into());
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
eprintln!("error creating {path}: {e}");
std::process::exit(1);
}));
write!(f, "genome").unwrap();
for &j in &kept { write!(f, ",{}", labels[j]).unwrap(); }
writeln!(f).unwrap();
for &i in &kept {
write!(f, "{}", labels[i]).unwrap();
for &j in &kept { write!(f, ",{}", shared[[i, j]]).unwrap(); }
writeln!(f).unwrap();
}
info!("shared-kmer matrix → {path}");
}
if let Some(shared) = shared_kmers.as_ref().filter(|_| args.dist.shared_kmers) {
write_shared_kmers(shared, &labels, &kept, &args.output);
}
// ── NJ tree ────────────────────────────────────────────────────────────────
+39
View File
@@ -0,0 +1,39 @@
//! `--shared-kmers`: the genome-by-genome matrix of shared k-mer counts.
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use ndarray::Array2;
use tracing::info;
/// Write `shared` (rows and columns of the genomes in `kept` only) as CSV to
/// `PREFIX_shared.csv`, or `shared.csv` without a prefix.
pub(super) fn write_shared_kmers(
shared: &Array2<u64>,
labels: &[String],
kept: &[usize],
output: &Option<PathBuf>,
) {
let path = output
.as_ref()
.map(|p| format!("{}_shared.csv", p.display()))
.unwrap_or_else(|| "shared.csv".into());
let mut f = BufWriter::new(File::create(&path).unwrap_or_else(|e| {
eprintln!("error creating {path}: {e}");
std::process::exit(1);
}));
write!(f, "genome").unwrap();
for &j in kept {
write!(f, ",{}", labels[j]).unwrap();
}
writeln!(f).unwrap();
for &i in kept {
write!(f, "{}", labels[i]).unwrap();
for &j in kept {
write!(f, ",{}", shared[[i, j]]).unwrap();
}
writeln!(f).unwrap();
}
info!("shared-kmer matrix → {path}");
}
+12 -19
View File
@@ -99,16 +99,12 @@ pub enum DistanceArg {
}
/// The distance matrix this invocation must compute, if any: the one asked for
/// with `--distance`, else `jaccard` when `--nj`, `--upgma` or
/// `--shared-kmers` needs a matrix to work from, else none — a matrix is never
/// computed (nor written) just because nothing said otherwise.
pub fn distance_to_compute(
distance: Option<DistanceArg>,
nj: bool,
upgma: bool,
shared_kmers: bool,
) -> Option<DistanceArg> {
distance.or_else(|| (nj || upgma || shared_kmers).then_some(DistanceArg::Jaccard))
/// with `--distance`, else `jaccard` when `--nj` or `--upgma` needs a matrix to
/// work from, else none — a matrix is never computed (nor written) just
/// because nothing said otherwise. `--shared-kmers` asks for its own matrix,
/// not for a distance.
pub fn distance_to_compute(distance: Option<DistanceArg>, nj: bool, upgma: bool) -> Option<DistanceArg> {
distance.or_else(|| (nj || upgma).then_some(DistanceArg::Jaccard))
}
#[cfg(test)]
@@ -117,27 +113,24 @@ mod distance_tests {
#[test]
fn nothing_asked_nothing_computed() {
assert!(distance_to_compute(None, false, false, false).is_none());
assert!(distance_to_compute(None, false, false).is_none());
}
#[test]
fn tree_and_shared_kmer_outputs_imply_jaccard() {
for (nj, upgma, shared) in [(true, false, false), (false, true, false), (false, false, true)] {
assert!(matches!(
distance_to_compute(None, nj, upgma, shared),
Some(DistanceArg::Jaccard)
));
fn tree_outputs_imply_jaccard() {
for (nj, upgma) in [(true, false), (false, true)] {
assert!(matches!(distance_to_compute(None, nj, upgma), Some(DistanceArg::Jaccard)));
}
}
#[test]
fn an_explicit_distance_wins() {
assert!(matches!(
distance_to_compute(Some(DistanceArg::Mash), true, true, true),
distance_to_compute(Some(DistanceArg::Mash), true, true),
Some(DistanceArg::Mash)
));
assert!(matches!(
distance_to_compute(Some(DistanceArg::SnpJc), false, false, false),
distance_to_compute(Some(DistanceArg::SnpJc), false, false),
Some(DistanceArg::SnpJc)
));
}
+58 -69
View File
@@ -51,20 +51,6 @@ pub struct DistanceOutput {
pub shared_kmers: Option<Array2<u64>>,
}
impl DistanceMetric {
pub fn requires_counts(self) -> bool {
matches!(
self,
DistanceMetric::BrayCurtis
| DistanceMetric::RelfreqBrayCurtis
| DistanceMetric::Euclidean
| DistanceMetric::RelfreqEuclidean
| DistanceMetric::Hellinger
| DistanceMetric::HellingerEuclidean
)
}
}
// ── Aggregators — bounded parallel reduction of every cached layer's own
// partial (of the wanted content kind) into one global partial, then handed
// off to `CountPartials`/`BitPartials`'s own provided finalisation methods.
@@ -188,10 +174,33 @@ impl BitPartials for AggregatedBit<'_> {
/// extension trait in this codebase.
pub trait Metrics {
fn distance(&self, metric: DistanceMetric, shared_kmers: bool, presence_threshold: u32) -> OKIResult<DistanceOutput>;
/// Genome-by-genome matrix of the number of k-mers shared by each pair of
/// genomes — on a count index a k-mer is present when its count reaches
/// `presence_threshold`. Independent of any distance: nothing else is
/// computed.
fn shared_kmers(&self, presence_threshold: u32) -> OKIResult<Array2<u64>>;
}
impl Metrics for IndexCache {
fn shared_kmers(&self, presence_threshold: u32) -> OKIResult<Array2<u64>> {
let n_genomes = self.meta().genomes().len();
if n_genomes < 2 {
return Err(OKIError::InvalidInput(
"shared k-mer counts require at least 2 genomes in the index".into(),
));
}
let (inter, _union) = if self.meta().config().with_counts {
AggregatedCount { cache: self, n_genomes }.partial_threshold_jaccard(presence_threshold)
} else {
AggregatedBit { cache: self, n_genomes }.partial_jaccard()
};
Ok(inter)
}
fn distance(&self, metric: DistanceMetric, shared_kmers: bool, presence_threshold: u32) -> OKIResult<DistanceOutput> {
use DistanceMetric::*;
let n_genomes = self.meta().genomes().len();
if n_genomes < 2 {
return Err(OKIError::InvalidInput(
@@ -199,64 +208,44 @@ impl Metrics for IndexCache {
));
}
let use_counts = self.meta().config().with_counts;
if metric.requires_counts() && !use_counts {
return Err(OKIError::InvalidInput(format!(
"{metric:?} requires a count index (with_counts = true)"
)));
}
let kmer_size = self.meta().config().kmer_size;
let use_counts = self.meta().config().with_counts;
let count = AggregatedCount { cache: self, n_genomes };
let bit = AggregatedBit { cache: self, n_genomes };
if use_counts {
let agg = AggregatedCount { cache: self, n_genomes };
// One construction decides both what the index can serve and how:
// the six count-based distances need a count index, Hamming needs a
// presence/absence one, Jaccard and Mash work on either.
let matrix = match (use_counts, metric) {
(true, BrayCurtis) => count.bray_dist_matrix(),
(true, RelfreqBrayCurtis) => count.relfreq_bray_dist_matrix(),
(true, Euclidean) => count.euclidean_dist_matrix(),
(true, RelfreqEuclidean) => count.relfreq_euclidean_dist_matrix(),
(true, Hellinger) => count.hellinger_dist_matrix(),
(true, HellingerEuclidean) => count.hellinger_euclidean_dist_matrix(),
(true, Jaccard) => count.threshold_jaccard_dist_matrix(presence_threshold),
(true, Mash) => count.threshold_mash_dist_matrix(kmer_size, presence_threshold),
(true, Hamming) => {
return Err(OKIError::InvalidInput(
"Hamming is only available for presence/absence indexes".into(),
));
}
(false, Jaccard) => bit.jaccard_dist_matrix(),
(false, Mash) => bit.mash_dist_matrix(kmer_size),
(false, Hamming) => bit.hamming_dist_matrix().mapv(|v| v as f64),
(
false,
BrayCurtis | RelfreqBrayCurtis | Euclidean | RelfreqEuclidean | Hellinger | HellingerEuclidean,
) => {
return Err(OKIError::InvalidInput(format!(
"{metric:?} requires a count index (with_counts = true); \
on an index without counts use --distance jaccard, mash or hamming"
)));
}
};
let matrix = match metric {
DistanceMetric::BrayCurtis => agg.bray_dist_matrix(),
DistanceMetric::RelfreqBrayCurtis => agg.relfreq_bray_dist_matrix(),
DistanceMetric::Euclidean => agg.euclidean_dist_matrix(),
DistanceMetric::RelfreqEuclidean => agg.relfreq_euclidean_dist_matrix(),
DistanceMetric::Hellinger => agg.hellinger_dist_matrix(),
DistanceMetric::HellingerEuclidean => agg.hellinger_euclidean_dist_matrix(),
DistanceMetric::Jaccard => agg.threshold_jaccard_dist_matrix(presence_threshold),
DistanceMetric::Mash => agg.threshold_mash_dist_matrix(kmer_size, presence_threshold),
DistanceMetric::Hamming => {
return Err(OKIError::InvalidInput(
"Hamming is only available for presence/absence indexes".into(),
));
}
};
let shared = shared_kmers.then(|| self.shared_kmers(presence_threshold)).transpose()?;
let shared = if shared_kmers {
let (inter, _) = agg.partial_threshold_jaccard(presence_threshold);
Some(inter)
} else {
None
};
Ok(DistanceOutput { matrix, shared_kmers: shared })
} else {
let agg = AggregatedBit { cache: self, n_genomes };
let matrix = match metric {
DistanceMetric::Jaccard => agg.jaccard_dist_matrix(),
DistanceMetric::Mash => agg.mash_dist_matrix(kmer_size),
DistanceMetric::Hamming => agg.hamming_dist_matrix().mapv(|v| v as f64),
other => {
return Err(OKIError::InvalidInput(format!(
"{other:?} requires a count index; use --metric jaccard or --metric hamming"
)));
}
};
let shared = if shared_kmers {
let (inter, _) = agg.partial_jaccard();
Some(inter)
} else {
None
};
Ok(DistanceOutput { matrix, shared_kmers: shared })
}
Ok(DistanceOutput { matrix, shared_kmers: shared })
}
}