From dd889854cb647e66638c18b1650c374f0fe2e3c0 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Thu, 13 Aug 2026 16:33:22 +0200 Subject: [PATCH] rename distance subcommand to phylo Rename the distance CLI subcommand to phylo across the codebase, documentation, and build configurations. Relocate source files from cmd/distance/ to a dedicated cmd/phylo/ module, update all internal routing references, and adjust benchmark scripts and Makefile targets to reflect the new command name. --- Makefile | 2 +- README.md | 10 +++++---- UserDocMD/architecture.md | 2 +- UserDocMD/index.md | 2 +- UserDocMD/usage/{distance.md => phylo.md} | 6 ++--- benchmark/Makefile | 22 +++++++++---------- benchmark/build_reference_dist.py | 4 ++-- benchmark/compare_all_dist.py | 4 ++-- docmd/implementation/filtering.md | 4 ++-- docmd/index.md | 2 +- docmd/theory/evolutionary_distances.md | 22 +++++++++---------- mkdocs-user.yml | 2 +- src/obikmer/src/cmd/mod.rs | 2 +- .../src/cmd/{distance => phylo}/iqtree.rs | 0 .../src/cmd/{distance => phylo}/mod.rs | 4 ++-- .../src/cmd/{distance => phylo}/phyg.rs | 0 .../src/cmd/{distance => phylo}/sankoff.rs | 0 .../src/cmd/{distance => phylo}/tnt.rs | 0 src/obikmer/src/main.rs | 7 +++--- 19 files changed, 49 insertions(+), 46 deletions(-) rename UserDocMD/usage/{distance.md => phylo.md} (96%) rename src/obikmer/src/cmd/{distance => phylo}/iqtree.rs (100%) rename src/obikmer/src/cmd/{distance => phylo}/mod.rs (99%) rename src/obikmer/src/cmd/{distance => phylo}/phyg.rs (100%) rename src/obikmer/src/cmd/{distance => phylo}/sankoff.rs (100%) rename src/obikmer/src/cmd/{distance => phylo}/tnt.rs (100%) diff --git a/Makefile b/Makefile index 5889db68..bebe14f9 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,7 @@ bump-version: .PHONY: release release: bump-version - @jj auto-describe + @jj auto-doc @jj git push --change @ @new_version=$$(grep '^version = ' $(CARGO_TOML) | head -n 1 | sed 's/version = "\(.*\)"/\1/'); \ git_hash=$$(jj log -r @ --no-graph -T 'commit_id'); \ diff --git a/README.md b/README.md index 0fb073e3..a9876151 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,11 @@ Non-ACGT characters act as hard breaks between k-mer segments in all formats. Annotates each sequence with per-genome k-mer match counts and optional per-position coverage vectors (--detail). Parallel over sequence chunks. - distance Compute a pairwise Bray-Curtis or Jaccard distance matrix - between all indexed genomes. - Optionally outputs a Newick NJ or UPGMA tree. + phylo Compute pairwise evolutionary-distance proxies between all + indexed genomes (Bray-Curtis, Jaccard, etc.), optionally + a Newick NJ/UPGMA tree, and optionally a central-position + SNP/Sankoff calibration with exports for external + phylogenetic tools (TNT, PhyG, IQ-TREE). annotate Add or update genome metadata (taxonomy, etc.) from a CSV file; or dump the current metadata as CSV. estimate Dry-run: resolve and print approximate-index parameters @@ -106,7 +108,7 @@ obikmer reindex --approx -z 5 --evidence-bits 8 index/ obikmer query index/ reads.fq.gz > annotated.fa # Pairwise distances -obikmer distance index/ > distances.tsv +obikmer phylo index/ > distances.tsv ``` ## Parameter constraints diff --git a/UserDocMD/architecture.md b/UserDocMD/architecture.md index 87bf9b91..7c1da714 100644 --- a/UserDocMD/architecture.md +++ b/UserDocMD/architecture.md @@ -20,7 +20,7 @@ An index directory is organized as `KmerIndex → partitions → layers`, with a ## Parallel execution and NUMA awareness -Partition-level work (index construction, `merge`, `filter`, `reindex`, `select`, `distance`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc: +Partition-level work (index construction, `merge`, `filter`, `reindex`, `select`, `phylo`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc: - On a multi-socket / multi-NUMA-node machine, one thread pool is pinned per NUMA node, and each partition is processed entirely by threads pinned to one node — keeping the memory a partition touches local to that node's DRAM. This matters because touching kmer data across NUMA nodes without pinning can degrade throughput by an order of magnitude or more on large multi-socket machines. - On a single-socket machine, Apple Silicon, or if hwloc cannot report NUMA topology, all cores are treated as one node with no pinning and negligible overhead — this is the default behavior on macOS. diff --git a/UserDocMD/index.md b/UserDocMD/index.md index cd1e2806..16248c99 100644 --- a/UserDocMD/index.md +++ b/UserDocMD/index.md @@ -24,7 +24,7 @@ All functionality is exposed through a single binary, `obikmer`, organized as su | [`query`](usage/query.md) | Query an index with sequences and annotate matches | | [`dump`](usage/dump.md) | Dump indexed kmers as CSV | | [`annotate`](usage/annotate.md) | Add, update, or dump genome metadata | -| [`distance`](usage/distance.md) | Compute pairwise distance matrices and phylogenetic exports | +| [`phylo`](usage/phylo.md) | Compute pairwise evolutionary-distance proxies, trees, and phylogenetic exports | | [`unitig`](usage/unitig.md) | Dump the unitigs of an index as FASTA | | [`estimate`](usage/estimate.md) | Estimate approximate-index parameters before indexing | | [`reindex`](usage/reindex.md) | Convert an index's evidence representation (exact ↔ approximate) | diff --git a/UserDocMD/usage/distance.md b/UserDocMD/usage/phylo.md similarity index 96% rename from UserDocMD/usage/distance.md rename to UserDocMD/usage/phylo.md index 9a55ccf4..4a71bd0c 100644 --- a/UserDocMD/usage/distance.md +++ b/UserDocMD/usage/phylo.md @@ -1,9 +1,9 @@ -# distance +# phylo -Compute pairwise distance matrices between the genomes stored in an index, optionally build trees (NJ/UPGMA), and optionally derive a central-position SNP model with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE). +Compute pairwise evolutionary-distance proxies between the genomes stored in an index — a plain distance matrix, optionally trees (NJ/UPGMA), and optionally a central-position SNP model with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE). ```bash -obikmer distance INDEX [OPTIONS] +obikmer phylo INDEX [OPTIONS] ``` ## Arguments diff --git a/benchmark/Makefile b/benchmark/Makefile index 14a33583..5bd9b7a6 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -89,50 +89,50 @@ $(REF_DIST_CSVS) &: $(REF_NPZS) build_reference_dist.py reference_dist: $(REF_DIST_CSVS) -# ── obikmer distance (presence index) ──────────────────────────────────────── +# ── obikmer phylo (presence index) ────────────────────────────────────────── $(OBIKMER_PRESENCE_DIST) &: global_index_presence/index.done $(BINARY) mkdir -p obikmer_dist/presence - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/presence/jaccard \ --metric jaccard --shared-kmers --nj \ global_index_presence - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/presence/hamming \ --metric hamming --nj \ global_index_presence obikmer_dist_presence: $(OBIKMER_PRESENCE_DIST) -# ── obikmer distance (count index) ─────────────────────────────────────────── +# ── obikmer phylo (count index) ───────────────────────────────────────────── $(OBIKMER_COUNT_DIST) &: global_index_count/index.done $(BINARY) mkdir -p obikmer_dist/count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/jaccard \ --metric jaccard --shared-kmers --nj \ global_index_count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/bray_curtis \ --metric bray-curtis --nj \ global_index_count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/relfreq_bray_curtis \ --metric relfreq-bray-curtis --nj \ global_index_count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/euclidean \ --metric euclidean --nj \ global_index_count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/relfreq_euclidean \ --metric relfreq-euclidean --nj \ global_index_count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/hellinger \ --metric hellinger --nj \ global_index_count - $(BINARY) distance \ + $(BINARY) phylo \ --output obikmer_dist/count/hellinger_euclidean \ --metric hellinger-euclidean --nj \ global_index_count diff --git a/benchmark/build_reference_dist.py b/benchmark/build_reference_dist.py index edee8586..bd851d67 100755 --- a/benchmark/build_reference_dist.py +++ b/benchmark/build_reference_dist.py @@ -2,10 +2,10 @@ """Compute reference pairwise distance matrices from per-specimen .npz kmer indexes. Reads all .npz files in reference_index/ (each containing sorted uint64 `kmers` -and uint32 `counts`), computes all distance metrics supported by `obikmer distance`, +and uint32 `counts`), computes all distance metrics supported by `obikmer phylo`, and writes one CSV per metric to reference_dist/. -Output CSV format matches `obikmer distance --output`: +Output CSV format matches `obikmer phylo --output`: - first row: "genome", then specimen names - subsequent rows: specimen name, then float or int values diff --git a/benchmark/compare_all_dist.py b/benchmark/compare_all_dist.py index f34ddc60..390feefc 100755 --- a/benchmark/compare_all_dist.py +++ b/benchmark/compare_all_dist.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""Compare all reference distance matrices against obikmer distance outputs. +"""Compare all reference distance matrices against obikmer phylo outputs. Reads from: reference_dist/ — ground-truth matrices computed by build_reference_dist.py - obikmer_dist/ — matrices produced by `obikmer distance` + obikmer_dist/ — matrices produced by `obikmer phylo` Handles label reordering: both matrices are sorted by genome label before element-wise comparison, so column/row order differences are irrelevant. diff --git a/docmd/implementation/filtering.md b/docmd/implementation/filtering.md index b45938e6..1021cbae 100644 --- a/docmd/implementation/filtering.md +++ b/docmd/implementation/filtering.md @@ -289,14 +289,14 @@ obikmer dump myindex --head 100 obikmer dump myindex --head 20 --ingroup "species=Betula_nana" --min-count 1 ``` -### `distance --presence-threshold N` +### `phylo --presence-threshold N` When computing Jaccard distance on a **count index**, a k-mer is considered present in a genome if its count is ≥ N (default 1). This option is independent of the `--presence-threshold` used in filtering. ```sh # Jaccard treating kmers with count ≥ 2 as present -obikmer distance myindex --metric jaccard --presence-threshold 2 +obikmer phylo myindex --metric jaccard --presence-threshold 2 ``` This parameter has no effect on presence/absence indexes (where values are already 0/1) or on metrics other than Jaccard. diff --git a/docmd/index.md b/docmd/index.md index 812cc86e..4c879040 100644 --- a/docmd/index.md +++ b/docmd/index.md @@ -13,7 +13,7 @@ | `query` | Query an index with sequences and annotate matches | | `dump` | Dump all indexed k-mers as CSV (kmer + per-genome counts or presence); supports the shared [kmer filtering](implementation/filtering.md) system; `--head N` limits output to the first N k-mers | | `annotate` | Add or update genome metadata from a CSV file; or dump metadata as CSV | -| `distance` | Compute pairwise distance matrix between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1) | +| `phylo` | Compute pairwise evolutionary-distance proxies between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1); optionally a central-position SNP/Sankoff calibration with exports for TNT/PhyG/IQ-TREE (see [evolutionary distances](theory/evolutionary_distances.md)) | | `unitig` | Build a global de Bruijn graph across all partitions and enumerate its unitigs as FASTA; supports the shared [kmer filtering](implementation/filtering.md) system | | `select` | Project and/or aggregate genome columns into a new or in-place index; the column-axis counterpart of `filter` (see [select](implementation/select.md)) | | `estimate` | Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing | diff --git a/docmd/theory/evolutionary_distances.md b/docmd/theory/evolutionary_distances.md index f6b20da1..bf5a3be6 100644 --- a/docmd/theory/evolutionary_distances.md +++ b/docmd/theory/evolutionary_distances.md @@ -205,7 +205,7 @@ single-copy at that locus. ## Context, detectability, and a 3-way ordinal distance per pair -Empirical follow-up to the pseudo-alignment idea above: `obikmer distance +Empirical follow-up to the pseudo-alignment idea above: `obikmer phylo --snp` was run on a real 20-genome benchmark index and the resulting FASTA fed to `raxml-ng`. Two problems surfaced, both traced back to conflating distinct notions under one symbol. @@ -552,7 +552,7 @@ informative across domains with this character type. Exploratory only — run entirely outside the repo (`/tmp/tnt_run`, TNT installed locally under `TNT/`), no new Rust code. Kept here as the record of what was learned. -**Pipeline.** `obikmer distance --snp` emits one IUPAC-coded pseudo-alignment +**Pipeline.** `obikmer phylo --snp` emits one IUPAC-coded pseudo-alignment row per genome (`snp_pseudo_alignment`, one column per family with `family_size() >= 2`). A small external Python script decodes IUPAC back to the 16-state bitmask, applies the set-edit-distance formula above, and @@ -630,7 +630,7 @@ it, no external script needed. `∅` is an ordinary 16th state throughout (never `-`), specifically to avoid gap-semantics confusion in downstream tools — see "A concrete Sankoff cost matrix" above for why. -**TNT (`--tnt`).** `write_sankoff_tnt` (`obikmer/src/cmd/distance.rs`) +**TNT (`--tnt`).** `write_sankoff_tnt` (`obikmer/src/cmd/phylo/mod.rs`) recodes to TNT's own `0-9A-F` xread alphabet (its default reader rejects the wider IUPAC set otherwise), scales and rounds costs to integers (`smatrix`/`cost` reject decimals), then re-runs integer Floyd-Warshall on @@ -902,7 +902,7 @@ sections below, which implement and then substantially revise this. ### `R`/`π` implemented; `--exclude-genome` added; rogue-taxon test negative (2026-08-12) -`write_iqtree` (`obikmer/src/cmd/distance/iqtree.rs`) implements exactly +`write_iqtree` (`obikmer/src/cmd/phylo/iqtree.rs`) implements exactly the `R = exp(-cost)` / empirical-`π` design above: writes `_iqtree.model` (lower-triangular `R`, PAML order, then `π`) and `_iqtree.fasta` (alignment recoded to the compact `0..k-1` @@ -949,7 +949,7 @@ tell: with `n_eligible=1`, the ratio can only be exactly `0` or `1`, so possibly ascertainment bias — see below — or possibly a few genuinely ultra-conserved loci; not resolved). -**`--exclude-genome LABEL`** (repeatable, `obikmer distance`) added for +**`--exclude-genome LABEL`** (repeatable, `obikmer phylo`) added for exactly this kind of test: zeroes the excluded genome's row/column in `RawSnpDistanceOutput` after `raw_snp_distance` runs (a pair with zero counts is already skipped by `calibrate_p_hat`/`base_pair_tally` — no @@ -1165,7 +1165,7 @@ silently wrong data for TNT/PhyG, and a hard failure for IQ-TREE's `+ASC` (verified: excluding 2 taxa on the benchmark left 116,351 such columns — matches the manual `+ASC` failures hit earlier in this same investigation, before `--exclude-genome` existed). Fixed in `drop_excluded` -(`obikmer/src/cmd/distance/mod.rs`): after dropping excluded rows, +(`obikmer/src/cmd/phylo/mod.rs`): after dropping excluded rows, re-scan each column among the *surviving* sequences and drop any that are now constant. Verified: 908,723 → 792,372 sites after excluding 2 taxa, zero monomorphic columns remain, `π` recomputed from the corrected @@ -1582,7 +1582,7 @@ index, not of any single genome — it cannot be computed correctly at mono-genome build time (a family may gain siblings, or its minorant may change, once more genomes are merged in later). Computing it eagerly at every `merge` would also waste work on intermediate merged states nobody -ever queries. Instead: compute it lazily, on first `distance` call against a +ever queries. Instead: compute it lazily, on first `phylo` call against a given index, and persist the result alongside that index for subsequent calls — the same lazy-derived-cache pattern `PersistentBitMatrix` already uses for `Columnar` -> `Packed`. This requires no explicit invalidation for @@ -1745,7 +1745,7 @@ above) is observed anywhere in the index — fixes both: own 3 lookups independently," not a strict improvement to adopt by default. 4. **Trigger and caching** (`obikindex::KmerIndex`/`distance.rs`): compute - lazily on first `distance` call for an SNP-family metric against a given + lazily on first `phylo` call for an SNP-family metric against a given index; check for an existing annex file first (mirrors `PersistentBitMatrix::open()`'s auto-detect-and-fall-back, `bitmatrix.rs:264-287`); if absent, run step 3 and persist; if present, @@ -1765,7 +1765,7 @@ above) is observed anywhere in the index — fixes both: slots are never looked up cross-partition during the sweep. Cost: `3 * N_distinct` existence-only lookups, computed once per index -state and amortised over every subsequent `distance` call that reuses the +state and amortised over every subsequent `phylo` call that reuses the cached annex — cheaper per-lookup than the sweep itself (hit/miss only, no column fetch). @@ -1795,11 +1795,11 @@ intermediate). Two options, to decide: Keeps one CLI surface (`--metric jukes-cantor`), at the cost of a branch in `distance()` that ignores the `LayeredStore` it built. - **(b)** A dedicated pathway (`KmerIndex::snp_distance`) and a distinct CLI - entry, if mixing a cross-partition sweep into the partition-local `distance` + entry, if mixing a cross-partition sweep into the partition-local `phylo` command is judged architecturally muddy. Recommendation: (a) for user ergonomics (all pairwise distances under -`distance`, all feeding NJ/UPGMA/`--shared-kmers` unchanged), but compute the +`phylo`, all feeding NJ/UPGMA/`--shared-kmers` unchanged), but compute the sweep lazily only when an SNP-family metric is requested, so the existing metrics keep their partition-local fast path untouched. diff --git a/mkdocs-user.yml b/mkdocs-user.yml index 1c9d1842..40ca9021 100644 --- a/mkdocs-user.yml +++ b/mkdocs-user.yml @@ -45,7 +45,7 @@ nav: - query: usage/query.md - dump: usage/dump.md - annotate: usage/annotate.md - - distance: usage/distance.md + - phylo: usage/phylo.md - unitig: usage/unitig.md - estimate: usage/estimate.md - reindex: usage/reindex.md diff --git a/src/obikmer/src/cmd/mod.rs b/src/obikmer/src/cmd/mod.rs index eba674ba..619edea6 100644 --- a/src/obikmer/src/cmd/mod.rs +++ b/src/obikmer/src/cmd/mod.rs @@ -4,7 +4,7 @@ pub mod pack; pub(crate) mod predicate; pub mod select; pub mod utils; -pub mod distance; +pub mod phylo; pub mod dump; pub mod estimate; pub mod index; diff --git a/src/obikmer/src/cmd/distance/iqtree.rs b/src/obikmer/src/cmd/phylo/iqtree.rs similarity index 100% rename from src/obikmer/src/cmd/distance/iqtree.rs rename to src/obikmer/src/cmd/phylo/iqtree.rs diff --git a/src/obikmer/src/cmd/distance/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs similarity index 99% rename from src/obikmer/src/cmd/distance/mod.rs rename to src/obikmer/src/cmd/phylo/mod.rs index 3d54cb44..a25f7a39 100644 --- a/src/obikmer/src/cmd/distance/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -55,7 +55,7 @@ impl From for DistanceMetric { } #[derive(Args)] -pub struct DistanceArgs { +pub struct PhyloArgs { /// Index directory pub index: PathBuf, @@ -220,7 +220,7 @@ pub struct DistanceArgs { pub output: Option, } -pub fn run(args: DistanceArgs) { +pub fn run(args: PhyloArgs) { let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| { eprintln!("error opening index: {e}"); std::process::exit(1); diff --git a/src/obikmer/src/cmd/distance/phyg.rs b/src/obikmer/src/cmd/phylo/phyg.rs similarity index 100% rename from src/obikmer/src/cmd/distance/phyg.rs rename to src/obikmer/src/cmd/phylo/phyg.rs diff --git a/src/obikmer/src/cmd/distance/sankoff.rs b/src/obikmer/src/cmd/phylo/sankoff.rs similarity index 100% rename from src/obikmer/src/cmd/distance/sankoff.rs rename to src/obikmer/src/cmd/phylo/sankoff.rs diff --git a/src/obikmer/src/cmd/distance/tnt.rs b/src/obikmer/src/cmd/phylo/tnt.rs similarity index 100% rename from src/obikmer/src/cmd/distance/tnt.rs rename to src/obikmer/src/cmd/phylo/tnt.rs diff --git a/src/obikmer/src/main.rs b/src/obikmer/src/main.rs index cb701e6f..73e49b01 100644 --- a/src/obikmer/src/main.rs +++ b/src/obikmer/src/main.rs @@ -30,8 +30,9 @@ enum Commands { Dump(cmd::dump::DumpArgs), /// Add or update genome metadata from a CSV file; or dump metadata as CSV Annotate(cmd::annotate::AnnotateArgs), - /// Compute pairwise distance matrix between genomes; optionally build NJ/UPGMA trees - Distance(cmd::distance::DistanceArgs), + /// Compute pairwise evolutionary-distance proxies between genomes (metric matrix, NJ/UPGMA + /// trees, SNP/Sankoff calibration, TNT/PhyG/IQ-TREE exports) + Phylo(cmd::phylo::PhyloArgs), /// Dump unitigs from a built index to stdout (debug) Unitig(cmd::unitig::UnitigArgs), /// Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing @@ -71,7 +72,7 @@ fn main() { Commands::Select(args) => cmd::select::run(args), Commands::Query(args) => cmd::query::run(args), Commands::Annotate(args) => cmd::annotate::run(args), - Commands::Distance(args) => cmd::distance::run(args), + Commands::Phylo(args) => cmd::phylo::run(args), Commands::Unitig(args) => cmd::unitig::run(args), Commands::Estimate(args) => cmd::estimate::run(args), Commands::Reindex(args) => cmd::reindex::run(args),