rename distance subcommand to phylo
Rename the distance CLI subcommand to phylo across the codebase, documentation, and build configurations. Relocate source files from cmd/distance/ to a dedicated cmd/phylo/ module, update all internal routing references, and adjust benchmark scripts and Makefile targets to reflect the new command name.
This commit is contained in:
Eric Coissac
@@ -105,7 +105,7 @@ bump-version:
|
||||
|
||||
.PHONY: release
|
||||
release: bump-version
|
||||
@jj auto-describe
|
||||
@jj auto-doc
|
||||
@jj git push --change @
|
||||
@new_version=$$(grep '^version = ' $(CARGO_TOML) | head -n 1 | sed 's/version = "\(.*\)"/\1/'); \
|
||||
git_hash=$$(jj log -r @ --no-graph -T 'commit_id'); \
|
||||
|
||||
@@ -66,9 +66,11 @@ Non-ACGT characters act as hard breaks between k-mer segments in all formats.
|
||||
Annotates each sequence with per-genome k-mer match counts
|
||||
and optional per-position coverage vectors (--detail).
|
||||
Parallel over sequence chunks.
|
||||
distance Compute a pairwise Bray-Curtis or Jaccard distance matrix
|
||||
between all indexed genomes.
|
||||
Optionally outputs a Newick NJ or UPGMA tree.
|
||||
phylo Compute pairwise evolutionary-distance proxies between all
|
||||
indexed genomes (Bray-Curtis, Jaccard, etc.), optionally
|
||||
a Newick NJ/UPGMA tree, and optionally a central-position
|
||||
SNP/Sankoff calibration with exports for external
|
||||
phylogenetic tools (TNT, PhyG, IQ-TREE).
|
||||
annotate Add or update genome metadata (taxonomy, etc.) from a CSV
|
||||
file; or dump the current metadata as CSV.
|
||||
estimate Dry-run: resolve and print approximate-index parameters
|
||||
@@ -106,7 +108,7 @@ obikmer reindex --approx -z 5 --evidence-bits 8 index/
|
||||
obikmer query index/ reads.fq.gz > annotated.fa
|
||||
|
||||
# Pairwise distances
|
||||
obikmer distance index/ > distances.tsv
|
||||
obikmer phylo index/ > distances.tsv
|
||||
```
|
||||
|
||||
## Parameter constraints
|
||||
|
||||
@@ -20,7 +20,7 @@ An index directory is organized as `KmerIndex → partitions → layers`, with a
|
||||
|
||||
## Parallel execution and NUMA awareness
|
||||
|
||||
Partition-level work (index construction, `merge`, `filter`, `reindex`, `select`, `distance`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc:
|
||||
Partition-level work (index construction, `merge`, `filter`, `reindex`, `select`, `phylo`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc:
|
||||
|
||||
- On a multi-socket / multi-NUMA-node machine, one thread pool is pinned per NUMA node, and each partition is processed entirely by threads pinned to one node — keeping the memory a partition touches local to that node's DRAM. This matters because touching kmer data across NUMA nodes without pinning can degrade throughput by an order of magnitude or more on large multi-socket machines.
|
||||
- On a single-socket machine, Apple Silicon, or if hwloc cannot report NUMA topology, all cores are treated as one node with no pinning and negligible overhead — this is the default behavior on macOS.
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ All functionality is exposed through a single binary, `obikmer`, organized as su
|
||||
| [`query`](usage/query.md) | Query an index with sequences and annotate matches |
|
||||
| [`dump`](usage/dump.md) | Dump indexed kmers as CSV |
|
||||
| [`annotate`](usage/annotate.md) | Add, update, or dump genome metadata |
|
||||
| [`distance`](usage/distance.md) | Compute pairwise distance matrices and phylogenetic exports |
|
||||
| [`phylo`](usage/phylo.md) | Compute pairwise evolutionary-distance proxies, trees, and phylogenetic exports |
|
||||
| [`unitig`](usage/unitig.md) | Dump the unitigs of an index as FASTA |
|
||||
| [`estimate`](usage/estimate.md) | Estimate approximate-index parameters before indexing |
|
||||
| [`reindex`](usage/reindex.md) | Convert an index's evidence representation (exact ↔ approximate) |
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# distance
|
||||
# phylo
|
||||
|
||||
Compute pairwise distance matrices between the genomes stored in an index, optionally build trees (NJ/UPGMA), and optionally derive a central-position SNP model with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE).
|
||||
Compute pairwise evolutionary-distance proxies between the genomes stored in an index — a plain distance matrix, optionally trees (NJ/UPGMA), and optionally a central-position SNP model with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE).
|
||||
|
||||
```bash
|
||||
obikmer distance INDEX [OPTIONS]
|
||||
obikmer phylo INDEX [OPTIONS]
|
||||
```
|
||||
|
||||
## Arguments
|
||||
+11
-11
@@ -89,50 +89,50 @@ $(REF_DIST_CSVS) &: $(REF_NPZS) build_reference_dist.py
|
||||
|
||||
reference_dist: $(REF_DIST_CSVS)
|
||||
|
||||
# ── obikmer distance (presence index) ────────────────────────────────────────
|
||||
# ── obikmer phylo (presence index) ──────────────────────────────────────────
|
||||
|
||||
$(OBIKMER_PRESENCE_DIST) &: global_index_presence/index.done $(BINARY)
|
||||
mkdir -p obikmer_dist/presence
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/presence/jaccard \
|
||||
--metric jaccard --shared-kmers --nj \
|
||||
global_index_presence
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/presence/hamming \
|
||||
--metric hamming --nj \
|
||||
global_index_presence
|
||||
|
||||
obikmer_dist_presence: $(OBIKMER_PRESENCE_DIST)
|
||||
|
||||
# ── obikmer distance (count index) ───────────────────────────────────────────
|
||||
# ── obikmer phylo (count index) ─────────────────────────────────────────────
|
||||
|
||||
$(OBIKMER_COUNT_DIST) &: global_index_count/index.done $(BINARY)
|
||||
mkdir -p obikmer_dist/count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/jaccard \
|
||||
--metric jaccard --shared-kmers --nj \
|
||||
global_index_count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/bray_curtis \
|
||||
--metric bray-curtis --nj \
|
||||
global_index_count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/relfreq_bray_curtis \
|
||||
--metric relfreq-bray-curtis --nj \
|
||||
global_index_count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/euclidean \
|
||||
--metric euclidean --nj \
|
||||
global_index_count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/relfreq_euclidean \
|
||||
--metric relfreq-euclidean --nj \
|
||||
global_index_count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/hellinger \
|
||||
--metric hellinger --nj \
|
||||
global_index_count
|
||||
$(BINARY) distance \
|
||||
$(BINARY) phylo \
|
||||
--output obikmer_dist/count/hellinger_euclidean \
|
||||
--metric hellinger-euclidean --nj \
|
||||
global_index_count
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
"""Compute reference pairwise distance matrices from per-specimen .npz kmer indexes.
|
||||
|
||||
Reads all .npz files in reference_index/ (each containing sorted uint64 `kmers`
|
||||
and uint32 `counts`), computes all distance metrics supported by `obikmer distance`,
|
||||
and uint32 `counts`), computes all distance metrics supported by `obikmer phylo`,
|
||||
and writes one CSV per metric to reference_dist/.
|
||||
|
||||
Output CSV format matches `obikmer distance --output`:
|
||||
Output CSV format matches `obikmer phylo --output`:
|
||||
- first row: "genome", then specimen names
|
||||
- subsequent rows: specimen name, then float or int values
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare all reference distance matrices against obikmer distance outputs.
|
||||
"""Compare all reference distance matrices against obikmer phylo outputs.
|
||||
|
||||
Reads from:
|
||||
reference_dist/ — ground-truth matrices computed by build_reference_dist.py
|
||||
obikmer_dist/ — matrices produced by `obikmer distance`
|
||||
obikmer_dist/ — matrices produced by `obikmer phylo`
|
||||
|
||||
Handles label reordering: both matrices are sorted by genome label before
|
||||
element-wise comparison, so column/row order differences are irrelevant.
|
||||
|
||||
@@ -289,14 +289,14 @@ obikmer dump myindex --head 100
|
||||
obikmer dump myindex --head 20 --ingroup "species=Betula_nana" --min-count 1
|
||||
```
|
||||
|
||||
### `distance --presence-threshold N`
|
||||
### `phylo --presence-threshold N`
|
||||
|
||||
When computing Jaccard distance on a **count index**, a k-mer is considered present in a genome if its count is ≥ N (default 1).
|
||||
This option is independent of the `--presence-threshold` used in filtering.
|
||||
|
||||
```sh
|
||||
# Jaccard treating kmers with count ≥ 2 as present
|
||||
obikmer distance myindex --metric jaccard --presence-threshold 2
|
||||
obikmer phylo myindex --metric jaccard --presence-threshold 2
|
||||
```
|
||||
|
||||
This parameter has no effect on presence/absence indexes (where values are already 0/1) or on metrics other than Jaccard.
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
| `query` | Query an index with sequences and annotate matches |
|
||||
| `dump` | Dump all indexed k-mers as CSV (kmer + per-genome counts or presence); supports the shared [kmer filtering](implementation/filtering.md) system; `--head N` limits output to the first N k-mers |
|
||||
| `annotate` | Add or update genome metadata from a CSV file; or dump metadata as CSV |
|
||||
| `distance` | Compute pairwise distance matrix between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1) |
|
||||
| `phylo` | Compute pairwise evolutionary-distance proxies between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1); optionally a central-position SNP/Sankoff calibration with exports for TNT/PhyG/IQ-TREE (see [evolutionary distances](theory/evolutionary_distances.md)) |
|
||||
| `unitig` | Build a global de Bruijn graph across all partitions and enumerate its unitigs as FASTA; supports the shared [kmer filtering](implementation/filtering.md) system |
|
||||
| `select` | Project and/or aggregate genome columns into a new or in-place index; the column-axis counterpart of `filter` (see [select](implementation/select.md)) |
|
||||
| `estimate` | Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing |
|
||||
|
||||
@@ -205,7 +205,7 @@ single-copy at that locus.
|
||||
|
||||
## Context, detectability, and a 3-way ordinal distance per pair
|
||||
|
||||
Empirical follow-up to the pseudo-alignment idea above: `obikmer distance
|
||||
Empirical follow-up to the pseudo-alignment idea above: `obikmer phylo
|
||||
--snp` was run on a real 20-genome benchmark index and the resulting FASTA
|
||||
fed to `raxml-ng`. Two problems surfaced, both traced back to conflating
|
||||
distinct notions under one symbol.
|
||||
@@ -552,7 +552,7 @@ informative across domains with this character type. Exploratory only —
|
||||
run entirely outside the repo (`/tmp/tnt_run`, TNT installed locally under
|
||||
`TNT/`), no new Rust code. Kept here as the record of what was learned.
|
||||
|
||||
**Pipeline.** `obikmer distance --snp` emits one IUPAC-coded pseudo-alignment
|
||||
**Pipeline.** `obikmer phylo --snp` emits one IUPAC-coded pseudo-alignment
|
||||
row per genome (`snp_pseudo_alignment`, one column per family with
|
||||
`family_size() >= 2`). A small external Python script decodes IUPAC back to
|
||||
the 16-state bitmask, applies the set-edit-distance formula above, and
|
||||
@@ -630,7 +630,7 @@ it, no external script needed. `∅` is an ordinary 16th state throughout
|
||||
(never `-`), specifically to avoid gap-semantics confusion in downstream
|
||||
tools — see "A concrete Sankoff cost matrix" above for why.
|
||||
|
||||
**TNT (`--tnt`).** `write_sankoff_tnt` (`obikmer/src/cmd/distance.rs`)
|
||||
**TNT (`--tnt`).** `write_sankoff_tnt` (`obikmer/src/cmd/phylo/mod.rs`)
|
||||
recodes to TNT's own `0-9A-F` xread alphabet (its default reader rejects
|
||||
the wider IUPAC set otherwise), scales and rounds costs to integers
|
||||
(`smatrix`/`cost` reject decimals), then re-runs integer Floyd-Warshall on
|
||||
@@ -902,7 +902,7 @@ sections below, which implement and then substantially revise this.
|
||||
|
||||
### `R`/`π` implemented; `--exclude-genome` added; rogue-taxon test negative (2026-08-12)
|
||||
|
||||
`write_iqtree` (`obikmer/src/cmd/distance/iqtree.rs`) implements exactly
|
||||
`write_iqtree` (`obikmer/src/cmd/phylo/iqtree.rs`) implements exactly
|
||||
the `R = exp(-cost)` / empirical-`π` design above: writes
|
||||
`<prefix>_iqtree.model` (lower-triangular `R`, PAML order, then `π`) and
|
||||
`<prefix>_iqtree.fasta` (alignment recoded to the compact `0..k-1`
|
||||
@@ -949,7 +949,7 @@ tell: with `n_eligible=1`, the ratio can only be exactly `0` or `1`, so
|
||||
possibly ascertainment bias — see below — or possibly a few genuinely
|
||||
ultra-conserved loci; not resolved).
|
||||
|
||||
**`--exclude-genome LABEL`** (repeatable, `obikmer distance`) added for
|
||||
**`--exclude-genome LABEL`** (repeatable, `obikmer phylo`) added for
|
||||
exactly this kind of test: zeroes the excluded genome's row/column in
|
||||
`RawSnpDistanceOutput` after `raw_snp_distance` runs (a pair with zero
|
||||
counts is already skipped by `calibrate_p_hat`/`base_pair_tally` — no
|
||||
@@ -1165,7 +1165,7 @@ silently wrong data for TNT/PhyG, and a hard failure for IQ-TREE's `+ASC`
|
||||
(verified: excluding 2 taxa on the benchmark left 116,351 such columns —
|
||||
matches the manual `+ASC` failures hit earlier in this same investigation,
|
||||
before `--exclude-genome` existed). Fixed in `drop_excluded`
|
||||
(`obikmer/src/cmd/distance/mod.rs`): after dropping excluded rows,
|
||||
(`obikmer/src/cmd/phylo/mod.rs`): after dropping excluded rows,
|
||||
re-scan each column among the *surviving* sequences and drop any that are
|
||||
now constant. Verified: 908,723 → 792,372 sites after excluding 2 taxa,
|
||||
zero monomorphic columns remain, `π` recomputed from the corrected
|
||||
@@ -1582,7 +1582,7 @@ index, not of any single genome — it cannot be computed correctly at
|
||||
mono-genome build time (a family may gain siblings, or its minorant may
|
||||
change, once more genomes are merged in later). Computing it eagerly at
|
||||
every `merge` would also waste work on intermediate merged states nobody
|
||||
ever queries. Instead: compute it lazily, on first `distance` call against a
|
||||
ever queries. Instead: compute it lazily, on first `phylo` call against a
|
||||
given index, and persist the result alongside that index for subsequent
|
||||
calls — the same lazy-derived-cache pattern `PersistentBitMatrix` already
|
||||
uses for `Columnar` -> `Packed`. This requires no explicit invalidation for
|
||||
@@ -1745,7 +1745,7 @@ above) is observed anywhere in the index — fixes both:
|
||||
own 3 lookups independently," not a strict improvement to adopt by
|
||||
default.
|
||||
4. **Trigger and caching** (`obikindex::KmerIndex`/`distance.rs`): compute
|
||||
lazily on first `distance` call for an SNP-family metric against a given
|
||||
lazily on first `phylo` call for an SNP-family metric against a given
|
||||
index; check for an existing annex file first (mirrors
|
||||
`PersistentBitMatrix::open()`'s auto-detect-and-fall-back,
|
||||
`bitmatrix.rs:264-287`); if absent, run step 3 and persist; if present,
|
||||
@@ -1765,7 +1765,7 @@ above) is observed anywhere in the index — fixes both:
|
||||
slots are never looked up cross-partition during the sweep.
|
||||
|
||||
Cost: `3 * N_distinct` existence-only lookups, computed once per index
|
||||
state and amortised over every subsequent `distance` call that reuses the
|
||||
state and amortised over every subsequent `phylo` call that reuses the
|
||||
cached annex — cheaper per-lookup than the sweep itself (hit/miss only, no
|
||||
column fetch).
|
||||
|
||||
@@ -1795,11 +1795,11 @@ intermediate). Two options, to decide:
|
||||
Keeps one CLI surface (`--metric jukes-cantor`), at the cost of a branch in
|
||||
`distance()` that ignores the `LayeredStore` it built.
|
||||
- **(b)** A dedicated pathway (`KmerIndex::snp_distance`) and a distinct CLI
|
||||
entry, if mixing a cross-partition sweep into the partition-local `distance`
|
||||
entry, if mixing a cross-partition sweep into the partition-local `phylo`
|
||||
command is judged architecturally muddy.
|
||||
|
||||
Recommendation: (a) for user ergonomics (all pairwise distances under
|
||||
`distance`, all feeding NJ/UPGMA/`--shared-kmers` unchanged), but compute the
|
||||
`phylo`, all feeding NJ/UPGMA/`--shared-kmers` unchanged), but compute the
|
||||
sweep lazily only when an SNP-family metric is requested, so the existing
|
||||
metrics keep their partition-local fast path untouched.
|
||||
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ nav:
|
||||
- query: usage/query.md
|
||||
- dump: usage/dump.md
|
||||
- annotate: usage/annotate.md
|
||||
- distance: usage/distance.md
|
||||
- phylo: usage/phylo.md
|
||||
- unitig: usage/unitig.md
|
||||
- estimate: usage/estimate.md
|
||||
- reindex: usage/reindex.md
|
||||
|
||||
@@ -4,7 +4,7 @@ pub mod pack;
|
||||
pub(crate) mod predicate;
|
||||
pub mod select;
|
||||
pub mod utils;
|
||||
pub mod distance;
|
||||
pub mod phylo;
|
||||
pub mod dump;
|
||||
pub mod estimate;
|
||||
pub mod index;
|
||||
|
||||
@@ -55,7 +55,7 @@ impl From<MetricArg> for DistanceMetric {
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
pub struct DistanceArgs {
|
||||
pub struct PhyloArgs {
|
||||
/// Index directory
|
||||
pub index: PathBuf,
|
||||
|
||||
@@ -220,7 +220,7 @@ pub struct DistanceArgs {
|
||||
pub output: Option<PathBuf>,
|
||||
}
|
||||
|
||||
pub fn run(args: DistanceArgs) {
|
||||
pub fn run(args: PhyloArgs) {
|
||||
let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
|
||||
eprintln!("error opening index: {e}");
|
||||
std::process::exit(1);
|
||||
@@ -30,8 +30,9 @@ enum Commands {
|
||||
Dump(cmd::dump::DumpArgs),
|
||||
/// Add or update genome metadata from a CSV file; or dump metadata as CSV
|
||||
Annotate(cmd::annotate::AnnotateArgs),
|
||||
/// Compute pairwise distance matrix between genomes; optionally build NJ/UPGMA trees
|
||||
Distance(cmd::distance::DistanceArgs),
|
||||
/// Compute pairwise evolutionary-distance proxies between genomes (metric matrix, NJ/UPGMA
|
||||
/// trees, SNP/Sankoff calibration, TNT/PhyG/IQ-TREE exports)
|
||||
Phylo(cmd::phylo::PhyloArgs),
|
||||
/// Dump unitigs from a built index to stdout (debug)
|
||||
Unitig(cmd::unitig::UnitigArgs),
|
||||
/// Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing
|
||||
@@ -71,7 +72,7 @@ fn main() {
|
||||
Commands::Select(args) => cmd::select::run(args),
|
||||
Commands::Query(args) => cmd::query::run(args),
|
||||
Commands::Annotate(args) => cmd::annotate::run(args),
|
||||
Commands::Distance(args) => cmd::distance::run(args),
|
||||
Commands::Phylo(args) => cmd::phylo::run(args),
|
||||
Commands::Unitig(args) => cmd::unitig::run(args),
|
||||
Commands::Estimate(args) => cmd::estimate::run(args),
|
||||
Commands::Reindex(args) => cmd::reindex::run(args),
|
||||
|
||||
Reference in New Issue
Block a user