feat(distance): implement native Sankoff calibration and backends
Replaces external Python glue with native Rust modules for Sankoff model calibration, exporting calibrated cost matrices, FASTA alignments, and YAML parameters. Adds dedicated writers for TNT and PhyG that apply integer scaling and Floyd-Warshall metric closure to enforce triangle inequality. Integrates these exporters into the distance command pipeline to streamline downstream tree inference workflows, while updating theory documentation to reflect IQ-TREE integration and state renumbering improvements.
This commit is contained in:
Eric Coissac
@@ -1,3 +1,7 @@
|
||||
mod phyg;
|
||||
mod sankoff;
|
||||
mod tnt;
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -5,13 +9,17 @@ use clap::Args;
|
||||
use kodama::{Method, linkage};
|
||||
use obifastwrite::{JsonVal, write_record};
|
||||
use obikindex::{
|
||||
BasePairTally, DistanceMetric, KmerIndex, PHatEstimate, RawSnpDistanceOutput, SankoffWeights,
|
||||
DistanceMetric, KmerIndex, RawSnpDistanceOutput, SankoffWeights,
|
||||
SiblingAnnexStats, SnpAlignment, build_cost_matrix, c_ctx_from_p_hat, calibrate_p_hat,
|
||||
mean_substitution_cost, substitution_costs_from_tally,
|
||||
};
|
||||
use speedytree::{DistanceMatrix, Hybrid, NeighborJoiningSolver, to_newick};
|
||||
use tracing::info;
|
||||
|
||||
use phyg::write_sankoff_phyg;
|
||||
use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
|
||||
use tnt::write_sankoff_tnt;
|
||||
|
||||
#[derive(clap::ValueEnum, Clone, Copy, Debug)]
|
||||
pub enum MetricArg {
|
||||
Jaccard,
|
||||
@@ -468,387 +476,6 @@ fn write_snp_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<
|
||||
if n_sites == 1 { "" } else { "s" });
|
||||
}
|
||||
|
||||
// ── Sankoff pseudo-alignment → FASTA ────────────────────────────────────────
|
||||
//
|
||||
// Same data as `--snp`'s pseudo-alignment (`SnpAlignment`/
|
||||
// `snp_pseudo_alignment`), re-coded so its symbols match the accompanying
|
||||
// `--sankoff-matrix` output exactly: `0` for the empty/absent state instead
|
||||
// of `-`, which TNT/PhyG would otherwise read as their own gap character
|
||||
// rather than our "family absent" state.
|
||||
|
||||
fn write_sankoff_alignment_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff.fasta", p.display()))
|
||||
.unwrap_or_else(|| "sankoff.fasta".into());
|
||||
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
|
||||
for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
|
||||
let recoded: Vec<u8> = seq.iter().map(|&b| if b == b'-' { b'0' } else { b }).collect();
|
||||
write_record(&recoded, label, &[("n_sites", JsonVal::Num(n_sites as u64))], &mut f).unwrap_or_else(|e| {
|
||||
eprintln!("error writing {path}: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
}
|
||||
info!("Sankoff pseudo-alignment → {path} ({n_sites} site{})",
|
||||
if n_sites == 1 { "" } else { "s" });
|
||||
}
|
||||
|
||||
// ── Sankoff cost matrix → CSV ────────────────────────────────────────────────
|
||||
//
|
||||
// 16 states indexed by bitmask (bit 0=A, 1=C, 2=G, 3=T; state 0 is `∅`),
|
||||
// matching the convention already used for `--snp`'s IUPAC-coded output and
|
||||
// for the external TNT/PhyG scripts this feeds. Calibration report (p_hat,
|
||||
// its variance, how many pairs/loci went into it, the resulting c_ctx) goes
|
||||
// to the log, not the CSV, since it's a run-level fact, not per-cell data.
|
||||
|
||||
// IUPAC ambiguity code per state (same mapping as `siblings::iupac_code`,
|
||||
// already used for `--snp`'s pseudo-alignment — a biologist reads "R" as
|
||||
// "A or G" without needing this file's convention explained), with `0`
|
||||
// standing in for the empty state (`-` would collide with TNT/PhyG's own
|
||||
// gap/range syntax). Bit order: 0=A, 1=C, 2=G, 3=T. This project's
|
||||
// canonical alphabet — `write_sankoff_tnt` below recodes it to TNT's own
|
||||
// default alphabet at the adapter boundary, rather than using it here.
|
||||
const STATE_SYMBOL: [char; 16] = [
|
||||
'0', 'A', 'C', 'M', 'G', 'R', 'S', 'V', 'T', 'W', 'Y', 'H', 'K', 'D', 'B', 'N',
|
||||
];
|
||||
|
||||
fn write_sankoff_matrix_csv(
|
||||
matrix: &[[f64; 16]; 16],
|
||||
estimate: &PHatEstimate,
|
||||
weights: &SankoffWeights,
|
||||
output: &Option<PathBuf>,
|
||||
) {
|
||||
info!(
|
||||
p_hat = format_args!("{:.6}", estimate.p_hat),
|
||||
variance = format_args!("{:.3e}", estimate.variance),
|
||||
n_pairs_included = estimate.n_pairs_included,
|
||||
n_loci_total = estimate.n_loci_total,
|
||||
c_ctx = format_args!("{:.4}", weights.c_ctx),
|
||||
"Sankoff matrix calibration"
|
||||
);
|
||||
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff_matrix.csv", p.display()))
|
||||
.unwrap_or_else(|| "sankoff_matrix.csv".into());
|
||||
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
write!(f, "state").unwrap();
|
||||
for sym in STATE_SYMBOL { write!(f, ",{sym}").unwrap(); }
|
||||
writeln!(f).unwrap();
|
||||
for (s, row) in matrix.iter().enumerate() {
|
||||
write!(f, "{}", STATE_SYMBOL[s]).unwrap();
|
||||
for cost in row { write!(f, ",{cost:.4}").unwrap(); }
|
||||
writeln!(f).unwrap();
|
||||
}
|
||||
info!("Sankoff cost matrix → {path}");
|
||||
}
|
||||
|
||||
// ── Sankoff calibration parameters → YAML report ────────────────────────────
|
||||
//
|
||||
// Everything `--sankoff` estimates from real data, in one durable,
|
||||
// machine-readable file: `p_hat` and its variance (with how many pairs/loci
|
||||
// went into it), the derived `c_ctx`, and the base-pair substitution tally
|
||||
// (raw counts, not just the derived costs) — kept for the same reason raw
|
||||
// counts are kept anywhere else in this project: costs are a modelling
|
||||
// choice built *from* the counts, and reproducing/re-deriving them later
|
||||
// needs the counts, not just their current derived value. Structured (YAML,
|
||||
// not an ad hoc key=value text file) so R/Python/etc. can load it directly
|
||||
// rather than re-parsing free text.
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct SankoffSubstitution {
|
||||
pair: String,
|
||||
count: u64,
|
||||
cost: f64,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct SankoffParamsReport {
|
||||
ratio_ceiling: f64,
|
||||
flank_length_m: usize,
|
||||
p_hat: f64,
|
||||
p_hat_variance: f64,
|
||||
n_pairs_included: usize,
|
||||
n_loci_total: u64,
|
||||
/// Weighted-average substitution cost — see `c_ctx_from_p_hat`'s docs:
|
||||
/// this is what turns the raw expected mutation *count* behind `c_ctx`
|
||||
/// into an actual cost (not every mutation is worth a flat `1`).
|
||||
mean_sub_cost: f64,
|
||||
c_ctx: f64,
|
||||
substitutions: Vec<SankoffSubstitution>,
|
||||
}
|
||||
|
||||
fn write_sankoff_params(
|
||||
estimate: &PHatEstimate,
|
||||
tally: &BasePairTally,
|
||||
weights: &SankoffWeights,
|
||||
ratio_ceiling: f64,
|
||||
m: usize,
|
||||
mean_sub_cost: f64,
|
||||
output: &Option<PathBuf>,
|
||||
) {
|
||||
const BASE_LETTER: [char; 4] = ['A', 'C', 'G', 'T'];
|
||||
|
||||
let mut substitutions = Vec::with_capacity(6);
|
||||
for a in 0..4 {
|
||||
for b in (a + 1)..4 {
|
||||
substitutions.push(SankoffSubstitution {
|
||||
pair: format!("{}/{}", BASE_LETTER[a], BASE_LETTER[b]),
|
||||
count: tally.counts[a][b],
|
||||
cost: weights.sub_cost[a][b],
|
||||
});
|
||||
}
|
||||
}
|
||||
let report = SankoffParamsReport {
|
||||
ratio_ceiling,
|
||||
flank_length_m: m,
|
||||
p_hat: estimate.p_hat,
|
||||
p_hat_variance: estimate.variance,
|
||||
n_pairs_included: estimate.n_pairs_included,
|
||||
n_loci_total: estimate.n_loci_total,
|
||||
mean_sub_cost,
|
||||
c_ctx: weights.c_ctx,
|
||||
substitutions,
|
||||
};
|
||||
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff_params.yaml", p.display()))
|
||||
.unwrap_or_else(|| "sankoff_params.yaml".into());
|
||||
let f = std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
serde_yaml::to_writer(f, &report).unwrap_or_else(|e| {
|
||||
eprintln!("error writing {path}: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
info!("Sankoff calibration parameters → {path}");
|
||||
}
|
||||
|
||||
// ── Sankoff cost matrix + alignment → ready-to-run TNT script ──────────────
|
||||
//
|
||||
// TNT's *default* xread reader only accepts its own 0-9A-F alphabet (see
|
||||
// its manual: "up to 16 states are allowed by xread, using symbols 0-9 ...
|
||||
// and A-F") — the wider IUPAC set `STATE_SYMBOL` uses is rejected as an
|
||||
// "alien symbol" unless `nstates dna` is set, which imposes TNT's own fixed
|
||||
// DNA encoding instead, incompatible with a custom smatrix. And TNT's
|
||||
// `smatrix`/`cost` commands reject decimal costs ("found symbol . when
|
||||
// reading transformation costs"). So: recode to TNT's alphabet and
|
||||
// integer-scale the costs here, at this adapter's boundary, rather than
|
||||
// degrading the project's own canonical (IUPAC, real-valued) output.
|
||||
|
||||
const TNT_STATE_SYMBOL: [char; 16] = [
|
||||
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
|
||||
];
|
||||
|
||||
fn write_sankoff_tnt(
|
||||
matrix: &[[f64; 16]; 16],
|
||||
alignment: &SnpAlignment,
|
||||
labels: &[String],
|
||||
output: &Option<PathBuf>,
|
||||
cost_scale: f64,
|
||||
) {
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff.tnt", p.display()))
|
||||
.unwrap_or_else(|| "sankoff.tnt".into());
|
||||
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
|
||||
// IUPAC-ish symbol -> bitmask, to translate the alignment (which uses
|
||||
// `STATE_SYMBOL`, `-` already normalised to `0` by `snp_pseudo_alignment`
|
||||
// callers) into TNT's alphabet without re-deriving state indices.
|
||||
let mut iupac_to_state = [0u8; 128];
|
||||
for (state, &sym) in STATE_SYMBOL.iter().enumerate() {
|
||||
iupac_to_state[sym as usize] = state as u8;
|
||||
}
|
||||
|
||||
let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
|
||||
writeln!(f, "xread").unwrap();
|
||||
writeln!(f, "'obikmer central-position SNP families, calibrated Sankoff 16-state encoding'").unwrap();
|
||||
writeln!(f, "{n_sites} {}", labels.len()).unwrap();
|
||||
for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
|
||||
write!(f, "{label} ").unwrap();
|
||||
for &b in seq {
|
||||
let b = if b == b'-' { b'0' } else { b };
|
||||
let state = iupac_to_state[b as usize];
|
||||
write!(f, "{}", TNT_STATE_SYMBOL[state as usize]).unwrap();
|
||||
}
|
||||
writeln!(f).unwrap();
|
||||
}
|
||||
writeln!(f, ";\n").unwrap();
|
||||
|
||||
let scaled_matrix = scaled_metric_matrix(matrix, cost_scale);
|
||||
|
||||
writeln!(f, "smatrix =0 (family16)").unwrap();
|
||||
for i in 0..16 {
|
||||
for j in (i + 1)..16 {
|
||||
writeln!(f, "{}/{} {}", TNT_STATE_SYMBOL[i], TNT_STATE_SYMBOL[j], scaled_matrix[i][j]).unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(f, ";\n").unwrap();
|
||||
|
||||
writeln!(f, "ccode ( 0.{} ;", n_sites - 1).unwrap();
|
||||
writeln!(f, "smatrix +0 0.{} ;", n_sites - 1).unwrap();
|
||||
writeln!(f).unwrap();
|
||||
|
||||
// Basename only (not the full `path`/`output` prefix): TNT's natural
|
||||
// workflow is to `cd` into the output directory before `proc`-ing the
|
||||
// script, and an absolute path here would break if that directory is
|
||||
// later moved or copied elsewhere.
|
||||
let tre_name = output.as_ref()
|
||||
.and_then(|p| p.file_name())
|
||||
.map(|n| format!("{}_sankoff.tre", n.to_string_lossy()))
|
||||
.unwrap_or_else(|| "sankoff.tre".into());
|
||||
|
||||
// TNT's plain command parser has no comment syntax of its own — `/* */`
|
||||
// and `[ ]` are only recognised inside the (separately-enabled) macro
|
||||
// scripting language, and fail with "No command!" here otherwise
|
||||
// (verified against this file with the local TNT binary). `quote` is
|
||||
// the closest working equivalent: it prints free text and does not
|
||||
// otherwise affect parsing, so it doubles as an explanation of the
|
||||
// defaults below when the script is run. `;` ends a `quote` block like
|
||||
// any other TNT command, so the text itself must avoid semicolons.
|
||||
writeln!(f, "quote").unwrap();
|
||||
writeln!(f, "Default search below (edit or delete this block to run your own strategy):").unwrap();
|
||||
writeln!(f, " hold N : size of TNT's tree buffer (how many equally-parsimonious").unwrap();
|
||||
writeln!(f, " trees it keeps in memory at once), 20 is a small, fast").unwrap();
|
||||
writeln!(f, " default, raise it if mult reports it had to drop trees.").unwrap();
|
||||
writeln!(f, " mult : traditional search (random addition sequences followed by").unwrap();
|
||||
writeln!(f, " TBR branch-swapping, TNT's own default replication count),").unwrap();
|
||||
writeln!(f, " a reasonable first-pass strategy on this data's memory").unwrap();
|
||||
writeln!(f, " footprint, xmult's ratchet/drift/tree-fusion buffers ran").unwrap();
|
||||
writeln!(f, " this out of RAM at TNT's default mxram on this dataset.").unwrap();
|
||||
writeln!(f, " export - F : write the trees held in the buffer to file F, in").unwrap();
|
||||
writeln!(f, " TNT/Hennig86 format ('-' means trees, as opposed to data).").unwrap();
|
||||
writeln!(f, ";").unwrap();
|
||||
writeln!(f, "hold 20;").unwrap();
|
||||
writeln!(f, "mult;").unwrap();
|
||||
writeln!(f, "export - {tre_name};").unwrap();
|
||||
|
||||
info!(
|
||||
"TNT script → {path} (costs scaled x{cost_scale:.0}, runs a default `hold 20; mult;` \
|
||||
search and writes trees to {tre_name} in TNT's working directory — edit the trailing \
|
||||
comment block in the script to change this)\n\
|
||||
Run it with:\n \
|
||||
printf 'proc {path};\\nquit;\\n' | tnt\n\
|
||||
(or start `tnt` interactively and type `proc {path};`)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Scale `matrix` by `cost_scale` and round to integers (TNT's smatrix/cost
|
||||
/// commands reject decimals), then take the *metric closure* of the result
|
||||
/// (Floyd-Warshall over the 16 states again, on the now-integer values).
|
||||
///
|
||||
/// `matrix` is already a metric in its real-valued form (it's a
|
||||
/// shortest-path closure itself — see `build_cost_matrix`), but rounding
|
||||
/// each cell independently can still break the triangle inequality: e.g.
|
||||
/// two real costs of `1.734` each round to `173`, summing to `346`, while
|
||||
/// their own real sum `3.468` rounds to `347` — TNT then reports "triangle
|
||||
/// inequality violated ... Fixed" and silently substitutes its own
|
||||
/// corrected value. Re-closing after rounding makes that correction
|
||||
/// explicit and reproducible here instead, rather than left implicit and
|
||||
/// TNT-version-dependent.
|
||||
fn scaled_metric_matrix(matrix: &[[f64; 16]; 16], cost_scale: f64) -> [[i64; 16]; 16] {
|
||||
let mut m = [[0i64; 16]; 16];
|
||||
for i in 0..16 {
|
||||
for j in 0..16 {
|
||||
m[i][j] = (matrix[i][j] * cost_scale).round() as i64;
|
||||
}
|
||||
}
|
||||
for k in 0..16 {
|
||||
for i in 0..16 {
|
||||
for j in 0..16 {
|
||||
let via = m[i][k] + m[k][j];
|
||||
if via < m[i][j] {
|
||||
m[i][j] = via;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
// ── Sankoff cost matrix → PhyG custom-alphabet TCM + ready-to-run script ────
|
||||
//
|
||||
// PhyG's `tcm:STRING` format needs no alphabet recoding, unlike `--tnt`:
|
||||
// its parser reads the alphabet straight from the tcm file's own first
|
||||
// line, so `--sankoff`'s own `_sankoff.fasta` (already IUPAC+`0`) is reused
|
||||
// as-is via `prefasta:`. PhyG auto-adds its own indel/gap state as an
|
||||
// (n+1)-th row/column of the tcm — inert here since the alignment already
|
||||
// encodes absence as an ordinary state (`0`), never as `-` (see
|
||||
// `write_sankoff_alignment_fasta`'s own comment on why, and the RAxML-era
|
||||
// bug that motivated it). The gap row/column below reuses `matrix[i][0]`/
|
||||
// `matrix[0][j]` (cost to/from `∅`) as the closest principled value for a
|
||||
// state that, in practice, is never actually triggered.
|
||||
|
||||
fn write_sankoff_phyg(matrix: &[[f64; 16]; 16], output: &Option<PathBuf>, cost_scale: f64) {
|
||||
let scaled_matrix = scaled_metric_matrix(matrix, cost_scale);
|
||||
|
||||
let basename = |suffix: &str| -> String {
|
||||
output.as_ref()
|
||||
.and_then(|p| p.file_name())
|
||||
.map(|n| format!("{}{suffix}", n.to_string_lossy()))
|
||||
.unwrap_or_else(|| format!("sankoff{suffix}"))
|
||||
};
|
||||
let full_path = |suffix: &str| -> String {
|
||||
output.as_ref()
|
||||
.map(|p| format!("{}{suffix}", p.display()))
|
||||
.unwrap_or_else(|| format!("sankoff{suffix}"))
|
||||
};
|
||||
|
||||
let tcm_path = full_path("_sankoff.tcm");
|
||||
let mut f = BufWriter::new(std::fs::File::create(&tcm_path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {tcm_path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
let alphabet_line = STATE_SYMBOL.iter().map(|c| c.to_string()).collect::<Vec<_>>().join(" ");
|
||||
writeln!(f, "{alphabet_line}").unwrap();
|
||||
for i in 0..16 {
|
||||
let mut row: Vec<i64> = (0..16).map(|j| scaled_matrix[i][j]).collect();
|
||||
row.push(scaled_matrix[i][0]); // gap column: same cost as to/from ∅
|
||||
writeln!(f, "{}", row.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(" ")).unwrap();
|
||||
}
|
||||
let mut gap_row: Vec<i64> = (0..16).map(|j| scaled_matrix[0][j]).collect();
|
||||
gap_row.push(0);
|
||||
writeln!(f, "{}", gap_row.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(" ")).unwrap();
|
||||
info!("PhyG TCM → {tcm_path}");
|
||||
|
||||
let pg_path = full_path("_sankoff.pg");
|
||||
let mut f = BufWriter::new(std::fs::File::create(&pg_path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {pg_path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
let fasta_name = basename("_sankoff.fasta");
|
||||
let tcm_name = basename("_sankoff.tcm");
|
||||
let tre_name = basename("_sankoff.tre");
|
||||
writeln!(f, "read(prefasta:\"{fasta_name}\", tcm:\"{tcm_name}\")").unwrap();
|
||||
writeln!(f, "search(seconds:300, instances:4)").unwrap();
|
||||
writeln!(f, "report(\"{tre_name}\", graphs, newick, overwrite)").unwrap();
|
||||
|
||||
let pg_dir = std::path::Path::new(&pg_path).parent()
|
||||
.filter(|d| !d.as_os_str().is_empty())
|
||||
.map(|d| d.display().to_string())
|
||||
.unwrap_or_else(|| ".".into());
|
||||
let pg_name = std::path::Path::new(&pg_path).file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| pg_path.clone());
|
||||
info!(
|
||||
"PhyG script → {pg_path} (costs scaled x{cost_scale:.0}, runs a default 300s/4-instance \
|
||||
search and writes trees to {tre_name})\n\
|
||||
Run it with:\n \
|
||||
cd {pg_dir} && phyg {pg_name}\n\
|
||||
(`phyg` must run from that directory — `read()`/`report()` in the script use relative \
|
||||
file names)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── UPGMA Newick from kodama dendrogram ───────────────────────────────────────
|
||||
|
||||
fn upgma_to_newick(dendro: &kodama::Dendrogram<f64>, names: &[String]) -> String {
|
||||
@@ -0,0 +1,81 @@
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tracing::info;
|
||||
|
||||
use super::sankoff::{STATE_SYMBOL, scaled_metric_matrix};
|
||||
|
||||
// ── Sankoff cost matrix → PhyG custom-alphabet TCM + ready-to-run script ────
|
||||
//
|
||||
// PhyG's `tcm:STRING` format needs no alphabet recoding, unlike `--tnt`:
|
||||
// its parser reads the alphabet straight from the tcm file's own first
|
||||
// line, so `--sankoff`'s own `_sankoff.fasta` (already IUPAC+`0`) is reused
|
||||
// as-is via `prefasta:`. PhyG auto-adds its own indel/gap state as an
|
||||
// (n+1)-th row/column of the tcm — inert here since the alignment already
|
||||
// encodes absence as an ordinary state (`0`), never as `-` (see
|
||||
// `sankoff::write_sankoff_alignment_fasta`'s own comment on why, and the
|
||||
// RAxML-era bug that motivated it). The gap row/column below reuses
|
||||
// `matrix[i][0]`/`matrix[0][j]` (cost to/from `∅`) as the closest
|
||||
// principled value for a state that, in practice, is never actually
|
||||
// triggered.
|
||||
|
||||
pub(super) fn write_sankoff_phyg(matrix: &[[f64; 16]; 16], output: &Option<PathBuf>, cost_scale: f64) {
|
||||
let scaled_matrix = scaled_metric_matrix(matrix, cost_scale);
|
||||
|
||||
let basename = |suffix: &str| -> String {
|
||||
output.as_ref()
|
||||
.and_then(|p| p.file_name())
|
||||
.map(|n| format!("{}{suffix}", n.to_string_lossy()))
|
||||
.unwrap_or_else(|| format!("sankoff{suffix}"))
|
||||
};
|
||||
let full_path = |suffix: &str| -> String {
|
||||
output.as_ref()
|
||||
.map(|p| format!("{}{suffix}", p.display()))
|
||||
.unwrap_or_else(|| format!("sankoff{suffix}"))
|
||||
};
|
||||
|
||||
let tcm_path = full_path("_sankoff.tcm");
|
||||
let mut f = BufWriter::new(std::fs::File::create(&tcm_path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {tcm_path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
let alphabet_line = STATE_SYMBOL.iter().map(|c| c.to_string()).collect::<Vec<_>>().join(" ");
|
||||
writeln!(f, "{alphabet_line}").unwrap();
|
||||
for i in 0..16 {
|
||||
let mut row: Vec<i64> = (0..16).map(|j| scaled_matrix[i][j]).collect();
|
||||
row.push(scaled_matrix[i][0]); // gap column: same cost as to/from ∅
|
||||
writeln!(f, "{}", row.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(" ")).unwrap();
|
||||
}
|
||||
let mut gap_row: Vec<i64> = (0..16).map(|j| scaled_matrix[0][j]).collect();
|
||||
gap_row.push(0);
|
||||
writeln!(f, "{}", gap_row.iter().map(|v| v.to_string()).collect::<Vec<_>>().join(" ")).unwrap();
|
||||
info!("PhyG TCM → {tcm_path}");
|
||||
|
||||
let pg_path = full_path("_sankoff.pg");
|
||||
let mut f = BufWriter::new(std::fs::File::create(&pg_path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {pg_path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
let fasta_name = basename("_sankoff.fasta");
|
||||
let tcm_name = basename("_sankoff.tcm");
|
||||
let tre_name = basename("_sankoff.tre");
|
||||
writeln!(f, "read(prefasta:\"{fasta_name}\", tcm:\"{tcm_name}\")").unwrap();
|
||||
writeln!(f, "search(seconds:300, instances:4)").unwrap();
|
||||
writeln!(f, "report(\"{tre_name}\", graphs, newick, overwrite)").unwrap();
|
||||
|
||||
let pg_dir = std::path::Path::new(&pg_path).parent()
|
||||
.filter(|d| !d.as_os_str().is_empty())
|
||||
.map(|d| d.display().to_string())
|
||||
.unwrap_or_else(|| ".".into());
|
||||
let pg_name = std::path::Path::new(&pg_path).file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| pg_path.clone());
|
||||
info!(
|
||||
"PhyG script → {pg_path} (costs scaled x{cost_scale:.0}, runs a default 300s/4-instance \
|
||||
search and writes trees to {tre_name})\n\
|
||||
Run it with:\n \
|
||||
cd {pg_dir} && phyg {pg_name}\n\
|
||||
(`phyg` must run from that directory — `read()`/`report()` in the script use relative \
|
||||
file names)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use obifastwrite::{JsonVal, write_record};
|
||||
use obikindex::{BasePairTally, PHatEstimate, SankoffWeights, SnpAlignment};
|
||||
use tracing::info;
|
||||
|
||||
// ── Sankoff pseudo-alignment → FASTA ────────────────────────────────────────
|
||||
//
|
||||
// Same data as `--snp`'s pseudo-alignment (`SnpAlignment`/
|
||||
// `snp_pseudo_alignment`), re-coded so its symbols match the accompanying
|
||||
// `--sankoff-matrix` output exactly: `0` for the empty/absent state instead
|
||||
// of `-`, which TNT/PhyG would otherwise read as their own gap character
|
||||
// rather than our "family absent" state.
|
||||
|
||||
pub(super) fn write_sankoff_alignment_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff.fasta", p.display()))
|
||||
.unwrap_or_else(|| "sankoff.fasta".into());
|
||||
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
|
||||
for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
|
||||
let recoded: Vec<u8> = seq.iter().map(|&b| if b == b'-' { b'0' } else { b }).collect();
|
||||
write_record(&recoded, label, &[("n_sites", JsonVal::Num(n_sites as u64))], &mut f).unwrap_or_else(|e| {
|
||||
eprintln!("error writing {path}: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
}
|
||||
info!("Sankoff pseudo-alignment → {path} ({n_sites} site{})",
|
||||
if n_sites == 1 { "" } else { "s" });
|
||||
}
|
||||
|
||||
// ── Sankoff cost matrix → CSV ────────────────────────────────────────────────
|
||||
//
|
||||
// 16 states indexed by bitmask (bit 0=A, 1=C, 2=G, 3=T; state 0 is `∅`),
|
||||
// matching the convention already used for `--snp`'s IUPAC-coded output and
|
||||
// for the external TNT/PhyG scripts this feeds. Calibration report (p_hat,
|
||||
// its variance, how many pairs/loci went into it, the resulting c_ctx) goes
|
||||
// to the log, not the CSV, since it's a run-level fact, not per-cell data.
|
||||
|
||||
// IUPAC ambiguity code per state (same mapping as `siblings::iupac_code`,
|
||||
// already used for `--snp`'s pseudo-alignment — a biologist reads "R" as
|
||||
// "A or G" without needing this file's convention explained), with `0`
|
||||
// standing in for the empty state (`-` would collide with TNT/PhyG's own
|
||||
// gap/range syntax). Bit order: 0=A, 1=C, 2=G, 3=T. This project's
|
||||
// canonical alphabet — `tnt::write_sankoff_tnt` recodes it to TNT's own
|
||||
// default alphabet at the adapter boundary, rather than using it here.
|
||||
pub(super) const STATE_SYMBOL: [char; 16] = [
|
||||
'0', 'A', 'C', 'M', 'G', 'R', 'S', 'V', 'T', 'W', 'Y', 'H', 'K', 'D', 'B', 'N',
|
||||
];
|
||||
|
||||
pub(super) fn write_sankoff_matrix_csv(
|
||||
matrix: &[[f64; 16]; 16],
|
||||
estimate: &PHatEstimate,
|
||||
weights: &SankoffWeights,
|
||||
output: &Option<PathBuf>,
|
||||
) {
|
||||
info!(
|
||||
p_hat = format_args!("{:.6}", estimate.p_hat),
|
||||
variance = format_args!("{:.3e}", estimate.variance),
|
||||
n_pairs_included = estimate.n_pairs_included,
|
||||
n_loci_total = estimate.n_loci_total,
|
||||
c_ctx = format_args!("{:.4}", weights.c_ctx),
|
||||
"Sankoff matrix calibration"
|
||||
);
|
||||
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff_matrix.csv", p.display()))
|
||||
.unwrap_or_else(|| "sankoff_matrix.csv".into());
|
||||
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
write!(f, "state").unwrap();
|
||||
for sym in STATE_SYMBOL { write!(f, ",{sym}").unwrap(); }
|
||||
writeln!(f).unwrap();
|
||||
for (s, row) in matrix.iter().enumerate() {
|
||||
write!(f, "{}", STATE_SYMBOL[s]).unwrap();
|
||||
for cost in row { write!(f, ",{cost:.4}").unwrap(); }
|
||||
writeln!(f).unwrap();
|
||||
}
|
||||
info!("Sankoff cost matrix → {path}");
|
||||
}
|
||||
|
||||
// ── Sankoff calibration parameters → YAML report ────────────────────────────
|
||||
//
|
||||
// Everything `--sankoff` estimates from real data, in one durable,
|
||||
// machine-readable file: `p_hat` and its variance (with how many pairs/loci
|
||||
// went into it), the derived `c_ctx`, and the base-pair substitution tally
|
||||
// (raw counts, not just the derived costs) — kept for the same reason raw
|
||||
// counts are kept anywhere else in this project: costs are a modelling
|
||||
// choice built *from* the counts, and reproducing/re-deriving them later
|
||||
// needs the counts, not just their current derived value. Structured (YAML,
|
||||
// not an ad hoc key=value text file) so R/Python/etc. can load it directly
|
||||
// rather than re-parsing free text.
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct SankoffSubstitution {
|
||||
pair: String,
|
||||
count: u64,
|
||||
cost: f64,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct SankoffParamsReport {
|
||||
ratio_ceiling: f64,
|
||||
flank_length_m: usize,
|
||||
p_hat: f64,
|
||||
p_hat_variance: f64,
|
||||
n_pairs_included: usize,
|
||||
n_loci_total: u64,
|
||||
/// Weighted-average substitution cost — see `c_ctx_from_p_hat`'s docs:
|
||||
/// this is what turns the raw expected mutation *count* behind `c_ctx`
|
||||
/// into an actual cost (not every mutation is worth a flat `1`).
|
||||
mean_sub_cost: f64,
|
||||
c_ctx: f64,
|
||||
substitutions: Vec<SankoffSubstitution>,
|
||||
}
|
||||
|
||||
pub(super) fn write_sankoff_params(
|
||||
estimate: &PHatEstimate,
|
||||
tally: &BasePairTally,
|
||||
weights: &SankoffWeights,
|
||||
ratio_ceiling: f64,
|
||||
m: usize,
|
||||
mean_sub_cost: f64,
|
||||
output: &Option<PathBuf>,
|
||||
) {
|
||||
const BASE_LETTER: [char; 4] = ['A', 'C', 'G', 'T'];
|
||||
|
||||
let mut substitutions = Vec::with_capacity(6);
|
||||
for a in 0..4 {
|
||||
for b in (a + 1)..4 {
|
||||
substitutions.push(SankoffSubstitution {
|
||||
pair: format!("{}/{}", BASE_LETTER[a], BASE_LETTER[b]),
|
||||
count: tally.counts[a][b],
|
||||
cost: weights.sub_cost[a][b],
|
||||
});
|
||||
}
|
||||
}
|
||||
let report = SankoffParamsReport {
|
||||
ratio_ceiling,
|
||||
flank_length_m: m,
|
||||
p_hat: estimate.p_hat,
|
||||
p_hat_variance: estimate.variance,
|
||||
n_pairs_included: estimate.n_pairs_included,
|
||||
n_loci_total: estimate.n_loci_total,
|
||||
mean_sub_cost,
|
||||
c_ctx: weights.c_ctx,
|
||||
substitutions,
|
||||
};
|
||||
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff_params.yaml", p.display()))
|
||||
.unwrap_or_else(|| "sankoff_params.yaml".into());
|
||||
let f = std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
serde_yaml::to_writer(f, &report).unwrap_or_else(|e| {
|
||||
eprintln!("error writing {path}: {e}");
|
||||
std::process::exit(1);
|
||||
});
|
||||
info!("Sankoff calibration parameters → {path}");
|
||||
}
|
||||
|
||||
/// Scale `matrix` by `cost_scale` and round to integers (TNT's smatrix/cost
|
||||
/// and PhyG's `tcm:` commands both reject decimals), then take the *metric
|
||||
/// closure* of the result (Floyd-Warshall over the 16 states again, on the
|
||||
/// now-integer values).
|
||||
///
|
||||
/// `matrix` is already a metric in its real-valued form (it's a
|
||||
/// shortest-path closure itself — see `obikindex::build_cost_matrix`), but
|
||||
/// rounding each cell independently can still break the triangle
|
||||
/// inequality: e.g. two real costs of `1.734` each round to `173`, summing
|
||||
/// to `346`, while their own real sum `3.468` rounds to `347` — TNT then
|
||||
/// reports "triangle inequality violated ... Fixed" and silently
|
||||
/// substitutes its own corrected value. Re-closing after rounding makes
|
||||
/// that correction explicit and reproducible here instead, rather than
|
||||
/// left implicit and tool-version-dependent.
|
||||
pub(super) fn scaled_metric_matrix(matrix: &[[f64; 16]; 16], cost_scale: f64) -> [[i64; 16]; 16] {
|
||||
let mut m = [[0i64; 16]; 16];
|
||||
for i in 0..16 {
|
||||
for j in 0..16 {
|
||||
m[i][j] = (matrix[i][j] * cost_scale).round() as i64;
|
||||
}
|
||||
}
|
||||
for k in 0..16 {
|
||||
for i in 0..16 {
|
||||
for j in 0..16 {
|
||||
let via = m[i][k] + m[k][j];
|
||||
if via < m[i][j] {
|
||||
m[i][j] = via;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
m
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use obikindex::SnpAlignment;
|
||||
use tracing::info;
|
||||
|
||||
use super::sankoff::{STATE_SYMBOL, scaled_metric_matrix};
|
||||
|
||||
// ── Sankoff cost matrix + alignment → ready-to-run TNT script ──────────────
|
||||
//
|
||||
// TNT's *default* xread reader only accepts its own 0-9A-F alphabet (see
|
||||
// its manual: "up to 16 states are allowed by xread, using symbols 0-9 ...
|
||||
// and A-F") — the wider IUPAC set `STATE_SYMBOL` uses is rejected as an
|
||||
// "alien symbol" unless `nstates dna` is set, which imposes TNT's own fixed
|
||||
// DNA encoding instead, incompatible with a custom smatrix. And TNT's
|
||||
// `smatrix`/`cost` commands reject decimal costs ("found symbol . when
|
||||
// reading transformation costs"). So: recode to TNT's alphabet and
|
||||
// integer-scale the costs here, at this adapter's boundary, rather than
|
||||
// degrading the project's own canonical (IUPAC, real-valued) output.
|
||||
|
||||
const TNT_STATE_SYMBOL: [char; 16] = [
|
||||
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F',
|
||||
];
|
||||
|
||||
pub(super) fn write_sankoff_tnt(
|
||||
matrix: &[[f64; 16]; 16],
|
||||
alignment: &SnpAlignment,
|
||||
labels: &[String],
|
||||
output: &Option<PathBuf>,
|
||||
cost_scale: f64,
|
||||
) {
|
||||
let path = output.as_ref()
|
||||
.map(|p| format!("{}_sankoff.tnt", p.display()))
|
||||
.unwrap_or_else(|| "sankoff.tnt".into());
|
||||
let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
|
||||
eprintln!("error creating {path}: {e}");
|
||||
std::process::exit(1);
|
||||
}));
|
||||
|
||||
// IUPAC-ish symbol -> bitmask, to translate the alignment (which uses
|
||||
// `STATE_SYMBOL`, `-` already normalised to `0` by `snp_pseudo_alignment`
|
||||
// callers) into TNT's alphabet without re-deriving state indices.
|
||||
let mut iupac_to_state = [0u8; 128];
|
||||
for (state, &sym) in STATE_SYMBOL.iter().enumerate() {
|
||||
iupac_to_state[sym as usize] = state as u8;
|
||||
}
|
||||
|
||||
let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
|
||||
writeln!(f, "xread").unwrap();
|
||||
writeln!(f, "'obikmer central-position SNP families, calibrated Sankoff 16-state encoding'").unwrap();
|
||||
writeln!(f, "{n_sites} {}", labels.len()).unwrap();
|
||||
for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
|
||||
write!(f, "{label} ").unwrap();
|
||||
for &b in seq {
|
||||
let b = if b == b'-' { b'0' } else { b };
|
||||
let state = iupac_to_state[b as usize];
|
||||
write!(f, "{}", TNT_STATE_SYMBOL[state as usize]).unwrap();
|
||||
}
|
||||
writeln!(f).unwrap();
|
||||
}
|
||||
writeln!(f, ";\n").unwrap();
|
||||
|
||||
let scaled_matrix = scaled_metric_matrix(matrix, cost_scale);
|
||||
|
||||
writeln!(f, "smatrix =0 (family16)").unwrap();
|
||||
for i in 0..16 {
|
||||
for j in (i + 1)..16 {
|
||||
writeln!(f, "{}/{} {}", TNT_STATE_SYMBOL[i], TNT_STATE_SYMBOL[j], scaled_matrix[i][j]).unwrap();
|
||||
}
|
||||
}
|
||||
writeln!(f, ";\n").unwrap();
|
||||
|
||||
writeln!(f, "ccode ( 0.{} ;", n_sites - 1).unwrap();
|
||||
writeln!(f, "smatrix +0 0.{} ;", n_sites - 1).unwrap();
|
||||
writeln!(f).unwrap();
|
||||
|
||||
// Basename only (not the full `path`/`output` prefix): TNT's natural
|
||||
// workflow is to `cd` into the output directory before `proc`-ing the
|
||||
// script, and an absolute path here would break if that directory is
|
||||
// later moved or copied elsewhere.
|
||||
let tre_name = output.as_ref()
|
||||
.and_then(|p| p.file_name())
|
||||
.map(|n| format!("{}_sankoff.tre", n.to_string_lossy()))
|
||||
.unwrap_or_else(|| "sankoff.tre".into());
|
||||
|
||||
// TNT's plain command parser has no comment syntax of its own — `/* */`
|
||||
// and `[ ]` are only recognised inside the (separately-enabled) macro
|
||||
// scripting language, and fail with "No command!" here otherwise
|
||||
// (verified against this file with the local TNT binary). `quote` is
|
||||
// the closest working equivalent: it prints free text and does not
|
||||
// otherwise affect parsing, so it doubles as an explanation of the
|
||||
// defaults below when the script is run. `;` ends a `quote` block like
|
||||
// any other TNT command, so the text itself must avoid semicolons.
|
||||
writeln!(f, "quote").unwrap();
|
||||
writeln!(f, "Default search below (edit or delete this block to run your own strategy):").unwrap();
|
||||
writeln!(f, " hold N : size of TNT's tree buffer (how many equally-parsimonious").unwrap();
|
||||
writeln!(f, " trees it keeps in memory at once), 20 is a small, fast").unwrap();
|
||||
writeln!(f, " default, raise it if mult reports it had to drop trees.").unwrap();
|
||||
writeln!(f, " mult : traditional search (random addition sequences followed by").unwrap();
|
||||
writeln!(f, " TBR branch-swapping, TNT's own default replication count),").unwrap();
|
||||
writeln!(f, " a reasonable first-pass strategy on this data's memory").unwrap();
|
||||
writeln!(f, " footprint, xmult's ratchet/drift/tree-fusion buffers ran").unwrap();
|
||||
writeln!(f, " this out of RAM at TNT's default mxram on this dataset.").unwrap();
|
||||
writeln!(f, " export - F : write the trees held in the buffer to file F, in").unwrap();
|
||||
writeln!(f, " TNT/Hennig86 format ('-' means trees, as opposed to data).").unwrap();
|
||||
writeln!(f, ";").unwrap();
|
||||
writeln!(f, "hold 20;").unwrap();
|
||||
writeln!(f, "mult;").unwrap();
|
||||
writeln!(f, "export - {tre_name};").unwrap();
|
||||
|
||||
info!(
|
||||
"TNT script → {path} (costs scaled x{cost_scale:.0}, runs a default `hold 20; mult;` \
|
||||
search and writes trees to {tre_name} in TNT's working directory — edit the trailing \
|
||||
comment block in the script to change this)\n\
|
||||
Run it with:\n \
|
||||
printf 'proc {path};\\nquit;\\n' | tnt\n\
|
||||
(or start `tnt` interactively and type `proc {path};`)"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user