Add phylogenetic CLI options for family overlap and missing data

Introduces CLI flags for computing pairwise family overlap matrices and filtering genomes below a shared family threshold. Adds a free-loss mode that recodes locus non-detection states to missing data symbols in Sankoff-calibrated alignments, resolving ascertainment bias handling for IQ-TREE. Updates empirical transition parameters, removes the legacy model asset, and extends output writers for CSV diagnostics, FASTA pseudo-alignments, and Newick trees.

src/obikmer/src/cmd/phylo/args.rs

@@ -0,0 +1,243 @@
+use std::path::PathBuf;
+
+use clap::Args;
+use obikindex::DistanceMetric;
+
+#[derive(clap::ValueEnum, Clone, Copy, Debug)]
+pub enum MetricArg {
+    Jaccard,
+    Mash,
+    Hamming,
+    BrayCurtis,
+    #[value(name = "relfreq-bray-curtis")]
+    RelfreqBrayCurtis,
+    Euclidean,
+    #[value(name = "relfreq-euclidean")]
+    RelfreqEuclidean,
+    Hellinger,
+    #[value(name = "hellinger-euclidean")]
+    HellingerEuclidean,
+}
+
+impl From<MetricArg> for DistanceMetric {
+    fn from(m: MetricArg) -> Self {
+        match m {
+            MetricArg::Jaccard            => DistanceMetric::Jaccard,
+            MetricArg::Mash               => DistanceMetric::Mash,
+            MetricArg::Hamming            => DistanceMetric::Hamming,
+            MetricArg::BrayCurtis         => DistanceMetric::BrayCurtis,
+            MetricArg::RelfreqBrayCurtis  => DistanceMetric::RelfreqBrayCurtis,
+            MetricArg::Euclidean          => DistanceMetric::Euclidean,
+            MetricArg::RelfreqEuclidean   => DistanceMetric::RelfreqEuclidean,
+            MetricArg::Hellinger          => DistanceMetric::Hellinger,
+            MetricArg::HellingerEuclidean => DistanceMetric::HellingerEuclidean,
+        }
+    }
+}
+
+#[derive(Args)]
+pub struct PhyloArgs {
+    /// Index directory
+    pub index: PathBuf,
+
+    /// Distance metric to compute
+    #[arg(long, value_enum, default_value = "jaccard")]
+    pub metric: MetricArg,
+
+    /// Minimum count to consider a kmer present when computing Jaccard on count indexes
+    #[arg(long, default_value = "1")]
+    pub presence_threshold: u32,
+
+    /// Also output the shared-kmer count matrix (CSV)
+    #[arg(long)]
+    pub shared_kmers: bool,
+
+    /// Compute and write a Neighbor-Joining tree (Newick)
+    #[arg(long)]
+    pub nj: bool,
+
+    /// Compute and write a UPGMA tree (Newick)
+    #[arg(long)]
+    pub upgma: bool,
+
+    /// Build the sibling-count/minorant annex on this (multi-genome) index
+    /// — see `docmd/theory/evolutionary_distances.md`, Step 2b. Construction
+    /// only; does not by itself compute or write any statistics.
+    #[arg(long)]
+    pub sibling_annex: bool,
+
+    /// Exclude a genome (by its exact label) from every computation below
+    /// that reads the sibling annex — `--raw-snp-distance`/`--raw-snp-counts`,
+    /// `--snp`, and `--sankoff` (and everything `--sankoff` implies: the
+    /// cardinality/composition transition models, the exported
+    /// matrix/alignment, `--tnt`/`--phyg`/`--iqtree`). Repeatable. Does
+    /// *not* affect the plain `--metric` distance matrix/NJ/UPGMA path (a
+    /// different, unrelated computation). Applied by zeroing the excluded
+    /// genome's row/column after `raw_snp_distance` runs (a pair with zero
+    /// counts is already skipped by `base_pair_tally`/`cardinality_tally`,
+    /// so this needs no change to the underlying traversal) and by
+    /// dropping its row from `snp_pseudo_alignment`'s output — the annex
+    /// is still built/scanned for the excluded genome too, just not used
+    /// afterward. For a genome with almost no informative sites shared
+    /// with anything else (see `docmd/theory/evolutionary_distances.md`,
+    /// the IQ-TREE/Mash rogue-taxon discussion), its presence can
+    /// otherwise silently bias the transition models.
+    #[arg(long = "exclude-genome", value_name = "LABEL")]
+    pub exclude_genome: Vec<String>,
+
+    /// Auto-exclude any genome whose mean shared-family count against every
+    /// other genome (same statistic as `--family-overlap`'s matrix, averaged
+    /// over each row excluding the diagonal) falls below this threshold —
+    /// same exclusion machinery as `--exclude-genome`, applied on top of it
+    /// rather than instead of it. Empirically, genomes below ~1000 shared
+    /// families on the 20-genome benchmark are exactly the ones that placed
+    /// themselves arbitrarily under `--tnt`/`--iqtree` (near-zero branch
+    /// lengths, grafted inside unrelated clades) — too little real
+    /// constraint on where they belong. See
+    /// `docmd/theory/evolutionary_distances.md`, "Locus dropout under
+    /// incomplete coverage".
+    #[arg(long, value_name = "N")]
+    pub min_shared_family: Option<f64>,
+
+    /// Tally the sibling-count distribution (CSV) of an already-built annex
+    /// (run with `--sibling-annex` first, in this invocation or an earlier
+    /// one). A separate, occasional diagnostic pass — not run every time the
+    /// annex itself is (re)built.
+    #[arg(long)]
+    pub sibling_stats: bool,
+
+    /// Compute the raw p-distance restricted to loci that are single-copy
+    /// in both genomes of each pair (an already-built sibling annex is
+    /// required — run with `--sibling-annex` first, in this invocation or
+    /// an earlier one). A quick way to test the central-position SNP
+    /// estimator against a real index; not the full `SnpTally` design.
+    #[arg(long)]
+    pub raw_snp_distance: bool,
+
+    /// Write the raw per-pair counts (`n_snp`, `n_shared`, `n_eligible`)
+    /// behind `--raw-snp-distance`'s ratio, one row per genome pair — a
+    /// diagnostic table, not a matrix. The ratio alone can't distinguish
+    /// "identical at every eligible locus" from "almost no eligible loci
+    /// at all" (e.g. `0.0` from 0/2 looks the same as `0.0` from 0/2000),
+    /// and that distinction matters a lot for genome pairs near the edge
+    /// of what central-position families can resolve (see
+    /// `docmd/theory/evolutionary_distances.md`, "Run 3" and the
+    /// IQ-TREE/Mash comparison). Same annex requirement as
+    /// `--raw-snp-distance`.
+    #[arg(long)]
+    pub raw_snp_counts: bool,
+
+    /// Write a SNP-only pseudo-alignment (FASTA, IUPAC-coded) from an
+    /// already-built sibling annex — one row per genome, one column per
+    /// variable family (monomorphic families skipped), no flanking
+    /// sequence. See `docmd/theory/evolutionary_distances.md`,
+    /// "Multi-genome framing: family as pseudo-alignment column".
+    #[arg(long)]
+    pub snp: bool,
+
+    /// Write an NxN CSV (`<prefix>_family_overlap.csv`) of, for each genome
+    /// pair, how many variable families (same set `--snp`'s pseudo-alignment
+    /// uses — `family_size() >= 2`) both genomes actually carry a call for
+    /// (neither is `∅`). A direct read of how much informative content two
+    /// genomes actually share at the family level — the diagnostic for why
+    /// a genome with little overlap with anything else (e.g. an
+    /// under-covered or very divergent one) ends up placed unstably by
+    /// `--tnt`/`--iqtree`: little-to-no shared, real data to constrain it.
+    /// Same annex requirement as `--snp`.
+    #[arg(long)]
+    pub family_overlap: bool,
+
+    /// Calibrate a 16-state Sankoff cost matrix and its matching
+    /// pseudo-alignment from an already-built sibling annex (run with
+    /// `--sibling-annex` first, in this invocation or an earlier one), for
+    /// use with TNT/PhyG. See `docmd/theory/evolutionary_distances.md`,
+    /// "Sankoff parsimony as the resolution of the 16-state model problem".
+    #[arg(long)]
+    pub sankoff: bool,
+
+    /// Recode a family's non-detection (`∅`, no member observed in a
+    /// genome) as TNT/PhyG/IQ-TREE's own missing-data symbol (`?`) in
+    /// `--sankoff`'s FASTA and every export built from it (`--tnt`,
+    /// `--phyg`, `--iqtree`), instead of an ordinary, costed 16th alphabet
+    /// state (the default). For genome-skim/reduced-representation inputs
+    /// (coverage often < 1x), non-detection is dominated by sampling
+    /// failure, not true loss — scoring it as a real state risks grouping
+    /// genomes by shared undersampling rather than shared ancestry. `?`
+    /// (not `-`) because `-` still carries gap/indel semantics in these
+    /// tools; a non-detected family is not an observed deletion. See
+    /// `docmd/theory/evolutionary_distances.md`, "Locus dropout under
+    /// incomplete coverage".
+    #[arg(long)]
+    pub free_loss: bool,
+
+    /// Exclude genome pairs whose raw SNP ratio exceeds this value from the
+    /// `p_hat` calibration pooled by `--sankoff` — a pair this close to
+    /// saturation carries no information about `p_hat` and would bias it
+    /// upward if pooled in (unlike a low eligible-loci count, which barely
+    /// moves the pooled estimate either way — see design doc).
+    #[arg(long, default_value = "0.5")]
+    pub sankoff_ratio_ceiling: f64,
+
+    /// Also write <prefix>_sankoff.tnt, a ready-to-run TNT script (`proc
+    /// <file>;`) for the same matrix/alignment `--sankoff` computes —
+    /// recoded to TNT's default xread alphabet (0-9A-F only; TNT rejects
+    /// the wider IUPAC set `--sankoff`'s own output uses unless `nstates
+    /// dna` is set, which imposes TNT's own incompatible DNA encoding
+    /// instead) with integer-scaled costs (TNT's smatrix/cost commands
+    /// reject decimals). Implies `--sankoff`.
+    #[arg(long)]
+    pub tnt: bool,
+
+    /// Also write <prefix>_sankoff.tcm and <prefix>_sankoff.pg, a
+    /// custom-alphabet cost matrix and a ready-to-run PhyG script (`read`/
+    /// `search`/`report`) for the same matrix/alignment `--sankoff`
+    /// computes. Reuses `--sankoff`'s own `_sankoff.fasta` directly — PhyG's
+    /// `tcm:` alphabet is read from the matrix file itself, so the IUPAC+`0`
+    /// alphabet needs no recoding here, unlike `--tnt`. Implies `--sankoff`.
+    #[arg(long)]
+    pub phyg: bool,
+
+    /// Also write <prefix>_iqtree.model and <prefix>_iqtree.fasta, a
+    /// custom-model file and a matching
+    /// recoded alignment for genuine maximum-likelihood inference with
+    /// IQ-TREE (`iqtree3 -s ... --seqtype MORPH -m ...+ASC`) — real branch
+    /// lengths, unlike `--tnt`/`--phyg`'s parsimony step counts. The model
+    /// is the reversible `Q(i,j) = R(i,j)·π_j` construction: `R`
+    /// (exchangeability, symmetric) recovered from the same calibrated
+    /// cost matrix `--sankoff` computes, `π` the real empirical state
+    /// frequencies counted from the alignment (not IQ-TREE's `+FO`/`+F` —
+    /// neither applies to a custom-file model, see
+    /// `docmd/theory/evolutionary_distances.md`). Only the states that
+    /// actually occur in this alignment are kept, compactly renumbered
+    /// (IQ-TREE infers its state count from the alignment itself, and a
+    /// gap in the numbering would silently misalign the model file).
+    /// Implies `--sankoff`.
+    #[arg(long)]
+    pub iqtree: bool,
+
+    /// Scale factor applied before rounding real-valued costs to the
+    /// integers both `--tnt`'s smatrix/cost commands and `--phyg`'s `tcm:`
+    /// matrix require. Keep this small: the total tree score is this scale
+    /// times the sum of per-character costs across every character (908k+
+    /// for a typical run here), and there are hints in TNT's own manual
+    /// that at least some of its internal accumulators are 32-bit — a large
+    /// scale risks a silent integer overflow (undetectable, not just a
+    /// crash) far more costly than the resolution a bigger factor would
+    /// buy. Shared between `--tnt` and `--phyg` rather than split into two
+    /// flags: both scale the same calibrated matrix for the same reason
+    /// (integer-only cost commands), and no PhyG-specific accumulator-width
+    /// constraint has actually been found to justify a different default.
+    #[arg(long, default_value = "100")]
+    pub sankoff_cost_scale: f64,
+
+    /// Output prefix: <prefix>_dist.csv, <prefix>_shared.csv,
+    /// <prefix>_siblings.csv, <prefix>_rawsnp.csv, <prefix>_rawsnp_counts.csv,
+    /// <prefix>_snp.fasta, <prefix>_family_overlap.csv,
+    /// <prefix>_sankoff_matrix.csv, <prefix>_sankoff_params.yaml,
+    /// <prefix>_sankoff.fasta, <prefix>_sankoff.tnt, <prefix>_sankoff.tcm,
+    /// <prefix>_sankoff.pg, <prefix>_iqtree.model, <prefix>_iqtree.fasta,
+    /// <prefix>_nj.nwk, <prefix>_upgma.nwk.
+    /// If omitted, the distance matrix is written to stdout.
+    #[arg(short, long)]
+    pub output: Option<PathBuf>,
+}

src/obikmer/src/cmd/phylo/family_overlap.rs

@@ -0,0 +1,97 @@
+use std::io::{BufWriter, Write};
+use std::path::PathBuf;
+
+use obikindex::{KmerIndex, SnpAlignment};
+use tracing::info;
+
+// ── Family overlap: shared-family counts and the `--min-shared-family` /
+// `--family-overlap` diagnostics built from them ────────────────────────────
+//
+// Same variable-family columns as `--snp`'s pseudo-alignment. Off-diagonal
+// `[i][j]`: number of columns where both genome `i` and genome `j` carry a
+// call (neither is `∅`) — how much informative family content two genomes
+// actually share, the direct diagnostic for the rogue-taxon placement seen
+// under `--free-loss` (a genome with little overlap with anything else has
+// almost nothing left to constrain it). Diagonal `[i][i]` kept, deliberately
+// not skipped: with `i == j` the condition "both non-`∅`" degenerates to
+// "genome `i` non-`∅`", i.e. the total number of variable families genome
+// `i` carries at all — a genome-level count worth having alongside the
+// pairwise ones, not a separate computation.
+
+/// `counts[i][j]` = number of variable-family columns where both genome `i`
+/// and genome `j` carry a call (neither is `∅`). Shared between
+/// `write_family_overlap_csv` and `--min-shared-family`'s auto-exclusion so
+/// both read off the same definition of "shared family".
+fn family_overlap_counts(alignment: &SnpAlignment) -> Vec<Vec<u64>> {
+    let n = alignment.sequences.len();
+    let mut counts = vec![vec![0u64; n]; n];
+    for i in 0..n {
+        for j in 0..n {
+            counts[i][j] = alignment.sequences[i].iter().zip(alignment.sequences[j].iter())
+                .filter(|&(&a, &b)| a != b'-' && b != b'-')
+                .count() as u64;
+        }
+    }
+    counts
+}
+
+/// Mean of row `i` in a `family_overlap_counts` matrix, excluding the
+/// diagonal — how much informative content genome `i` shares with the
+/// *average* other genome, the statistic `--min-shared-family` thresholds.
+fn mean_offdiag(counts: &[Vec<u64>], i: usize) -> f64 {
+    let n = counts.len();
+    let sum: u64 = (0..n).filter(|&j| j != i).map(|j| counts[i][j]).sum();
+    sum as f64 / (n - 1) as f64
+}
+
+pub(super) fn write_family_overlap_csv(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
+    let path = output.as_ref()
+        .map(|p| format!("{}_family_overlap.csv", p.display()))
+        .unwrap_or_else(|| "family_overlap.csv".into());
+    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
+        eprintln!("error creating {path}: {e}");
+        std::process::exit(1);
+    }));
+    let n = labels.len();
+    let counts = family_overlap_counts(alignment);
+    write!(f, "genome").unwrap();
+    for g in labels { write!(f, ",{g}").unwrap(); }
+    writeln!(f).unwrap();
+    for (i, gi) in labels.iter().enumerate() {
+        write!(f, "{gi}").unwrap();
+        for j in 0..n {
+            write!(f, ",{}", counts[i][j]).unwrap();
+        }
+        writeln!(f).unwrap();
+    }
+    info!("family overlap matrix → {path}");
+}
+
+/// Sets `mask[i] = true` for every genome whose mean shared-family count
+/// (`mean_offdiag`) falls below `threshold`, skipping genomes already
+/// excluded (`mask[i]` already `true`, e.g. via `--exclude-genome`). Builds
+/// its own `SnpAlignment` pass — same redundant-per-flag pattern already
+/// used throughout `run()` (`--snp`/`--sankoff`/`--family-overlap` each call
+/// `snp_pseudo_alignment` independently too).
+pub(super) fn apply_min_shared_family_exclusion(
+    idx: &KmerIndex,
+    labels: &[String],
+    threshold: f64,
+    mask: &mut [bool],
+) {
+    let alignment = idx.snp_pseudo_alignment().unwrap_or_else(|e| {
+        eprintln!("error computing SNP pseudo-alignment for --min-shared-family: {e}");
+        std::process::exit(1);
+    });
+    let counts = family_overlap_counts(&alignment);
+    for (i, label) in labels.iter().enumerate() {
+        if mask[i] {
+            continue; // already excluded via --exclude-genome
+        }
+        let mean = mean_offdiag(&counts, i);
+        if mean < threshold {
+            info!("--min-shared-family: excluding {label} (mean shared families = {mean:.1} < {threshold})");
+            mask[i] = true;
+        }
+    }
+}

src/obikmer/src/cmd/phylo/iqtree.rs

@@ -68,13 +68,59 @@ impl CompactAlphabet {
     }
 }
 
-fn compact_alphabet(alignment: &SnpAlignment) -> CompactAlphabet {
+/// Under `--free-loss`, non-detection (`-`) becomes IQ-TREE's own missing
+/// symbol (`?`) — ignored when IQ-TREE checks a site's constancy for
+/// `+ASC`. A family kept as "variable" by `snp_pseudo_alignment`
+/// (`family_size() >= 2`, a whole-annex property, oblivious to any one
+/// column's actual calls) can still turn constant *among the genomes that
+/// actually have data* once the non-detected ones are excluded from that
+/// check — the same failure mode as the `--exclude-genome`/`drop_excluded`
+/// fix in `mod.rs` (see `docmd/theory/evolutionary_distances.md`, "Two
+/// consistency bugs found and fixed post-implementation"), just triggered
+/// by hiding cells instead of dropping whole rows. Same remedy: rescan
+/// columns treating `-` as ignored, drop any where the remaining calls
+/// agree on a single state. Parsimony (`--tnt`/`--phyg`) has no
+/// no-invariant-site requirement, so this only runs on IQ-TREE's own copy
+/// of the alignment, never mutating the one the caller also hands to those
+/// two exports.
+fn drop_ascertainment_noninformative(alignment: &SnpAlignment) -> SnpAlignment {
+    let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
+    let keep: Vec<bool> = (0..n_sites).map(|site| {
+        let mut first: Option<u8> = None;
+        for seq in &alignment.sequences {
+            let b = seq[site];
+            if b == b'-' {
+                continue;
+            }
+            match first {
+                None => first = Some(b),
+                Some(f) if f != b => return true,
+                _ => {}
+            }
+        }
+        false // all calls missing, or all calls agree — non-informative
+    }).collect();
+
+    let sequences = alignment.sequences.iter()
+        .map(|seq| seq.iter().zip(keep.iter()).filter(|&(_, &k)| k).map(|(&b, _)| b).collect())
+        .collect();
+    SnpAlignment { sequences }
+}
+
+fn compact_alphabet(alignment: &SnpAlignment, free_loss: bool) -> CompactAlphabet {
     let iupac_to_state = state_index_table();
 
     let mut occurs = [false; 16];
     let mut counts = [0u64; 16];
     for seq in &alignment.sequences {
         for &b in seq {
+            if free_loss && b == b'-' {
+                // `?`: IQ-TREE's own missing-data symbol for `--seqtype
+                // MORPH`, marginalised by Felsenstein pruning — not a
+                // numbered state, so excluded from `occurs`/`counts` and
+                // from the compact alphabet built below.
+                continue;
+            }
             let b = if b == b'-' { b'0' } else { b };
             let state = iupac_to_state[b as usize] as usize;
             occurs[state] = true;
@@ -133,6 +179,7 @@ fn write_iqtree_alignment(
     labels: &[String],
     alphabet: &CompactAlphabet,
     output: &Option<PathBuf>,
+    free_loss: bool,
 ) -> (String, usize) {
     let iupac_to_state = state_index_table();
 
@@ -146,6 +193,9 @@ fn write_iqtree_alignment(
     let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
     for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
         let recoded: Vec<u8> = seq.iter().map(|&b| {
+            if free_loss && b == b'-' {
+                return b'?';
+            }
             let b = if b == b'-' { b'0' } else { b };
             let old = iupac_to_state[b as usize] as usize;
             let compact = alphabet.old_to_compact[old]
@@ -165,10 +215,27 @@ pub(super) fn write_iqtree(
     alignment: &SnpAlignment,
     labels: &[String],
     output: &Option<PathBuf>,
+    free_loss: bool,
 ) {
-    let alphabet = compact_alphabet(alignment);
+    let filtered;
+    let alignment = if free_loss {
+        let before = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
+        filtered = drop_ascertainment_noninformative(alignment);
+        let after = filtered.sequences.first().map(|s| s.len()).unwrap_or(0);
+        if after != before {
+            info!(
+                "--free-loss: {before} → {after} sites (dropped columns non-informative once `-` \
+                 is treated as missing — required for +ASC)"
+            );
+        }
+        &filtered
+    } else {
+        alignment
+    };
+
+    let alphabet = compact_alphabet(alignment, free_loss);
     let model_path = write_iqtree_model(matrix, &alphabet, output);
-    let (fasta_path, n_sites) = write_iqtree_alignment(alignment, labels, &alphabet, output);
+    let (fasta_path, n_sites) = write_iqtree_alignment(alignment, labels, &alphabet, output, free_loss);
 
     let prefix_name = output.as_ref()
         .and_then(|p| p.file_name())

src/obikmer/src/cmd/phylo/mod.rs

@@ -1,225 +1,29 @@
+mod args;
+mod family_overlap;
 mod iqtree;
+mod outputs;
 mod phyg;
 mod sankoff;
 mod tnt;
 
 use std::io::{self, BufWriter, Write};
-use std::path::PathBuf;
 
-use clap::Args;
 use kodama::{Method, linkage};
-use obifastwrite::{JsonVal, write_record};
 use obikindex::{
-    DistanceMetric, KmerIndex, RawSnpDistanceOutput,
-    SiblingAnnexStats, SnpAlignment,
+    KmerIndex, RawSnpDistanceOutput, SnpAlignment,
     cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix,
 };
 use speedytree::{DistanceMatrix, Hybrid, NeighborJoiningSolver, to_newick};
 use tracing::info;
 
+pub use args::PhyloArgs;
+use family_overlap::{apply_min_shared_family_exclusion, write_family_overlap_csv};
 use iqtree::write_iqtree;
+use outputs::{write_raw_snp_counts_csv, write_raw_snp_distance_csv, write_sibling_stats_csv, write_snp_fasta, upgma_to_newick};
 use phyg::write_sankoff_phyg;
 use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params};
 use tnt::write_sankoff_tnt;
 
-#[derive(clap::ValueEnum, Clone, Copy, Debug)]
-pub enum MetricArg {
-    Jaccard,
-    Mash,
-    Hamming,
-    BrayCurtis,
-    #[value(name = "relfreq-bray-curtis")]
-    RelfreqBrayCurtis,
-    Euclidean,
-    #[value(name = "relfreq-euclidean")]
-    RelfreqEuclidean,
-    Hellinger,
-    #[value(name = "hellinger-euclidean")]
-    HellingerEuclidean,
-}
-
-impl From<MetricArg> for DistanceMetric {
-    fn from(m: MetricArg) -> Self {
-        match m {
-            MetricArg::Jaccard            => DistanceMetric::Jaccard,
-            MetricArg::Mash               => DistanceMetric::Mash,
-            MetricArg::Hamming            => DistanceMetric::Hamming,
-            MetricArg::BrayCurtis         => DistanceMetric::BrayCurtis,
-            MetricArg::RelfreqBrayCurtis  => DistanceMetric::RelfreqBrayCurtis,
-            MetricArg::Euclidean          => DistanceMetric::Euclidean,
-            MetricArg::RelfreqEuclidean   => DistanceMetric::RelfreqEuclidean,
-            MetricArg::Hellinger          => DistanceMetric::Hellinger,
-            MetricArg::HellingerEuclidean => DistanceMetric::HellingerEuclidean,
-        }
-    }
-}
-
-#[derive(Args)]
-pub struct PhyloArgs {
-    /// Index directory
-    pub index: PathBuf,
-
-    /// Distance metric to compute
-    #[arg(long, value_enum, default_value = "jaccard")]
-    pub metric: MetricArg,
-
-    /// Minimum count to consider a kmer present when computing Jaccard on count indexes
-    #[arg(long, default_value = "1")]
-    pub presence_threshold: u32,
-
-    /// Also output the shared-kmer count matrix (CSV)
-    #[arg(long)]
-    pub shared_kmers: bool,
-
-    /// Compute and write a Neighbor-Joining tree (Newick)
-    #[arg(long)]
-    pub nj: bool,
-
-    /// Compute and write a UPGMA tree (Newick)
-    #[arg(long)]
-    pub upgma: bool,
-
-    /// Build the sibling-count/minorant annex on this (multi-genome) index
-    /// — see `docmd/theory/evolutionary_distances.md`, Step 2b. Construction
-    /// only; does not by itself compute or write any statistics.
-    #[arg(long)]
-    pub sibling_annex: bool,
-
-    /// Exclude a genome (by its exact label) from every computation below
-    /// that reads the sibling annex — `--raw-snp-distance`/`--raw-snp-counts`,
-    /// `--snp`, and `--sankoff` (and everything `--sankoff` implies: the
-    /// cardinality/composition transition models, the exported
-    /// matrix/alignment, `--tnt`/`--phyg`/`--iqtree`). Repeatable. Does
-    /// *not* affect the plain `--metric` distance matrix/NJ/UPGMA path (a
-    /// different, unrelated computation). Applied by zeroing the excluded
-    /// genome's row/column after `raw_snp_distance` runs (a pair with zero
-    /// counts is already skipped by `base_pair_tally`/`cardinality_tally`,
-    /// so this needs no change to the underlying traversal) and by
-    /// dropping its row from `snp_pseudo_alignment`'s output — the annex
-    /// is still built/scanned for the excluded genome too, just not used
-    /// afterward. For a genome with almost no informative sites shared
-    /// with anything else (see `docmd/theory/evolutionary_distances.md`,
-    /// the IQ-TREE/Mash rogue-taxon discussion), its presence can
-    /// otherwise silently bias the transition models.
-    #[arg(long = "exclude-genome", value_name = "LABEL")]
-    pub exclude_genome: Vec<String>,
-
-    /// Tally the sibling-count distribution (CSV) of an already-built annex
-    /// (run with `--sibling-annex` first, in this invocation or an earlier
-    /// one). A separate, occasional diagnostic pass — not run every time the
-    /// annex itself is (re)built.
-    #[arg(long)]
-    pub sibling_stats: bool,
-
-    /// Compute the raw p-distance restricted to loci that are single-copy
-    /// in both genomes of each pair (an already-built sibling annex is
-    /// required — run with `--sibling-annex` first, in this invocation or
-    /// an earlier one). A quick way to test the central-position SNP
-    /// estimator against a real index; not the full `SnpTally` design.
-    #[arg(long)]
-    pub raw_snp_distance: bool,
-
-    /// Write the raw per-pair counts (`n_snp`, `n_shared`, `n_eligible`)
-    /// behind `--raw-snp-distance`'s ratio, one row per genome pair — a
-    /// diagnostic table, not a matrix. The ratio alone can't distinguish
-    /// "identical at every eligible locus" from "almost no eligible loci
-    /// at all" (e.g. `0.0` from 0/2 looks the same as `0.0` from 0/2000),
-    /// and that distinction matters a lot for genome pairs near the edge
-    /// of what central-position families can resolve (see
-    /// `docmd/theory/evolutionary_distances.md`, "Run 3" and the
-    /// IQ-TREE/Mash comparison). Same annex requirement as
-    /// `--raw-snp-distance`.
-    #[arg(long)]
-    pub raw_snp_counts: bool,
-
-    /// Write a SNP-only pseudo-alignment (FASTA, IUPAC-coded) from an
-    /// already-built sibling annex — one row per genome, one column per
-    /// variable family (monomorphic families skipped), no flanking
-    /// sequence. See `docmd/theory/evolutionary_distances.md`,
-    /// "Multi-genome framing: family as pseudo-alignment column".
-    #[arg(long)]
-    pub snp: bool,
-
-    /// Calibrate a 16-state Sankoff cost matrix and its matching
-    /// pseudo-alignment from an already-built sibling annex (run with
-    /// `--sibling-annex` first, in this invocation or an earlier one), for
-    /// use with TNT/PhyG. See `docmd/theory/evolutionary_distances.md`,
-    /// "Sankoff parsimony as the resolution of the 16-state model problem".
-    #[arg(long)]
-    pub sankoff: bool,
-
-    /// Exclude genome pairs whose raw SNP ratio exceeds this value from the
-    /// `p_hat` calibration pooled by `--sankoff` — a pair this close to
-    /// saturation carries no information about `p_hat` and would bias it
-    /// upward if pooled in (unlike a low eligible-loci count, which barely
-    /// moves the pooled estimate either way — see design doc).
-    #[arg(long, default_value = "0.5")]
-    pub sankoff_ratio_ceiling: f64,
-
-    /// Also write <prefix>_sankoff.tnt, a ready-to-run TNT script (`proc
-    /// <file>;`) for the same matrix/alignment `--sankoff` computes —
-    /// recoded to TNT's default xread alphabet (0-9A-F only; TNT rejects
-    /// the wider IUPAC set `--sankoff`'s own output uses unless `nstates
-    /// dna` is set, which imposes TNT's own incompatible DNA encoding
-    /// instead) with integer-scaled costs (TNT's smatrix/cost commands
-    /// reject decimals). Implies `--sankoff`.
-    #[arg(long)]
-    pub tnt: bool,
-
-    /// Also write <prefix>_sankoff.tcm and <prefix>_sankoff.pg, a
-    /// custom-alphabet cost matrix and a ready-to-run PhyG script (`read`/
-    /// `search`/`report`) for the same matrix/alignment `--sankoff`
-    /// computes. Reuses `--sankoff`'s own `_sankoff.fasta` directly — PhyG's
-    /// `tcm:` alphabet is read from the matrix file itself, so the IUPAC+`0`
-    /// alphabet needs no recoding here, unlike `--tnt`. Implies `--sankoff`.
-    #[arg(long)]
-    pub phyg: bool,
-
-    /// Also write <prefix>_iqtree.model and <prefix>_iqtree.fasta, a
-    /// custom-model file and a matching
-    /// recoded alignment for genuine maximum-likelihood inference with
-    /// IQ-TREE (`iqtree3 -s ... --seqtype MORPH -m ...+ASC`) — real branch
-    /// lengths, unlike `--tnt`/`--phyg`'s parsimony step counts. The model
-    /// is the reversible `Q(i,j) = R(i,j)·π_j` construction: `R`
-    /// (exchangeability, symmetric) recovered from the same calibrated
-    /// cost matrix `--sankoff` computes, `π` the real empirical state
-    /// frequencies counted from the alignment (not IQ-TREE's `+FO`/`+F` —
-    /// neither applies to a custom-file model, see
-    /// `docmd/theory/evolutionary_distances.md`). Only the states that
-    /// actually occur in this alignment are kept, compactly renumbered
-    /// (IQ-TREE infers its state count from the alignment itself, and a
-    /// gap in the numbering would silently misalign the model file).
-    /// Implies `--sankoff`.
-    #[arg(long)]
-    pub iqtree: bool,
-
-    /// Scale factor applied before rounding real-valued costs to the
-    /// integers both `--tnt`'s smatrix/cost commands and `--phyg`'s `tcm:`
-    /// matrix require. Keep this small: the total tree score is this scale
-    /// times the sum of per-character costs across every character (908k+
-    /// for a typical run here), and there are hints in TNT's own manual
-    /// that at least some of its internal accumulators are 32-bit — a large
-    /// scale risks a silent integer overflow (undetectable, not just a
-    /// crash) far more costly than the resolution a bigger factor would
-    /// buy. Shared between `--tnt` and `--phyg` rather than split into two
-    /// flags: both scale the same calibrated matrix for the same reason
-    /// (integer-only cost commands), and no PhyG-specific accumulator-width
-    /// constraint has actually been found to justify a different default.
-    #[arg(long, default_value = "100")]
-    pub sankoff_cost_scale: f64,
-
-    /// Output prefix: <prefix>_dist.csv, <prefix>_shared.csv,
-    /// <prefix>_siblings.csv, <prefix>_rawsnp.csv, <prefix>_rawsnp_counts.csv,
-    /// <prefix>_snp.fasta,
-    /// <prefix>_sankoff_matrix.csv, <prefix>_sankoff_params.yaml,
-    /// <prefix>_sankoff.fasta, <prefix>_sankoff.tnt, <prefix>_sankoff.tcm,
-    /// <prefix>_sankoff.pg, <prefix>_iqtree.model, <prefix>_iqtree.fasta,
-    /// <prefix>_nj.nwk, <prefix>_upgma.nwk.
-    /// If omitted, the distance matrix is written to stdout.
-    #[arg(short, long)]
-    pub output: Option<PathBuf>,
-}
-
 pub fn run(args: PhyloArgs) {
     let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
         eprintln!("error opening index: {e}");
@@ -250,6 +54,9 @@ pub fn run(args: PhyloArgs) {
                 }
             }
         }
+        if let Some(threshold) = args.min_shared_family {
+            apply_min_shared_family_exclusion(&idx, &labels, threshold, &mut mask);
+        }
         mask
     };
     let zero_excluded_pairs = |result: &mut RawSnpDistanceOutput| {
@@ -352,6 +159,14 @@ pub fn run(args: PhyloArgs) {
         let (alignment, kept_labels) = drop_excluded(alignment);
         write_snp_fasta(&alignment, &kept_labels, &args.output);
     }
+    if args.family_overlap {
+        let alignment = idx.snp_pseudo_alignment().unwrap_or_else(|e| {
+            eprintln!("error computing SNP pseudo-alignment: {e}");
+            std::process::exit(1);
+        });
+        let (alignment, kept_labels) = drop_excluded(alignment);
+        write_family_overlap_csv(&alignment, &kept_labels, &args.output);
+    }
     if args.sankoff || args.tnt || args.phyg || args.iqtree {
         let mut raw = idx.raw_snp_distance().unwrap_or_else(|e| {
             eprintln!("error computing raw SNP distance: {e}");
@@ -378,16 +193,16 @@ pub fn run(args: PhyloArgs) {
             std::process::exit(1);
         });
         let (alignment, kept_labels) = drop_excluded(alignment);
-        write_sankoff_alignment_fasta(&alignment, &kept_labels, &args.output);
+        write_sankoff_alignment_fasta(&alignment, &kept_labels, &args.output, args.free_loss);
 
         if args.tnt {
-            write_sankoff_tnt(&matrix, &alignment, &kept_labels, &args.output, args.sankoff_cost_scale);
+            write_sankoff_tnt(&matrix, &alignment, &kept_labels, &args.output, args.sankoff_cost_scale, args.free_loss);
         }
         if args.phyg {
             write_sankoff_phyg(&matrix, &args.output, args.sankoff_cost_scale);
         }
         if args.iqtree {
-            write_iqtree(&matrix, &alignment, &kept_labels, &args.output);
+            write_iqtree(&matrix, &alignment, &kept_labels, &args.output, args.free_loss);
         }
     }
 
@@ -403,6 +218,7 @@ pub fn run(args: PhyloArgs) {
         || args.raw_snp_distance
         || args.raw_snp_counts
         || args.snp
+        || args.family_overlap
         || args.sankoff
         || args.tnt
         || args.phyg
@@ -521,165 +337,3 @@ pub fn run(args: PhyloArgs) {
         info!("UPGMA tree → {path}");
     }
 }
-
-// ── Family-size distribution → CSV ──────────────────────────────────────────
-//
-// Each row is a family (the up-to-4 k-mers sharing flanks, differing only at
-// the centre), counted once — at its minorant — regardless of how many of
-// its members are observed. Family size 1..4 (not "sibling count" 0..3):
-// see `docmd/theory/evolutionary_distances.md`, "Definitions".
-
-fn write_sibling_stats_csv(stats: &SiblingAnnexStats, labels: &[String], output: &Option<PathBuf>) {
-    // One row per genome (4 columns, family size 1-4: number of families of
-    // that size for which the genome carries at least one member), plus a
-    // `global` row — the actual deduplicated family-size histogram
-    // (`stats.counts`), NOT a sum of the per-genome columns (a family shared
-    // by several genomes would otherwise be counted once per genome it
-    // appears in, inflating the total beyond the real family count).
-    let path = output.as_ref()
-        .map(|p| format!("{}_siblings.csv", p.display()))
-        .unwrap_or_else(|| "siblings.csv".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    writeln!(f, "genome,1,2,3,4").unwrap();
-    for (label, counts) in labels.iter().zip(stats.per_genome.iter()) {
-        writeln!(f, "{label},{},{},{},{}", counts[0], counts[1], counts[2], counts[3]).unwrap();
-    }
-    writeln!(
-        f, "global,{},{},{},{}",
-        stats.counts[0], stats.counts[1], stats.counts[2], stats.counts[3],
-    ).unwrap();
-    let total: u64 = stats.counts.iter().sum();
-    info!("family-size distribution → {path} (total {total} famil{})",
-        if total == 1 { "y" } else { "ies" });
-}
-
-// ── Raw single-copy SNP distance → CSV ──────────────────────────────────────
-//
-// p_hat[i,j] = snp[i,j] / (snp[i,j] + shared[i,j]) over loci single-copy in
-// both i and j — see `RawSnpDistanceOutput` / `KmerIndex::raw_snp_distance`.
-// A single file: the distance matrix, with an eligible-loci count alongside
-// each value so a 0/0 pair (no eligible locus at all) is distinguishable
-// from a genuinely identical pair.
-
-fn write_raw_snp_distance_csv(result: &RawSnpDistanceOutput, labels: &[String], output: &Option<PathBuf>) {
-    let path = output.as_ref()
-        .map(|p| format!("{}_rawsnp.csv", p.display()))
-        .unwrap_or_else(|| "rawsnp.csv".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    let n = labels.len();
-    write!(f, "genome").unwrap();
-    for g in labels { write!(f, ",{g}").unwrap(); }
-    writeln!(f).unwrap();
-    for (i, g) in labels.iter().enumerate() {
-        write!(f, "{g}").unwrap();
-        for j in 0..n {
-            let snp = result.snp[[i, j]];
-            let shared = result.shared[[i, j]];
-            let eligible = snp + shared;
-            if eligible == 0 {
-                write!(f, ",NA").unwrap();
-            } else {
-                write!(f, ",{:.6}", snp as f64 / eligible as f64).unwrap();
-            }
-        }
-        writeln!(f).unwrap();
-    }
-    info!("raw single-copy SNP distance matrix → {path}");
-}
-
-// ── Raw single-copy SNP distance → per-pair diagnostic counts ──────────────
-//
-// A pair table (one row per unordered genome pair), not a matrix: the ratio
-// alone can't distinguish "identical across every eligible locus" from
-// "almost no eligible locus at all" — both can read `0.0`/`NA` in
-// `--raw-snp-distance`'s output. Distinguishing them matters most exactly
-// where it's easy to miss: genome pairs near the edge of what
-// central-position families can resolve at all (deep cross-lineage splits,
-// see `docmd/theory/evolutionary_distances.md`, "Run 3" and the later
-// IQ-TREE/Mash comparison — a `ratio=0.0` backed by 2 eligible loci is not
-// the same claim as one backed by 2000).
-
-fn write_raw_snp_counts_csv(result: &RawSnpDistanceOutput, labels: &[String], output: &Option<PathBuf>) {
-    let path = output.as_ref()
-        .map(|p| format!("{}_rawsnp_counts.csv", p.display()))
-        .unwrap_or_else(|| "rawsnp_counts.csv".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    let n = labels.len();
-    writeln!(f, "genome_a,genome_b,n_snp,n_shared,n_eligible,ratio").unwrap();
-    for i in 0..n {
-        for j in (i + 1)..n {
-            let snp = result.snp[[i, j]];
-            let shared = result.shared[[i, j]];
-            let eligible = snp + shared;
-            write!(f, "{},{},{snp},{shared},{eligible}", labels[i], labels[j]).unwrap();
-            if eligible == 0 {
-                writeln!(f, ",NA").unwrap();
-            } else {
-                writeln!(f, ",{:.6}", snp as f64 / eligible as f64).unwrap();
-            }
-        }
-    }
-    info!("raw single-copy SNP distance counts (diagnostic) → {path}");
-}
-
-// ── SNP-only pseudo-alignment → FASTA ───────────────────────────────────────
-//
-// One record per genome, IUPAC-coded, no flanking sequence — see
-// `SnpAlignment` / `KmerIndex::snp_pseudo_alignment`. Uses the project's
-// existing FASTA writer (`obifastwrite::write_record`) rather than
-// hand-rolling one.
-
-fn write_snp_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
-    let path = output.as_ref()
-        .map(|p| format!("{}_snp.fasta", p.display()))
-        .unwrap_or_else(|| "snp.fasta".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
-    for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
-        write_record(seq, label, &[("n_sites", JsonVal::Num(n_sites as u64))], &mut f).unwrap_or_else(|e| {
-            eprintln!("error writing {path}: {e}");
-            std::process::exit(1);
-        });
-    }
-    info!("SNP pseudo-alignment → {path} ({n_sites} site{})",
-        if n_sites == 1 { "" } else { "s" });
-}
-
-// ── UPGMA Newick from kodama dendrogram ───────────────────────────────────────
-
-fn upgma_to_newick(dendro: &kodama::Dendrogram<f64>, names: &[String]) -> String {
-    let n = names.len();
-    // node_labels[i]: Newick subtree string for node i (leaves 0..n, internals n..)
-    let mut labels: Vec<String> = names.to_vec();
-    // height of each node: leaves = 0, internal = dissimilarity/2
-    let mut heights: Vec<f64> = vec![0.0; 2 * n - 1];
-
-    for (k, step) in dendro.steps().iter().enumerate() {
-        let new_node = n + k;
-        let h = step.dissimilarity / 2.0;
-        heights[new_node] = h;
-        let c1 = step.cluster1;
-        let c2 = step.cluster2;
-        let bl1 = (h - heights[c1]).max(0.0);
-        let bl2 = (h - heights[c2]).max(0.0);
-        labels.push(format!(
-            "({label1}:{bl1:.6},{label2}:{bl2:.6})",
-            label1 = labels[c1],
-            label2 = labels[c2],
-        ));
-    }
-
-    format!("{};", labels.last().unwrap())
-}

src/obikmer/src/cmd/phylo/outputs.rs

@@ -0,0 +1,168 @@
+use std::io::{BufWriter, Write};
+use std::path::PathBuf;
+
+use obifastwrite::{JsonVal, write_record};
+use obikindex::{RawSnpDistanceOutput, SiblingAnnexStats, SnpAlignment};
+use tracing::info;
+
+// ── Family-size distribution → CSV ──────────────────────────────────────────
+//
+// Each row is a family (the up-to-4 k-mers sharing flanks, differing only at
+// the centre), counted once — at its minorant — regardless of how many of
+// its members are observed. Family size 1..4 (not "sibling count" 0..3):
+// see `docmd/theory/evolutionary_distances.md`, "Definitions".
+
+pub(super) fn write_sibling_stats_csv(stats: &SiblingAnnexStats, labels: &[String], output: &Option<PathBuf>) {
+    // One row per genome (4 columns, family size 1-4: number of families of
+    // that size for which the genome carries at least one member), plus a
+    // `global` row — the actual deduplicated family-size histogram
+    // (`stats.counts`), NOT a sum of the per-genome columns (a family shared
+    // by several genomes would otherwise be counted once per genome it
+    // appears in, inflating the total beyond the real family count).
+    let path = output.as_ref()
+        .map(|p| format!("{}_siblings.csv", p.display()))
+        .unwrap_or_else(|| "siblings.csv".into());
+    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
+        eprintln!("error creating {path}: {e}");
+        std::process::exit(1);
+    }));
+    writeln!(f, "genome,1,2,3,4").unwrap();
+    for (label, counts) in labels.iter().zip(stats.per_genome.iter()) {
+        writeln!(f, "{label},{},{},{},{}", counts[0], counts[1], counts[2], counts[3]).unwrap();
+    }
+    writeln!(
+        f, "global,{},{},{},{}",
+        stats.counts[0], stats.counts[1], stats.counts[2], stats.counts[3],
+    ).unwrap();
+    let total: u64 = stats.counts.iter().sum();
+    info!("family-size distribution → {path} (total {total} famil{})",
+        if total == 1 { "y" } else { "ies" });
+}
+
+// ── Raw single-copy SNP distance → CSV ──────────────────────────────────────
+//
+// p_hat[i,j] = snp[i,j] / (snp[i,j] + shared[i,j]) over loci single-copy in
+// both i and j — see `RawSnpDistanceOutput` / `KmerIndex::raw_snp_distance`.
+// A single file: the distance matrix, with an eligible-loci count alongside
+// each value so a 0/0 pair (no eligible locus at all) is distinguishable
+// from a genuinely identical pair.
+
+pub(super) fn write_raw_snp_distance_csv(result: &RawSnpDistanceOutput, labels: &[String], output: &Option<PathBuf>) {
+    let path = output.as_ref()
+        .map(|p| format!("{}_rawsnp.csv", p.display()))
+        .unwrap_or_else(|| "rawsnp.csv".into());
+    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
+        eprintln!("error creating {path}: {e}");
+        std::process::exit(1);
+    }));
+    let n = labels.len();
+    write!(f, "genome").unwrap();
+    for g in labels { write!(f, ",{g}").unwrap(); }
+    writeln!(f).unwrap();
+    for (i, g) in labels.iter().enumerate() {
+        write!(f, "{g}").unwrap();
+        for j in 0..n {
+            let snp = result.snp[[i, j]];
+            let shared = result.shared[[i, j]];
+            let eligible = snp + shared;
+            if eligible == 0 {
+                write!(f, ",NA").unwrap();
+            } else {
+                write!(f, ",{:.6}", snp as f64 / eligible as f64).unwrap();
+            }
+        }
+        writeln!(f).unwrap();
+    }
+    info!("raw single-copy SNP distance matrix → {path}");
+}
+
+// ── Raw single-copy SNP distance → per-pair diagnostic counts ──────────────
+//
+// A pair table (one row per unordered genome pair), not a matrix: the ratio
+// alone can't distinguish "identical across every eligible locus" from
+// "almost no eligible locus at all" — both can read `0.0`/`NA` in
+// `--raw-snp-distance`'s output. Distinguishing them matters most exactly
+// where it's easy to miss: genome pairs near the edge of what
+// central-position families can resolve at all (deep cross-lineage splits,
+// see `docmd/theory/evolutionary_distances.md`, "Run 3" and the later
+// IQ-TREE/Mash comparison — a `ratio=0.0` backed by 2 eligible loci is not
+// the same claim as one backed by 2000).
+
+pub(super) fn write_raw_snp_counts_csv(result: &RawSnpDistanceOutput, labels: &[String], output: &Option<PathBuf>) {
+    let path = output.as_ref()
+        .map(|p| format!("{}_rawsnp_counts.csv", p.display()))
+        .unwrap_or_else(|| "rawsnp_counts.csv".into());
+    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
+        eprintln!("error creating {path}: {e}");
+        std::process::exit(1);
+    }));
+    let n = labels.len();
+    writeln!(f, "genome_a,genome_b,n_snp,n_shared,n_eligible,ratio").unwrap();
+    for i in 0..n {
+        for j in (i + 1)..n {
+            let snp = result.snp[[i, j]];
+            let shared = result.shared[[i, j]];
+            let eligible = snp + shared;
+            write!(f, "{},{},{snp},{shared},{eligible}", labels[i], labels[j]).unwrap();
+            if eligible == 0 {
+                writeln!(f, ",NA").unwrap();
+            } else {
+                writeln!(f, ",{:.6}", snp as f64 / eligible as f64).unwrap();
+            }
+        }
+    }
+    info!("raw single-copy SNP distance counts (diagnostic) → {path}");
+}
+
+// ── SNP-only pseudo-alignment → FASTA ───────────────────────────────────────
+//
+// One record per genome, IUPAC-coded, no flanking sequence — see
+// `SnpAlignment` / `KmerIndex::snp_pseudo_alignment`. Uses the project's
+// existing FASTA writer (`obifastwrite::write_record`) rather than
+// hand-rolling one.
+
+pub(super) fn write_snp_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
+    let path = output.as_ref()
+        .map(|p| format!("{}_snp.fasta", p.display()))
+        .unwrap_or_else(|| "snp.fasta".into());
+    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
+        eprintln!("error creating {path}: {e}");
+        std::process::exit(1);
+    }));
+    let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
+    for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
+        write_record(seq, label, &[("n_sites", JsonVal::Num(n_sites as u64))], &mut f).unwrap_or_else(|e| {
+            eprintln!("error writing {path}: {e}");
+            std::process::exit(1);
+        });
+    }
+    info!("SNP pseudo-alignment → {path} ({n_sites} site{})",
+        if n_sites == 1 { "" } else { "s" });
+}
+
+// ── UPGMA Newick from kodama dendrogram ───────────────────────────────────────
+
+pub(super) fn upgma_to_newick(dendro: &kodama::Dendrogram<f64>, names: &[String]) -> String {
+    let n = names.len();
+    // node_labels[i]: Newick subtree string for node i (leaves 0..n, internals n..)
+    let mut labels: Vec<String> = names.to_vec();
+    // height of each node: leaves = 0, internal = dissimilarity/2
+    let mut heights: Vec<f64> = vec![0.0; 2 * n - 1];
+
+    for (k, step) in dendro.steps().iter().enumerate() {
+        let new_node = n + k;
+        let h = step.dissimilarity / 2.0;
+        heights[new_node] = h;
+        let c1 = step.cluster1;
+        let c2 = step.cluster2;
+        let bl1 = (h - heights[c1]).max(0.0);
+        let bl2 = (h - heights[c2]).max(0.0);
+        labels.push(format!(
+            "({label1}:{bl1:.6},{label2}:{bl2:.6})",
+            label1 = labels[c1],
+            label2 = labels[c2],
+        ));
+    }
+
+    format!("{};", labels.last().unwrap())
+}

src/obikmer/src/cmd/phylo/sankoff.rs

@@ -11,9 +11,15 @@ use tracing::info;
 // `snp_pseudo_alignment`), re-coded so its symbols match the accompanying
 // `--sankoff-matrix` output exactly: `0` for the empty/absent state instead
 // of `-`, which TNT/PhyG would otherwise read as their own gap character
-// rather than our "family absent" state.
+// rather than our "family absent" state. Unless `free_loss` (`--free-loss`)
+// is set, in which case `∅` is recoded to `?` instead — TNT/PhyG's own
+// missing-data symbol, deliberately *not* `-` (still gap/indel semantics in
+// both tools) — so non-detection costs nothing rather than being scored as
+// an ordinary, calibrated state transition. See
+// `docmd/theory/evolutionary_distances.md`, "Locus dropout under incomplete
+// coverage".
 
-pub(super) fn write_sankoff_alignment_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
+pub(super) fn write_sankoff_alignment_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>, free_loss: bool) {
     let path = output.as_ref()
         .map(|p| format!("{}_sankoff.fasta", p.display()))
         .unwrap_or_else(|| "sankoff.fasta".into());
@@ -21,9 +27,10 @@ pub(super) fn write_sankoff_alignment_fasta(alignment: &SnpAlignment, labels: &[
         eprintln!("error creating {path}: {e}");
         std::process::exit(1);
     }));
+    let absent_symbol = if free_loss { b'?' } else { b'0' };
     let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
     for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
-        let recoded: Vec<u8> = seq.iter().map(|&b| if b == b'-' { b'0' } else { b }).collect();
+        let recoded: Vec<u8> = seq.iter().map(|&b| if b == b'-' { absent_symbol } else { b }).collect();
         write_record(&recoded, label, &[("n_sites", JsonVal::Num(n_sites as u64))], &mut f).unwrap_or_else(|e| {
             eprintln!("error writing {path}: {e}");
             std::process::exit(1);

src/obikmer/src/cmd/phylo/tnt.rs

@@ -28,6 +28,7 @@ pub(super) fn write_sankoff_tnt(
     labels: &[String],
     output: &Option<PathBuf>,
     cost_scale: f64,
+    free_loss: bool,
 ) {
     let path = output.as_ref()
         .map(|p| format!("{}_sankoff.tnt", p.display()))
@@ -49,6 +50,13 @@ pub(super) fn write_sankoff_tnt(
     for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
         write!(f, "{label} ").unwrap();
         for &b in seq {
+            if free_loss && b == b'-' {
+                // `?`: TNT's own missing-data symbol, read directly, not
+                // routed through `TNT_STATE_SYMBOL` (there is no state for
+                // it) — see `write_sankoff_alignment_fasta`'s doc comment.
+                write!(f, "?").unwrap();
+                continue;
+            }
             let b = if b == b'-' { b'0' } else { b };
             let state = iupac_to_state[b as usize];
             write!(f, "{}", TNT_STATE_SYMBOL[state as usize]).unwrap();