#!/usr/bin/env python3 """Family overlap and clade-closure ("nesting") of a SNP pseudo-alignment. Each alignment column is one family; a genome "carries a call" at a column when its character is not an absence symbol (default '-' and '?'). Outputs (prefix given by -o): _family_overlap.csv genome x genome matrix: number of columns where both genomes carry a call (diagonal = 0, same convention as `obikmer phylo --family-overlap`) _nesting.csv per clade-size stratum: how closed the families are under the tree Nesting measure: for a family carried by the genome set S, let C(S) be the smallest tree clade containing S. completeness = |S| / |C(S)|. A value of 1 means the family is clade-closed (every genome of the clade carries it); low values mean dropout or patchy conservation inside the clade. The tree is rooted as written in the Newick file unless --outgroup is given (rooted at the neighbour of that tip, which then becomes a child of the root). Optional lift test (--lift-test): do families covering a rare pair (A,B) also cover an abundant pair (A,C) more often than the margins alone imply? For each genome A, B ranges over its --lift-frac rarest partners and C over its most abundant ones (ranked on pair overlap). Statistics, on a random column subsample: T = mean over triples of O_ABC / O_AB = P(C | A,B) L = sum n_A * O_ABC / sum O_AB * O_AC (lift; 1 if B,C independent given A) Null: curveball swap randomisation of the genome x family presence matrix, which preserves every genome's family count and every family's size exactly. Each null replicate re-selects the rare/abundant partners from its own overlaps, so the selection procedure is identical to the observed one. Optional quartet test (--quartet-test): does flattening pair overlaps also flatten quartet coverage? --quartets N genome quartets are drawn uniformly; for each, the number of columns where all four genomes carry a call is counted exactly. Reported per stratum of the quartet's depth (size of the smallest tree clade containing the four genomes): count distribution, share of uncovered quartets, and the ratio of the 4-way count to the smallest of the six pair overlaps (1 = perfect nesting; the pair overlap is an upper bound of the 4-way count). Usage ----- pseudo_alignment_nesting.py ALIGNMENT.fasta TREEFILE -o prefix [--outgroup LABEL] [--absent '-?'] [--chunk 131072] [--quartet-test [--quartets 20000]] [--lift-test [--lift-columns 200000] [--lift-frac 0.1] [--null-reps 20] [--trades 5000] [--seed N]] """ import argparse import sys import numpy as np def read_fasta_presence(path, absent): """Return (labels, packed presence bits per genome, alignment length).""" absent_lut = np.zeros(256, dtype=bool) for ch in absent: absent_lut[ord(ch)] = True labels, packed, length = [], [], None def flush(parts): nonlocal length seq = np.frombuffer(b"".join(parts), dtype=np.uint8) if length is None: length = len(seq) elif len(seq) != length: sys.exit(f"{labels[-1]}: length {len(seq)} != {length}") packed.append(np.packbits(~absent_lut[seq])) parts = None with open(path, "rb") as fh: for line in fh: if line.startswith(b">"): if parts is not None: flush(parts) labels.append(line[1:].split()[0].decode()) parts = [] else: parts.append(line.strip()) if parts is not None: flush(parts) return labels, np.vstack(packed), length def parse_newick(text): """Parse Newick into (names, parent, length) arrays; internal labels ignored.""" names, parent, length = [], [], [] stack, cur = [], None i, n = 0, len(text) def new_node(): names.append(None) parent.append(-1) length.append(0.0) return len(names) - 1 while i < n: c = text[i] if c in " \n\r\t": i += 1 elif c == "(": if stack: node = stack[-1] # the placeholder child becomes this internal node else: node = new_node() stack.append(node) cur = None i += 1 child = new_node() parent[child] = node stack.append(child) elif c == ",": stack.pop() child = new_node() parent[child] = stack[-1] stack.append(child) i += 1 elif c == ")": stack.pop() cur = stack[-1] i += 1 j = i while j < n and text[j] not in ":,);": j += 1 i = j # internal label (support values) discarded elif c == ":": j = i + 1 while j < n and text[j] not in ",);": j += 1 length[stack[-1] if cur is None else cur] = float(text[i + 1:j]) cur = None i = j elif c == ";": break else: j = i while j < n and text[j] not in ":,);": j += 1 names[stack[-1]] = text[i:j] i = j return names, parent, length def leaf_order_matrices(names, parent, length, outgroup): """DFS leaf order plus LCA-clade-size and patristic matrices in that order.""" adj = [[] for _ in names] for v, p in enumerate(parent): if p >= 0: adj[v].append((p, length[v])) adj[p].append((v, length[v])) root = next(v for v, p in enumerate(parent) if p < 0) if outgroup is not None: tip = names.index(outgroup) root = adj[tip][0][0] order, depth, children = [], {root: 0.0}, {} stack, seen = [root], {root} while stack: v = stack.pop() kids = [(u, w) for u, w in adj[v] if u not in seen] for u, w in kids: seen.add(u) depth[u] = depth[v] + w children[v] = [u for u, _ in kids] stack.extend(reversed(children[v])) order.append(v) leaves = [v for v in order if not children[v]] rank = {v: r for r, v in enumerate(leaves)} n = len(leaves) span = {} for v in reversed(order): span[v] = (rank[v], rank[v] + 1) if not children[v] else ( span[children[v][0]][0], span[children[v][-1]][1]) clade = np.ones((n, n), dtype=np.int32) lca_depth = np.zeros((n, n)) for v in order: kids = children[v] for a in range(len(kids)): for b in range(a + 1, len(kids)): (a0, a1), (b0, b1) = span[kids[a]], span[kids[b]] size = span[v][1] - span[v][0] clade[a0:a1, b0:b1] = clade[b0:b1, a0:a1] = size lca_depth[a0:a1, b0:b1] = lca_depth[b0:b1, a0:a1] = depth[v] leaf_depth = np.array([depth[v] for v in leaves]) patristic = leaf_depth[:, None] + leaf_depth[None, :] - 2 * lca_depth return [names[v] for v in leaves], clade, patristic def lift_stats(pres, frac): """(L, T) for the rare/abundant partner triples of every genome (see module doc).""" n = pres.shape[0] pf = pres.astype(np.float32) overlap = pf @ pf.T k = max(1, round(frac * (n - 1))) num = den = ratio_sum = 0.0 n_ratio = 0 for a in range(n): order = [j for j in np.argsort(overlap[a], kind="stable") if j != a] rare, abundant = order[:k], order[-k:] cols = pres[a] n_a = float(cols.sum()) if n_a == 0: continue o_abc = pf[rare][:, cols] @ pf[abundant][:, cols].T o_ab, o_ac = overlap[a, rare].astype(np.float64), overlap[a, abundant].astype(np.float64) num += n_a * float(o_abc.sum()) den += float((o_ab[:, None] * o_ac[None, :]).sum()) ok = o_ab > 0 ratio_sum += float((o_abc[ok] / o_ab[ok, None]).sum()) n_ratio += int(ok.sum()) * k return num / den, ratio_sum / n_ratio def curveball(pres, trades, rng): """In-place curveball trades: preserve every row sum and every column sum.""" n = pres.shape[0] for _ in range(trades): i, j = rng.choice(n, size=2, replace=False) a, b = pres[i], pres[j] diff = np.flatnonzero(a ^ b) k = int((a & ~b).sum()) diff = rng.permutation(diff) a[diff] = False b[diff] = False a[diff[:k]] = True b[diff[k:]] = True def quartet_test(packed, overlap, tree_rank, clade, n_quartets, bin_edges, rng): """Exact 4-way co-presence counts for random genome quartets. Returns (counts, ratio_to_min_pair, depth_stratum), one entry per quartet. """ n = packed.shape[0] quartets = np.argpartition(rng.random((n_quartets, n)), 3, axis=1)[:, :4] counts = np.zeros(n_quartets, dtype=np.int64) block = 2048 # bytes of packed columns per pass: bounds the (Q, block) gathers for a in range(0, packed.shape[1], block): blk = packed[:, a:a + block] acc = blk[quartets[:, 0]] & blk[quartets[:, 1]] acc &= blk[quartets[:, 2]] acc &= blk[quartets[:, 3]] counts += np.bitwise_count(acc).sum(axis=1, dtype=np.int64) pair_min = np.full(n_quartets, np.iinfo(np.int64).max) for x in range(4): for y in range(x + 1, 4): pair_min = np.minimum(pair_min, overlap[quartets[:, x], quartets[:, y]]) ranks = tree_rank[quartets] csize = clade[ranks.min(axis=1), ranks.max(axis=1)] stratum = np.searchsorted(bin_edges, csize, side="right") - 1 return counts, counts / np.maximum(pair_min, 1), stratum def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("alignment") ap.add_argument("treefile") ap.add_argument("-o", "--output", required=True, help="output file prefix") ap.add_argument("--outgroup", help="tip label to root the tree on") ap.add_argument("--absent", default="-?", help="absence symbols (default '-?')") ap.add_argument("--chunk", type=int, default=131072, help="columns per chunk, multiple of 8 (default 131072)") ap.add_argument("--quartet-test", action="store_true", help="measure exact 4-way coverage of random genome quartets") ap.add_argument("--quartets", type=int, default=20000, help="number of random quartets (default 20000)") ap.add_argument("--lift-test", action="store_true", help="run the margin-preserving lift test (see module doc)") ap.add_argument("--lift-columns", type=int, default=200000, help="random columns used by the lift test (default 200000)") ap.add_argument("--lift-frac", type=float, default=0.1, help="fraction of partners ranked rare / abundant (default 0.1)") ap.add_argument("--null-reps", type=int, default=20) ap.add_argument("--trades", type=int, default=5000, help="curveball trades between replicates, also burn-in (default 5000)") ap.add_argument("--seed", type=int, default=None) args = ap.parse_args() if args.chunk % 8: sys.exit("--chunk must be a multiple of 8") rng = np.random.default_rng(args.seed) labels, packed, length = read_fasta_presence(args.alignment, args.absent) keep_p = min(1.0, args.lift_columns / length) if args.lift_test else 0.0 sampled = [] with open(args.treefile) as fh: names, parent, blen = parse_newick(fh.read()) leaf_labels, clade, patristic = leaf_order_matrices(names, parent, blen, args.outgroup) if set(leaf_labels) != set(labels): sys.exit("tree tips and alignment records differ") n = len(labels) tree_row = np.array([labels.index(l) for l in leaf_labels]) # tree rank -> alignment row bin_edges = np.array([2, 3, 5, 9, 17, 33, 65, 129, n + 1]) # clade-size strata nb = len(bin_edges) - 1 cnt = np.zeros(nb, dtype=np.int64) closed = np.zeros(nb, dtype=np.int64) sum_ratio = np.zeros(nb) sum_missing = np.zeros(nb) overlap = np.zeros((n, n), dtype=np.int64) n_cols_lt2 = 0 for a in range(0, length, args.chunk): b = min(a + args.chunk, length) pres = np.unpackbits(packed[:, a // 8:(b + 7) // 8], axis=1)[:, :b - a] if keep_p: sampled.append(pres[:, rng.random(b - a) < keep_p].astype(bool)) overlap += np.rint(pres.astype(np.float32) @ pres.astype(np.float32).T).astype(np.int64) t = pres[tree_row] size = t.sum(axis=0) keep = size >= 2 n_cols_lt2 += int((~keep).sum()) t, size = t[:, keep], size[keep] lo = t.argmax(axis=0) hi = n - 1 - t[::-1].argmax(axis=0) csize = clade[lo, hi] stratum = np.searchsorted(bin_edges, csize, side="right") - 1 cnt += np.bincount(stratum, minlength=nb) closed += np.bincount(stratum, weights=(size == csize), minlength=nb).astype(np.int64) sum_ratio += np.bincount(stratum, weights=size / csize, minlength=nb) sum_missing += np.bincount(stratum, weights=csize - size, minlength=nb) np.fill_diagonal(overlap, 0) with open(f"{args.output}_family_overlap.csv", "w") as fh: fh.write("genome," + ",".join(labels) + "\n") for l, row in zip(labels, overlap): fh.write(l + "," + ",".join(map(str, row)) + "\n") with open(f"{args.output}_nesting.csv", "w") as fh: fh.write("clade_size_min,clade_size_max,n_families,mean_completeness," "frac_clade_closed,mean_missing_in_clade\n") for k in range(nb): if cnt[k]: fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{cnt[k]}," f"{sum_ratio[k] / cnt[k]:.4f},{closed[k] / cnt[k]:.4f}," f"{sum_missing[k] / cnt[k]:.3f}\n") off = overlap[np.triu_indices(n, 1)] print(f"genomes={n} columns={length} (columns with <2 calls skipped for nesting: {n_cols_lt2})") print(f"overlap off-diagonal: min={off.min()} median={int(np.median(off))} " f"max={off.max()} CV={off.std() / off.mean():.3f}") tot = cnt.sum() print(f"clade-closed families: {closed.sum()}/{tot} ({closed.sum() / tot:.3f}); " f"mean completeness {sum_ratio.sum() / tot:.3f}") print(f"wrote {args.output}_family_overlap.csv and {args.output}_nesting.csv") if args.quartet_test: tree_rank = np.empty(n, dtype=np.int64) tree_rank[tree_row] = np.arange(n) counts, ratio, stratum = quartet_test( packed, overlap, tree_rank, clade, args.quartets, bin_edges, rng) print(f"quartet test: {args.quartets} random quartets over {length} columns") q = np.quantile(counts, [0, .01, .1, .5, .9, .99, 1]) print("4-way count quantiles 0/1/10/50/90/99/100 %: " + " ".join(f"{v:.0f}" for v in q) + f"; uncovered: {(counts == 0).mean():.4f}; CV={counts.std() / max(counts.mean(), 1e-9):.2f}") with open(f"{args.output}_quartets.csv", "w") as fh: fh.write("depth_clade_min,depth_clade_max,n_quartets,median_count,q10_count," "frac_uncovered,median_ratio_to_min_pair\n") for k in range(nb): m = stratum == k if m.any(): c = counts[m] fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{int(m.sum())},{np.median(c):.0f}," f"{np.quantile(c, .1):.0f},{(c == 0).mean():.4f},{np.median(ratio[m]):.4f}\n") print(f"wrote {args.output}_quartets.csv") if args.lift_test: pres = np.ascontiguousarray(np.hstack(sampled)) print(f"lift test on {pres.shape[1]} columns, {args.null_reps} null replicates") rows = [("observed",) + lift_stats(pres, args.lift_frac)] curveball(pres, args.trades, rng) # burn-in for r in range(args.null_reps): rows.append((f"null{r + 1}",) + lift_stats(pres, args.lift_frac)) curveball(pres, args.trades, rng) with open(f"{args.output}_lift_test.csv", "w") as fh: fh.write("replicate,L,T\n") for name, l, t in rows: fh.write(f"{name},{l:.6f},{t:.6f}\n") for idx, label in ((1, "L (lift)"), (2, "T = P(C|A,B)")): obs = rows[0][idx] null = np.array([r[idx] for r in rows[1:]]) p = (1 + (null >= obs).sum()) / (1 + len(null)) print(f"{label}: observed={obs:.4f} null mean={null.mean():.4f} " f"sd={null.std(ddof=1):.4f} z={(obs - null.mean()) / null.std(ddof=1):.1f} " f"p(null>=obs)={p:.3f}") print(f"wrote {args.output}_lift_test.csv") if __name__ == "__main__": main()