Files
obikmer/scripts/pseudo_alignment_nesting.py
T

395 lines
17 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
"""Family overlap and clade-closure ("nesting") of a SNP pseudo-alignment.
Each alignment column is one family; a genome "carries a call" at a column
when its character is not an absence symbol (default '-' and '?').
Outputs (prefix given by -o):
<prefix>_family_overlap.csv genome x genome matrix: number of columns where
both genomes carry a call (diagonal = 0, same
convention as `obikmer phylo --family-overlap`)
<prefix>_nesting.csv per clade-size stratum: how closed the families
are under the tree
Nesting measure: for a family carried by the genome set S, let C(S) be the
smallest tree clade containing S. completeness = |S| / |C(S)|. A value of 1
means the family is clade-closed (every genome of the clade carries it); low
values mean dropout or patchy conservation inside the clade.
The tree is rooted as written in the Newick file unless --outgroup is given
(rooted at the neighbour of that tip, which then becomes a child of the root).
Optional lift test (--lift-test): do families covering a rare pair (A,B) also
cover an abundant pair (A,C) more often than the margins alone imply? For each
genome A, B ranges over its --lift-frac rarest partners and C over its most
abundant ones (ranked on pair overlap). Statistics, on a random column subsample:
T = mean over triples of O_ABC / O_AB = P(C | A,B)
L = sum n_A * O_ABC / sum O_AB * O_AC (lift; 1 if B,C independent given A)
Null: curveball swap randomisation of the genome x family presence matrix, which
preserves every genome's family count and every family's size exactly. Each null
replicate re-selects the rare/abundant partners from its own overlaps, so the
selection procedure is identical to the observed one.
Optional quartet test (--quartet-test): does flattening pair overlaps also flatten
quartet coverage? --quartets N genome quartets are drawn uniformly; for each, the
number of columns where all four genomes carry a call is counted exactly. Reported
per stratum of the quartet's depth (size of the smallest tree clade containing the
four genomes): count distribution, share of uncovered quartets, and the ratio of the
4-way count to the smallest of the six pair overlaps (1 = perfect nesting; the pair
overlap is an upper bound of the 4-way count).
Usage
-----
pseudo_alignment_nesting.py ALIGNMENT.fasta TREEFILE -o prefix
[--outgroup LABEL] [--absent '-?'] [--chunk 131072]
[--quartet-test [--quartets 20000]]
[--lift-test [--lift-columns 200000] [--lift-frac 0.1]
[--null-reps 20] [--trades 5000] [--seed N]]
"""
import argparse
import sys
import numpy as np
def read_fasta_presence(path, absent):
"""Return (labels, packed presence bits per genome, alignment length)."""
absent_lut = np.zeros(256, dtype=bool)
for ch in absent:
absent_lut[ord(ch)] = True
labels, packed, length = [], [], None
def flush(parts):
nonlocal length
seq = np.frombuffer(b"".join(parts), dtype=np.uint8)
if length is None:
length = len(seq)
elif len(seq) != length:
sys.exit(f"{labels[-1]}: length {len(seq)} != {length}")
packed.append(np.packbits(~absent_lut[seq]))
parts = None
with open(path, "rb") as fh:
for line in fh:
if line.startswith(b">"):
if parts is not None:
flush(parts)
labels.append(line[1:].split()[0].decode())
parts = []
else:
parts.append(line.strip())
if parts is not None:
flush(parts)
return labels, np.vstack(packed), length
def parse_newick(text):
"""Parse Newick into (names, parent, length) arrays; internal labels ignored."""
names, parent, length = [], [], []
stack, cur = [], None
i, n = 0, len(text)
def new_node():
names.append(None)
parent.append(-1)
length.append(0.0)
return len(names) - 1
while i < n:
c = text[i]
if c in " \n\r\t":
i += 1
elif c == "(":
if stack:
node = stack[-1] # the placeholder child becomes this internal node
else:
node = new_node()
stack.append(node)
cur = None
i += 1
child = new_node()
parent[child] = node
stack.append(child)
elif c == ",":
stack.pop()
child = new_node()
parent[child] = stack[-1]
stack.append(child)
i += 1
elif c == ")":
stack.pop()
cur = stack[-1]
i += 1
j = i
while j < n and text[j] not in ":,);":
j += 1
i = j # internal label (support values) discarded
elif c == ":":
j = i + 1
while j < n and text[j] not in ",);":
j += 1
length[stack[-1] if cur is None else cur] = float(text[i + 1:j])
cur = None
i = j
elif c == ";":
break
else:
j = i
while j < n and text[j] not in ":,);":
j += 1
names[stack[-1]] = text[i:j]
i = j
return names, parent, length
def leaf_order_matrices(names, parent, length, outgroup):
"""DFS leaf order plus LCA-clade-size and patristic matrices in that order."""
adj = [[] for _ in names]
for v, p in enumerate(parent):
if p >= 0:
adj[v].append((p, length[v]))
adj[p].append((v, length[v]))
root = next(v for v, p in enumerate(parent) if p < 0)
if outgroup is not None:
tip = names.index(outgroup)
root = adj[tip][0][0]
order, depth, children = [], {root: 0.0}, {}
stack, seen = [root], {root}
while stack:
v = stack.pop()
kids = [(u, w) for u, w in adj[v] if u not in seen]
for u, w in kids:
seen.add(u)
depth[u] = depth[v] + w
children[v] = [u for u, _ in kids]
stack.extend(reversed(children[v]))
order.append(v)
leaves = [v for v in order if not children[v]]
rank = {v: r for r, v in enumerate(leaves)}
n = len(leaves)
span = {}
for v in reversed(order):
span[v] = (rank[v], rank[v] + 1) if not children[v] else (
span[children[v][0]][0], span[children[v][-1]][1])
clade = np.ones((n, n), dtype=np.int32)
lca_depth = np.zeros((n, n))
for v in order:
kids = children[v]
for a in range(len(kids)):
for b in range(a + 1, len(kids)):
(a0, a1), (b0, b1) = span[kids[a]], span[kids[b]]
size = span[v][1] - span[v][0]
clade[a0:a1, b0:b1] = clade[b0:b1, a0:a1] = size
lca_depth[a0:a1, b0:b1] = lca_depth[b0:b1, a0:a1] = depth[v]
leaf_depth = np.array([depth[v] for v in leaves])
patristic = leaf_depth[:, None] + leaf_depth[None, :] - 2 * lca_depth
return [names[v] for v in leaves], clade, patristic
def lift_stats(pres, frac):
"""(L, T) for the rare/abundant partner triples of every genome (see module doc)."""
n = pres.shape[0]
pf = pres.astype(np.float32)
overlap = pf @ pf.T
k = max(1, round(frac * (n - 1)))
num = den = ratio_sum = 0.0
n_ratio = 0
for a in range(n):
order = [j for j in np.argsort(overlap[a], kind="stable") if j != a]
rare, abundant = order[:k], order[-k:]
cols = pres[a]
n_a = float(cols.sum())
if n_a == 0:
continue
o_abc = pf[rare][:, cols] @ pf[abundant][:, cols].T
o_ab, o_ac = overlap[a, rare].astype(np.float64), overlap[a, abundant].astype(np.float64)
num += n_a * float(o_abc.sum())
den += float((o_ab[:, None] * o_ac[None, :]).sum())
ok = o_ab > 0
ratio_sum += float((o_abc[ok] / o_ab[ok, None]).sum())
n_ratio += int(ok.sum()) * k
return num / den, ratio_sum / n_ratio
def curveball(pres, trades, rng):
"""In-place curveball trades: preserve every row sum and every column sum."""
n = pres.shape[0]
for _ in range(trades):
i, j = rng.choice(n, size=2, replace=False)
a, b = pres[i], pres[j]
diff = np.flatnonzero(a ^ b)
k = int((a & ~b).sum())
diff = rng.permutation(diff)
a[diff] = False
b[diff] = False
a[diff[:k]] = True
b[diff[k:]] = True
def quartet_test(packed, overlap, tree_rank, clade, n_quartets, bin_edges, rng):
"""Exact 4-way co-presence counts for random genome quartets.
Returns (counts, ratio_to_min_pair, depth_stratum), one entry per quartet.
"""
n = packed.shape[0]
quartets = np.argpartition(rng.random((n_quartets, n)), 3, axis=1)[:, :4]
counts = np.zeros(n_quartets, dtype=np.int64)
block = 2048 # bytes of packed columns per pass: bounds the (Q, block) gathers
for a in range(0, packed.shape[1], block):
blk = packed[:, a:a + block]
acc = blk[quartets[:, 0]] & blk[quartets[:, 1]]
acc &= blk[quartets[:, 2]]
acc &= blk[quartets[:, 3]]
counts += np.bitwise_count(acc).sum(axis=1, dtype=np.int64)
pair_min = np.full(n_quartets, np.iinfo(np.int64).max)
for x in range(4):
for y in range(x + 1, 4):
pair_min = np.minimum(pair_min, overlap[quartets[:, x], quartets[:, y]])
ranks = tree_rank[quartets]
csize = clade[ranks.min(axis=1), ranks.max(axis=1)]
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
return counts, counts / np.maximum(pair_min, 1), stratum
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("alignment")
ap.add_argument("treefile")
ap.add_argument("-o", "--output", required=True, help="output file prefix")
ap.add_argument("--outgroup", help="tip label to root the tree on")
ap.add_argument("--absent", default="-?", help="absence symbols (default '-?')")
ap.add_argument("--chunk", type=int, default=131072,
help="columns per chunk, multiple of 8 (default 131072)")
ap.add_argument("--quartet-test", action="store_true",
help="measure exact 4-way coverage of random genome quartets")
ap.add_argument("--quartets", type=int, default=20000,
help="number of random quartets (default 20000)")
ap.add_argument("--lift-test", action="store_true",
help="run the margin-preserving lift test (see module doc)")
ap.add_argument("--lift-columns", type=int, default=200000,
help="random columns used by the lift test (default 200000)")
ap.add_argument("--lift-frac", type=float, default=0.1,
help="fraction of partners ranked rare / abundant (default 0.1)")
ap.add_argument("--null-reps", type=int, default=20)
ap.add_argument("--trades", type=int, default=5000,
help="curveball trades between replicates, also burn-in (default 5000)")
ap.add_argument("--seed", type=int, default=None)
args = ap.parse_args()
if args.chunk % 8:
sys.exit("--chunk must be a multiple of 8")
rng = np.random.default_rng(args.seed)
labels, packed, length = read_fasta_presence(args.alignment, args.absent)
keep_p = min(1.0, args.lift_columns / length) if args.lift_test else 0.0
sampled = []
with open(args.treefile) as fh:
names, parent, blen = parse_newick(fh.read())
leaf_labels, clade, patristic = leaf_order_matrices(names, parent, blen, args.outgroup)
if set(leaf_labels) != set(labels):
sys.exit("tree tips and alignment records differ")
n = len(labels)
tree_row = np.array([labels.index(l) for l in leaf_labels]) # tree rank -> alignment row
bin_edges = np.array([2, 3, 5, 9, 17, 33, 65, 129, n + 1]) # clade-size strata
nb = len(bin_edges) - 1
cnt = np.zeros(nb, dtype=np.int64)
closed = np.zeros(nb, dtype=np.int64)
sum_ratio = np.zeros(nb)
sum_missing = np.zeros(nb)
overlap = np.zeros((n, n), dtype=np.int64)
n_cols_lt2 = 0
for a in range(0, length, args.chunk):
b = min(a + args.chunk, length)
pres = np.unpackbits(packed[:, a // 8:(b + 7) // 8], axis=1)[:, :b - a]
if keep_p:
sampled.append(pres[:, rng.random(b - a) < keep_p].astype(bool))
overlap += np.rint(pres.astype(np.float32) @ pres.astype(np.float32).T).astype(np.int64)
t = pres[tree_row]
size = t.sum(axis=0)
keep = size >= 2
n_cols_lt2 += int((~keep).sum())
t, size = t[:, keep], size[keep]
lo = t.argmax(axis=0)
hi = n - 1 - t[::-1].argmax(axis=0)
csize = clade[lo, hi]
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
cnt += np.bincount(stratum, minlength=nb)
closed += np.bincount(stratum, weights=(size == csize), minlength=nb).astype(np.int64)
sum_ratio += np.bincount(stratum, weights=size / csize, minlength=nb)
sum_missing += np.bincount(stratum, weights=csize - size, minlength=nb)
np.fill_diagonal(overlap, 0)
with open(f"{args.output}_family_overlap.csv", "w") as fh:
fh.write("genome," + ",".join(labels) + "\n")
for l, row in zip(labels, overlap):
fh.write(l + "," + ",".join(map(str, row)) + "\n")
with open(f"{args.output}_nesting.csv", "w") as fh:
fh.write("clade_size_min,clade_size_max,n_families,mean_completeness,"
"frac_clade_closed,mean_missing_in_clade\n")
for k in range(nb):
if cnt[k]:
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{cnt[k]},"
f"{sum_ratio[k] / cnt[k]:.4f},{closed[k] / cnt[k]:.4f},"
f"{sum_missing[k] / cnt[k]:.3f}\n")
off = overlap[np.triu_indices(n, 1)]
print(f"genomes={n} columns={length} (columns with <2 calls skipped for nesting: {n_cols_lt2})")
print(f"overlap off-diagonal: min={off.min()} median={int(np.median(off))} "
f"max={off.max()} CV={off.std() / off.mean():.3f}")
tot = cnt.sum()
print(f"clade-closed families: {closed.sum()}/{tot} ({closed.sum() / tot:.3f}); "
f"mean completeness {sum_ratio.sum() / tot:.3f}")
print(f"wrote {args.output}_family_overlap.csv and {args.output}_nesting.csv")
if args.quartet_test:
tree_rank = np.empty(n, dtype=np.int64)
tree_rank[tree_row] = np.arange(n)
counts, ratio, stratum = quartet_test(
packed, overlap, tree_rank, clade, args.quartets, bin_edges, rng)
print(f"quartet test: {args.quartets} random quartets over {length} columns")
q = np.quantile(counts, [0, .01, .1, .5, .9, .99, 1])
print("4-way count quantiles 0/1/10/50/90/99/100 %: " + " ".join(f"{v:.0f}" for v in q)
+ f"; uncovered: {(counts == 0).mean():.4f}; CV={counts.std() / max(counts.mean(), 1e-9):.2f}")
with open(f"{args.output}_quartets.csv", "w") as fh:
fh.write("depth_clade_min,depth_clade_max,n_quartets,median_count,q10_count,"
"frac_uncovered,median_ratio_to_min_pair\n")
for k in range(nb):
m = stratum == k
if m.any():
c = counts[m]
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{int(m.sum())},{np.median(c):.0f},"
f"{np.quantile(c, .1):.0f},{(c == 0).mean():.4f},{np.median(ratio[m]):.4f}\n")
print(f"wrote {args.output}_quartets.csv")
if args.lift_test:
pres = np.ascontiguousarray(np.hstack(sampled))
print(f"lift test on {pres.shape[1]} columns, {args.null_reps} null replicates")
rows = [("observed",) + lift_stats(pres, args.lift_frac)]
curveball(pres, args.trades, rng) # burn-in
for r in range(args.null_reps):
rows.append((f"null{r + 1}",) + lift_stats(pres, args.lift_frac))
curveball(pres, args.trades, rng)
with open(f"{args.output}_lift_test.csv", "w") as fh:
fh.write("replicate,L,T\n")
for name, l, t in rows:
fh.write(f"{name},{l:.6f},{t:.6f}\n")
for idx, label in ((1, "L (lift)"), (2, "T = P(C|A,B)")):
obs = rows[0][idx]
null = np.array([r[idx] for r in rows[1:]])
p = (1 + (null >= obs).sum()) / (1 + len(null))
print(f"{label}: observed={obs:.4f} null mean={null.mean():.4f} "
f"sd={null.std(ddof=1):.4f} z={(obs - null.mean()) / null.std(ddof=1):.1f} "
f"p(null>=obs)={p:.3f}")
print(f"wrote {args.output}_lift_test.csv")
if __name__ == "__main__":
main()