395 lines
17 KiB
Python
395 lines
17 KiB
Python
#!/usr/bin/env python3
|
|||
|
|
"""Family overlap and clade-closure ("nesting") of a SNP pseudo-alignment.
|
||
|
|
|
||
|
|
Each alignment column is one family; a genome "carries a call" at a column
|
||
|
|
when its character is not an absence symbol (default '-' and '?').
|
||
|
|
|
||
|
|
Outputs (prefix given by -o):
|
||
|
|
<prefix>_family_overlap.csv genome x genome matrix: number of columns where
|
||
|
|
both genomes carry a call (diagonal = 0, same
|
||
|
|
convention as `obikmer phylo --family-overlap`)
|
||
|
|
<prefix>_nesting.csv per clade-size stratum: how closed the families
|
||
|
|
are under the tree
|
||
|
|
|
||
|
|
Nesting measure: for a family carried by the genome set S, let C(S) be the
|
||
|
|
smallest tree clade containing S. completeness = |S| / |C(S)|. A value of 1
|
||
|
|
means the family is clade-closed (every genome of the clade carries it); low
|
||
|
|
values mean dropout or patchy conservation inside the clade.
|
||
|
|
|
||
|
|
The tree is rooted as written in the Newick file unless --outgroup is given
|
||
|
|
(rooted at the neighbour of that tip, which then becomes a child of the root).
|
||
|
|
|
||
|
|
Optional lift test (--lift-test): do families covering a rare pair (A,B) also
|
||
|
|
cover an abundant pair (A,C) more often than the margins alone imply? For each
|
||
|
|
genome A, B ranges over its --lift-frac rarest partners and C over its most
|
||
|
|
abundant ones (ranked on pair overlap). Statistics, on a random column subsample:
|
||
|
|
T = mean over triples of O_ABC / O_AB = P(C | A,B)
|
||
|
|
L = sum n_A * O_ABC / sum O_AB * O_AC (lift; 1 if B,C independent given A)
|
||
|
|
Null: curveball swap randomisation of the genome x family presence matrix, which
|
||
|
|
preserves every genome's family count and every family's size exactly. Each null
|
||
|
|
replicate re-selects the rare/abundant partners from its own overlaps, so the
|
||
|
|
selection procedure is identical to the observed one.
|
||
|
|
|
||
|
|
Optional quartet test (--quartet-test): does flattening pair overlaps also flatten
|
||
|
|
quartet coverage? --quartets N genome quartets are drawn uniformly; for each, the
|
||
|
|
number of columns where all four genomes carry a call is counted exactly. Reported
|
||
|
|
per stratum of the quartet's depth (size of the smallest tree clade containing the
|
||
|
|
four genomes): count distribution, share of uncovered quartets, and the ratio of the
|
||
|
|
4-way count to the smallest of the six pair overlaps (1 = perfect nesting; the pair
|
||
|
|
overlap is an upper bound of the 4-way count).
|
||
|
|
|
||
|
|
Usage
|
||
|
|
-----
|
||
|
|
pseudo_alignment_nesting.py ALIGNMENT.fasta TREEFILE -o prefix
|
||
|
|
[--outgroup LABEL] [--absent '-?'] [--chunk 131072]
|
||
|
|
[--quartet-test [--quartets 20000]]
|
||
|
|
[--lift-test [--lift-columns 200000] [--lift-frac 0.1]
|
||
|
|
[--null-reps 20] [--trades 5000] [--seed N]]
|
||
|
|
"""
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import sys
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
|
||
|
|
def read_fasta_presence(path, absent):
|
||
|
|
"""Return (labels, packed presence bits per genome, alignment length)."""
|
||
|
|
absent_lut = np.zeros(256, dtype=bool)
|
||
|
|
for ch in absent:
|
||
|
|
absent_lut[ord(ch)] = True
|
||
|
|
labels, packed, length = [], [], None
|
||
|
|
|
||
|
|
def flush(parts):
|
||
|
|
nonlocal length
|
||
|
|
seq = np.frombuffer(b"".join(parts), dtype=np.uint8)
|
||
|
|
if length is None:
|
||
|
|
length = len(seq)
|
||
|
|
elif len(seq) != length:
|
||
|
|
sys.exit(f"{labels[-1]}: length {len(seq)} != {length}")
|
||
|
|
packed.append(np.packbits(~absent_lut[seq]))
|
||
|
|
|
||
|
|
parts = None
|
||
|
|
with open(path, "rb") as fh:
|
||
|
|
for line in fh:
|
||
|
|
if line.startswith(b">"):
|
||
|
|
if parts is not None:
|
||
|
|
flush(parts)
|
||
|
|
labels.append(line[1:].split()[0].decode())
|
||
|
|
parts = []
|
||
|
|
else:
|
||
|
|
parts.append(line.strip())
|
||
|
|
if parts is not None:
|
||
|
|
flush(parts)
|
||
|
|
return labels, np.vstack(packed), length
|
||
|
|
|
||
|
|
|
||
|
|
def parse_newick(text):
|
||
|
|
"""Parse Newick into (names, parent, length) arrays; internal labels ignored."""
|
||
|
|
names, parent, length = [], [], []
|
||
|
|
stack, cur = [], None
|
||
|
|
i, n = 0, len(text)
|
||
|
|
|
||
|
|
def new_node():
|
||
|
|
names.append(None)
|
||
|
|
parent.append(-1)
|
||
|
|
length.append(0.0)
|
||
|
|
return len(names) - 1
|
||
|
|
|
||
|
|
while i < n:
|
||
|
|
c = text[i]
|
||
|
|
if c in " \n\r\t":
|
||
|
|
i += 1
|
||
|
|
elif c == "(":
|
||
|
|
if stack:
|
||
|
|
node = stack[-1] # the placeholder child becomes this internal node
|
||
|
|
else:
|
||
|
|
node = new_node()
|
||
|
|
stack.append(node)
|
||
|
|
cur = None
|
||
|
|
i += 1
|
||
|
|
child = new_node()
|
||
|
|
parent[child] = node
|
||
|
|
stack.append(child)
|
||
|
|
elif c == ",":
|
||
|
|
stack.pop()
|
||
|
|
child = new_node()
|
||
|
|
parent[child] = stack[-1]
|
||
|
|
stack.append(child)
|
||
|
|
i += 1
|
||
|
|
elif c == ")":
|
||
|
|
stack.pop()
|
||
|
|
cur = stack[-1]
|
||
|
|
i += 1
|
||
|
|
j = i
|
||
|
|
while j < n and text[j] not in ":,);":
|
||
|
|
j += 1
|
||
|
|
i = j # internal label (support values) discarded
|
||
|
|
elif c == ":":
|
||
|
|
j = i + 1
|
||
|
|
while j < n and text[j] not in ",);":
|
||
|
|
j += 1
|
||
|
|
length[stack[-1] if cur is None else cur] = float(text[i + 1:j])
|
||
|
|
cur = None
|
||
|
|
i = j
|
||
|
|
elif c == ";":
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
j = i
|
||
|
|
while j < n and text[j] not in ":,);":
|
||
|
|
j += 1
|
||
|
|
names[stack[-1]] = text[i:j]
|
||
|
|
i = j
|
||
|
|
return names, parent, length
|
||
|
|
|
||
|
|
|
||
|
|
def leaf_order_matrices(names, parent, length, outgroup):
|
||
|
|
"""DFS leaf order plus LCA-clade-size and patristic matrices in that order."""
|
||
|
|
adj = [[] for _ in names]
|
||
|
|
for v, p in enumerate(parent):
|
||
|
|
if p >= 0:
|
||
|
|
adj[v].append((p, length[v]))
|
||
|
|
adj[p].append((v, length[v]))
|
||
|
|
root = next(v for v, p in enumerate(parent) if p < 0)
|
||
|
|
if outgroup is not None:
|
||
|
|
tip = names.index(outgroup)
|
||
|
|
root = adj[tip][0][0]
|
||
|
|
|
||
|
|
order, depth, children = [], {root: 0.0}, {}
|
||
|
|
stack, seen = [root], {root}
|
||
|
|
while stack:
|
||
|
|
v = stack.pop()
|
||
|
|
kids = [(u, w) for u, w in adj[v] if u not in seen]
|
||
|
|
for u, w in kids:
|
||
|
|
seen.add(u)
|
||
|
|
depth[u] = depth[v] + w
|
||
|
|
children[v] = [u for u, _ in kids]
|
||
|
|
stack.extend(reversed(children[v]))
|
||
|
|
order.append(v)
|
||
|
|
|
||
|
|
leaves = [v for v in order if not children[v]]
|
||
|
|
rank = {v: r for r, v in enumerate(leaves)}
|
||
|
|
n = len(leaves)
|
||
|
|
span = {}
|
||
|
|
for v in reversed(order):
|
||
|
|
span[v] = (rank[v], rank[v] + 1) if not children[v] else (
|
||
|
|
span[children[v][0]][0], span[children[v][-1]][1])
|
||
|
|
|
||
|
|
clade = np.ones((n, n), dtype=np.int32)
|
||
|
|
lca_depth = np.zeros((n, n))
|
||
|
|
for v in order:
|
||
|
|
kids = children[v]
|
||
|
|
for a in range(len(kids)):
|
||
|
|
for b in range(a + 1, len(kids)):
|
||
|
|
(a0, a1), (b0, b1) = span[kids[a]], span[kids[b]]
|
||
|
|
size = span[v][1] - span[v][0]
|
||
|
|
clade[a0:a1, b0:b1] = clade[b0:b1, a0:a1] = size
|
||
|
|
lca_depth[a0:a1, b0:b1] = lca_depth[b0:b1, a0:a1] = depth[v]
|
||
|
|
leaf_depth = np.array([depth[v] for v in leaves])
|
||
|
|
patristic = leaf_depth[:, None] + leaf_depth[None, :] - 2 * lca_depth
|
||
|
|
return [names[v] for v in leaves], clade, patristic
|
||
|
|
|
||
|
|
|
||
|
|
def lift_stats(pres, frac):
|
||
|
|
"""(L, T) for the rare/abundant partner triples of every genome (see module doc)."""
|
||
|
|
n = pres.shape[0]
|
||
|
|
pf = pres.astype(np.float32)
|
||
|
|
overlap = pf @ pf.T
|
||
|
|
k = max(1, round(frac * (n - 1)))
|
||
|
|
num = den = ratio_sum = 0.0
|
||
|
|
n_ratio = 0
|
||
|
|
for a in range(n):
|
||
|
|
order = [j for j in np.argsort(overlap[a], kind="stable") if j != a]
|
||
|
|
rare, abundant = order[:k], order[-k:]
|
||
|
|
cols = pres[a]
|
||
|
|
n_a = float(cols.sum())
|
||
|
|
if n_a == 0:
|
||
|
|
continue
|
||
|
|
o_abc = pf[rare][:, cols] @ pf[abundant][:, cols].T
|
||
|
|
o_ab, o_ac = overlap[a, rare].astype(np.float64), overlap[a, abundant].astype(np.float64)
|
||
|
|
num += n_a * float(o_abc.sum())
|
||
|
|
den += float((o_ab[:, None] * o_ac[None, :]).sum())
|
||
|
|
ok = o_ab > 0
|
||
|
|
ratio_sum += float((o_abc[ok] / o_ab[ok, None]).sum())
|
||
|
|
n_ratio += int(ok.sum()) * k
|
||
|
|
return num / den, ratio_sum / n_ratio
|
||
|
|
|
||
|
|
|
||
|
|
def curveball(pres, trades, rng):
|
||
|
|
"""In-place curveball trades: preserve every row sum and every column sum."""
|
||
|
|
n = pres.shape[0]
|
||
|
|
for _ in range(trades):
|
||
|
|
i, j = rng.choice(n, size=2, replace=False)
|
||
|
|
a, b = pres[i], pres[j]
|
||
|
|
diff = np.flatnonzero(a ^ b)
|
||
|
|
k = int((a & ~b).sum())
|
||
|
|
diff = rng.permutation(diff)
|
||
|
|
a[diff] = False
|
||
|
|
b[diff] = False
|
||
|
|
a[diff[:k]] = True
|
||
|
|
b[diff[k:]] = True
|
||
|
|
|
||
|
|
|
||
|
|
def quartet_test(packed, overlap, tree_rank, clade, n_quartets, bin_edges, rng):
|
||
|
|
"""Exact 4-way co-presence counts for random genome quartets.
|
||
|
|
|
||
|
|
Returns (counts, ratio_to_min_pair, depth_stratum), one entry per quartet.
|
||
|
|
"""
|
||
|
|
n = packed.shape[0]
|
||
|
|
quartets = np.argpartition(rng.random((n_quartets, n)), 3, axis=1)[:, :4]
|
||
|
|
counts = np.zeros(n_quartets, dtype=np.int64)
|
||
|
|
block = 2048 # bytes of packed columns per pass: bounds the (Q, block) gathers
|
||
|
|
for a in range(0, packed.shape[1], block):
|
||
|
|
blk = packed[:, a:a + block]
|
||
|
|
acc = blk[quartets[:, 0]] & blk[quartets[:, 1]]
|
||
|
|
acc &= blk[quartets[:, 2]]
|
||
|
|
acc &= blk[quartets[:, 3]]
|
||
|
|
counts += np.bitwise_count(acc).sum(axis=1, dtype=np.int64)
|
||
|
|
|
||
|
|
pair_min = np.full(n_quartets, np.iinfo(np.int64).max)
|
||
|
|
for x in range(4):
|
||
|
|
for y in range(x + 1, 4):
|
||
|
|
pair_min = np.minimum(pair_min, overlap[quartets[:, x], quartets[:, y]])
|
||
|
|
ranks = tree_rank[quartets]
|
||
|
|
csize = clade[ranks.min(axis=1), ranks.max(axis=1)]
|
||
|
|
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
|
||
|
|
return counts, counts / np.maximum(pair_min, 1), stratum
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||
|
|
ap.add_argument("alignment")
|
||
|
|
ap.add_argument("treefile")
|
||
|
|
ap.add_argument("-o", "--output", required=True, help="output file prefix")
|
||
|
|
ap.add_argument("--outgroup", help="tip label to root the tree on")
|
||
|
|
ap.add_argument("--absent", default="-?", help="absence symbols (default '-?')")
|
||
|
|
ap.add_argument("--chunk", type=int, default=131072,
|
||
|
|
help="columns per chunk, multiple of 8 (default 131072)")
|
||
|
|
ap.add_argument("--quartet-test", action="store_true",
|
||
|
|
help="measure exact 4-way coverage of random genome quartets")
|
||
|
|
ap.add_argument("--quartets", type=int, default=20000,
|
||
|
|
help="number of random quartets (default 20000)")
|
||
|
|
ap.add_argument("--lift-test", action="store_true",
|
||
|
|
help="run the margin-preserving lift test (see module doc)")
|
||
|
|
ap.add_argument("--lift-columns", type=int, default=200000,
|
||
|
|
help="random columns used by the lift test (default 200000)")
|
||
|
|
ap.add_argument("--lift-frac", type=float, default=0.1,
|
||
|
|
help="fraction of partners ranked rare / abundant (default 0.1)")
|
||
|
|
ap.add_argument("--null-reps", type=int, default=20)
|
||
|
|
ap.add_argument("--trades", type=int, default=5000,
|
||
|
|
help="curveball trades between replicates, also burn-in (default 5000)")
|
||
|
|
ap.add_argument("--seed", type=int, default=None)
|
||
|
|
args = ap.parse_args()
|
||
|
|
if args.chunk % 8:
|
||
|
|
sys.exit("--chunk must be a multiple of 8")
|
||
|
|
|
||
|
|
rng = np.random.default_rng(args.seed)
|
||
|
|
labels, packed, length = read_fasta_presence(args.alignment, args.absent)
|
||
|
|
keep_p = min(1.0, args.lift_columns / length) if args.lift_test else 0.0
|
||
|
|
sampled = []
|
||
|
|
with open(args.treefile) as fh:
|
||
|
|
names, parent, blen = parse_newick(fh.read())
|
||
|
|
leaf_labels, clade, patristic = leaf_order_matrices(names, parent, blen, args.outgroup)
|
||
|
|
if set(leaf_labels) != set(labels):
|
||
|
|
sys.exit("tree tips and alignment records differ")
|
||
|
|
n = len(labels)
|
||
|
|
tree_row = np.array([labels.index(l) for l in leaf_labels]) # tree rank -> alignment row
|
||
|
|
|
||
|
|
bin_edges = np.array([2, 3, 5, 9, 17, 33, 65, 129, n + 1]) # clade-size strata
|
||
|
|
nb = len(bin_edges) - 1
|
||
|
|
cnt = np.zeros(nb, dtype=np.int64)
|
||
|
|
closed = np.zeros(nb, dtype=np.int64)
|
||
|
|
sum_ratio = np.zeros(nb)
|
||
|
|
sum_missing = np.zeros(nb)
|
||
|
|
overlap = np.zeros((n, n), dtype=np.int64)
|
||
|
|
n_cols_lt2 = 0
|
||
|
|
|
||
|
|
for a in range(0, length, args.chunk):
|
||
|
|
b = min(a + args.chunk, length)
|
||
|
|
pres = np.unpackbits(packed[:, a // 8:(b + 7) // 8], axis=1)[:, :b - a]
|
||
|
|
if keep_p:
|
||
|
|
sampled.append(pres[:, rng.random(b - a) < keep_p].astype(bool))
|
||
|
|
overlap += np.rint(pres.astype(np.float32) @ pres.astype(np.float32).T).astype(np.int64)
|
||
|
|
|
||
|
|
t = pres[tree_row]
|
||
|
|
size = t.sum(axis=0)
|
||
|
|
keep = size >= 2
|
||
|
|
n_cols_lt2 += int((~keep).sum())
|
||
|
|
t, size = t[:, keep], size[keep]
|
||
|
|
lo = t.argmax(axis=0)
|
||
|
|
hi = n - 1 - t[::-1].argmax(axis=0)
|
||
|
|
csize = clade[lo, hi]
|
||
|
|
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
|
||
|
|
cnt += np.bincount(stratum, minlength=nb)
|
||
|
|
closed += np.bincount(stratum, weights=(size == csize), minlength=nb).astype(np.int64)
|
||
|
|
sum_ratio += np.bincount(stratum, weights=size / csize, minlength=nb)
|
||
|
|
sum_missing += np.bincount(stratum, weights=csize - size, minlength=nb)
|
||
|
|
|
||
|
|
np.fill_diagonal(overlap, 0)
|
||
|
|
with open(f"{args.output}_family_overlap.csv", "w") as fh:
|
||
|
|
fh.write("genome," + ",".join(labels) + "\n")
|
||
|
|
for l, row in zip(labels, overlap):
|
||
|
|
fh.write(l + "," + ",".join(map(str, row)) + "\n")
|
||
|
|
|
||
|
|
with open(f"{args.output}_nesting.csv", "w") as fh:
|
||
|
|
fh.write("clade_size_min,clade_size_max,n_families,mean_completeness,"
|
||
|
|
"frac_clade_closed,mean_missing_in_clade\n")
|
||
|
|
for k in range(nb):
|
||
|
|
if cnt[k]:
|
||
|
|
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{cnt[k]},"
|
||
|
|
f"{sum_ratio[k] / cnt[k]:.4f},{closed[k] / cnt[k]:.4f},"
|
||
|
|
f"{sum_missing[k] / cnt[k]:.3f}\n")
|
||
|
|
|
||
|
|
off = overlap[np.triu_indices(n, 1)]
|
||
|
|
print(f"genomes={n} columns={length} (columns with <2 calls skipped for nesting: {n_cols_lt2})")
|
||
|
|
print(f"overlap off-diagonal: min={off.min()} median={int(np.median(off))} "
|
||
|
|
f"max={off.max()} CV={off.std() / off.mean():.3f}")
|
||
|
|
tot = cnt.sum()
|
||
|
|
print(f"clade-closed families: {closed.sum()}/{tot} ({closed.sum() / tot:.3f}); "
|
||
|
|
f"mean completeness {sum_ratio.sum() / tot:.3f}")
|
||
|
|
print(f"wrote {args.output}_family_overlap.csv and {args.output}_nesting.csv")
|
||
|
|
|
||
|
|
if args.quartet_test:
|
||
|
|
tree_rank = np.empty(n, dtype=np.int64)
|
||
|
|
tree_rank[tree_row] = np.arange(n)
|
||
|
|
counts, ratio, stratum = quartet_test(
|
||
|
|
packed, overlap, tree_rank, clade, args.quartets, bin_edges, rng)
|
||
|
|
print(f"quartet test: {args.quartets} random quartets over {length} columns")
|
||
|
|
q = np.quantile(counts, [0, .01, .1, .5, .9, .99, 1])
|
||
|
|
print("4-way count quantiles 0/1/10/50/90/99/100 %: " + " ".join(f"{v:.0f}" for v in q)
|
||
|
|
+ f"; uncovered: {(counts == 0).mean():.4f}; CV={counts.std() / max(counts.mean(), 1e-9):.2f}")
|
||
|
|
with open(f"{args.output}_quartets.csv", "w") as fh:
|
||
|
|
fh.write("depth_clade_min,depth_clade_max,n_quartets,median_count,q10_count,"
|
||
|
|
"frac_uncovered,median_ratio_to_min_pair\n")
|
||
|
|
for k in range(nb):
|
||
|
|
m = stratum == k
|
||
|
|
if m.any():
|
||
|
|
c = counts[m]
|
||
|
|
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{int(m.sum())},{np.median(c):.0f},"
|
||
|
|
f"{np.quantile(c, .1):.0f},{(c == 0).mean():.4f},{np.median(ratio[m]):.4f}\n")
|
||
|
|
print(f"wrote {args.output}_quartets.csv")
|
||
|
|
|
||
|
|
if args.lift_test:
|
||
|
|
pres = np.ascontiguousarray(np.hstack(sampled))
|
||
|
|
print(f"lift test on {pres.shape[1]} columns, {args.null_reps} null replicates")
|
||
|
|
rows = [("observed",) + lift_stats(pres, args.lift_frac)]
|
||
|
|
curveball(pres, args.trades, rng) # burn-in
|
||
|
|
for r in range(args.null_reps):
|
||
|
|
rows.append((f"null{r + 1}",) + lift_stats(pres, args.lift_frac))
|
||
|
|
curveball(pres, args.trades, rng)
|
||
|
|
with open(f"{args.output}_lift_test.csv", "w") as fh:
|
||
|
|
fh.write("replicate,L,T\n")
|
||
|
|
for name, l, t in rows:
|
||
|
|
fh.write(f"{name},{l:.6f},{t:.6f}\n")
|
||
|
|
for idx, label in ((1, "L (lift)"), (2, "T = P(C|A,B)")):
|
||
|
|
obs = rows[0][idx]
|
||
|
|
null = np.array([r[idx] for r in rows[1:]])
|
||
|
|
p = (1 + (null >= obs).sum()) / (1 + len(null))
|
||
|
|
print(f"{label}: observed={obs:.4f} null mean={null.mean():.4f} "
|
||
|
|
f"sd={null.std(ddof=1):.4f} z={(obs - null.mean()) / null.std(ddof=1):.1f} "
|
||
|
|
f"p(null>=obs)={p:.3f}")
|
||
|
|
print(f"wrote {args.output}_lift_test.csv")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|