This commit introduces new theoretical frameworks for k-mer indexing, pair coverage sampling, and sibling partition unification. It also implements new algorithms for calculating family entropy and statistical tests for overlap analysis. Documentation is updated across the board to reflect these new features and theoretical concepts.
396 lines
17 KiB
Python
396 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Family overlap and clade-closure ("nesting") of a SNP pseudo-alignment.
|
|
|
|
Each alignment column is one family; a genome "carries a call" at a column
|
|
when its character is not an absence symbol (default '-' and '?').
|
|
|
|
Outputs (prefix given by -o):
|
|
<prefix>_family_overlap.csv genome x genome matrix: number of columns where
|
|
both genomes carry a call (diagonal = 0, same
|
|
convention as `obikmer phylo --family-overlap`)
|
|
<prefix>_nesting.csv per clade-size stratum: how closed the families
|
|
are under the tree
|
|
|
|
Nesting measure: for a family carried by the genome set S, let C(S) be the
|
|
smallest tree clade containing S. completeness = |S| / |C(S)|. A value of 1
|
|
means the family is clade-closed (every genome of the clade carries it); low
|
|
values mean dropout or patchy conservation inside the clade.
|
|
|
|
The tree is rooted as written in the Newick file unless --outgroup is given
|
|
(rooted at the neighbour of that tip, which then becomes a child of the root).
|
|
|
|
Optional lift test (--lift-test): do families covering a rare pair (A,B) also
|
|
cover an abundant pair (A,C) more often than the margins alone imply? For each
|
|
genome A, B ranges over its --lift-frac rarest partners and C over its most
|
|
abundant ones (ranked on pair overlap). Statistics, on a random column subsample:
|
|
T = mean over triples of O_ABC / O_AB = P(C | A,B)
|
|
L = sum n_A * O_ABC / sum O_AB * O_AC (lift; 1 if B,C independent given A)
|
|
Null: curveball swap randomisation of the genome x family presence matrix, which
|
|
preserves every genome's family count and every family's size exactly. Each null
|
|
replicate re-selects the rare/abundant partners from its own overlaps, so the
|
|
selection procedure is identical to the observed one.
|
|
|
|
Optional quartet test (--quartet-test): does flattening pair overlaps also flatten
|
|
quartet coverage? --quartets N genome quartets are drawn uniformly; for each, the
|
|
number of columns where all four genomes carry a call is counted exactly. Reported
|
|
per stratum of the quartet's depth (size of the smallest tree clade containing the
|
|
four genomes): count distribution, share of uncovered quartets, and the ratio of the
|
|
4-way count to the smallest of the six pair overlaps (1 = perfect nesting; the pair
|
|
overlap is an upper bound of the 4-way count).
|
|
|
|
Usage
|
|
-----
|
|
pseudo_alignment_nesting.py ALIGNMENT.fasta TREEFILE -o prefix
|
|
[--outgroup LABEL] [--absent '-?'] [--chunk 131072]
|
|
[--quartet-test [--quartets 20000]]
|
|
[--lift-test [--lift-columns 200000] [--lift-frac 0.1]
|
|
[--null-reps 20] [--trades 5000] [--seed N]]
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
|
|
import numpy as np
|
|
|
|
|
|
def read_fasta_presence(path, absent):
|
|
"""Return (labels, packed presence bits per genome, alignment length)."""
|
|
absent_lut = np.zeros(256, dtype=bool)
|
|
for ch in absent:
|
|
absent_lut[ord(ch)] = True
|
|
labels, packed, length = [], [], None
|
|
|
|
def flush(parts):
|
|
nonlocal length
|
|
seq = np.frombuffer(b"".join(parts), dtype=np.uint8)
|
|
if length is None:
|
|
length = len(seq)
|
|
elif len(seq) != length:
|
|
sys.exit(f"{labels[-1]}: length {len(seq)} != {length}")
|
|
packed.append(np.packbits(~absent_lut[seq]))
|
|
|
|
parts = None
|
|
with open(path, "rb") as fh:
|
|
for line in fh:
|
|
if line.startswith(b">"):
|
|
if parts is not None:
|
|
flush(parts)
|
|
labels.append(line[1:].split()[0].decode())
|
|
parts = []
|
|
else:
|
|
parts.append(line.strip())
|
|
if parts is not None:
|
|
flush(parts)
|
|
return labels, np.vstack(packed), length
|
|
|
|
|
|
def parse_newick(text):
|
|
"""Parse Newick into (names, parent, length) arrays; internal labels ignored."""
|
|
names, parent, length = [], [], []
|
|
stack, cur = [], None
|
|
i, n = 0, len(text)
|
|
|
|
def new_node():
|
|
names.append(None)
|
|
parent.append(-1)
|
|
length.append(0.0)
|
|
return len(names) - 1
|
|
|
|
while i < n:
|
|
c = text[i]
|
|
if c in " \n\r\t":
|
|
i += 1
|
|
elif c == "(":
|
|
if stack:
|
|
node = stack[-1] # the placeholder child becomes this internal node
|
|
else:
|
|
node = new_node()
|
|
stack.append(node)
|
|
cur = None
|
|
i += 1
|
|
child = new_node()
|
|
parent[child] = node
|
|
stack.append(child)
|
|
elif c == ",":
|
|
stack.pop()
|
|
child = new_node()
|
|
parent[child] = stack[-1]
|
|
stack.append(child)
|
|
i += 1
|
|
elif c == ")":
|
|
stack.pop()
|
|
cur = stack[-1]
|
|
i += 1
|
|
j = i
|
|
while j < n and text[j] not in ":,);":
|
|
j += 1
|
|
i = j # internal label (support values) discarded
|
|
elif c == ":":
|
|
j = i + 1
|
|
while j < n and text[j] not in ",);":
|
|
j += 1
|
|
length[stack[-1] if cur is None else cur] = float(text[i + 1:j])
|
|
cur = None
|
|
i = j
|
|
elif c == ";":
|
|
break
|
|
else:
|
|
j = i
|
|
while j < n and text[j] not in ":,);":
|
|
j += 1
|
|
names[stack[-1]] = text[i:j]
|
|
i = j
|
|
return names, parent, length
|
|
|
|
|
|
def leaf_order_matrices(names, parent, length, outgroup):
|
|
"""DFS leaf order plus LCA-clade-size and patristic matrices in that order."""
|
|
adj = [[] for _ in names]
|
|
for v, p in enumerate(parent):
|
|
if p >= 0:
|
|
adj[v].append((p, length[v]))
|
|
adj[p].append((v, length[v]))
|
|
root = next(v for v, p in enumerate(parent) if p < 0)
|
|
if outgroup is not None:
|
|
tip = names.index(outgroup)
|
|
root = adj[tip][0][0]
|
|
|
|
order, depth, children = [], {root: 0.0}, {}
|
|
stack, seen = [root], {root}
|
|
while stack:
|
|
v = stack.pop()
|
|
kids = [(u, w) for u, w in adj[v] if u not in seen]
|
|
for u, w in kids:
|
|
seen.add(u)
|
|
depth[u] = depth[v] + w
|
|
children[v] = [u for u, _ in kids]
|
|
stack.extend(reversed(children[v]))
|
|
order.append(v)
|
|
|
|
leaves = [v for v in order if not children[v]]
|
|
rank = {v: r for r, v in enumerate(leaves)}
|
|
n = len(leaves)
|
|
span = {}
|
|
for v in reversed(order):
|
|
span[v] = (rank[v], rank[v] + 1) if not children[v] else (
|
|
span[children[v][0]][0], span[children[v][-1]][1])
|
|
|
|
clade = np.ones((n, n), dtype=np.int32)
|
|
lca_depth = np.zeros((n, n))
|
|
for v in order:
|
|
kids = children[v]
|
|
for a in range(len(kids)):
|
|
for b in range(a + 1, len(kids)):
|
|
(a0, a1), (b0, b1) = span[kids[a]], span[kids[b]]
|
|
size = span[v][1] - span[v][0]
|
|
clade[a0:a1, b0:b1] = clade[b0:b1, a0:a1] = size
|
|
lca_depth[a0:a1, b0:b1] = lca_depth[b0:b1, a0:a1] = depth[v]
|
|
leaf_depth = np.array([depth[v] for v in leaves])
|
|
patristic = leaf_depth[:, None] + leaf_depth[None, :] - 2 * lca_depth
|
|
return [names[v] for v in leaves], clade, patristic
|
|
|
|
|
|
def lift_stats(pres, frac):
|
|
"""(L, T) for the rare/abundant partner triples of every genome (see module doc)."""
|
|
n = pres.shape[0]
|
|
pf = pres.astype(np.float32)
|
|
overlap = pf @ pf.T
|
|
k = max(1, round(frac * (n - 1)))
|
|
num = den = ratio_sum = 0.0
|
|
n_ratio = 0
|
|
for a in range(n):
|
|
order = [j for j in np.argsort(overlap[a], kind="stable") if j != a]
|
|
rare, abundant = order[:k], order[-k:]
|
|
cols = pres[a]
|
|
n_a = float(cols.sum())
|
|
if n_a == 0:
|
|
continue
|
|
o_abc = pf[rare][:, cols] @ pf[abundant][:, cols].T
|
|
o_ab, o_ac = overlap[a, rare].astype(np.float64), overlap[a, abundant].astype(np.float64)
|
|
num += n_a * float(o_abc.sum())
|
|
den += float((o_ab[:, None] * o_ac[None, :]).sum())
|
|
ok = o_ab > 0
|
|
ratio_sum += float((o_abc[ok] / o_ab[ok, None]).sum())
|
|
n_ratio += int(ok.sum()) * k
|
|
return num / den, ratio_sum / n_ratio
|
|
|
|
|
|
def curveball(pres, trades, rng):
|
|
"""In-place curveball trades: preserve every row sum and every column sum."""
|
|
n = pres.shape[0]
|
|
for _ in range(trades):
|
|
i, j = rng.choice(n, size=2, replace=False)
|
|
a, b = pres[i], pres[j]
|
|
diff = np.flatnonzero(a ^ b)
|
|
k = int((a & ~b).sum())
|
|
diff = rng.permutation(diff)
|
|
a[diff] = False
|
|
b[diff] = False
|
|
a[diff[:k]] = True
|
|
b[diff[k:]] = True
|
|
|
|
|
|
def quartet_test(packed, overlap, tree_rank, clade, n_quartets, bin_edges, rng):
|
|
"""Exact 4-way co-presence counts for random genome quartets.
|
|
|
|
Returns (counts, ratio_to_min_pair, depth_stratum), one entry per quartet.
|
|
"""
|
|
n = packed.shape[0]
|
|
quartets = np.argpartition(rng.random((n_quartets, n)), 3, axis=1)[:, :4]
|
|
counts = np.zeros(n_quartets, dtype=np.int64)
|
|
block = 2048 # bytes of packed columns per pass: bounds the (Q, block) gathers
|
|
for a in range(0, packed.shape[1], block):
|
|
blk = packed[:, a:a + block]
|
|
acc = blk[quartets[:, 0]] & blk[quartets[:, 1]]
|
|
acc &= blk[quartets[:, 2]]
|
|
acc &= blk[quartets[:, 3]]
|
|
counts += np.bitwise_count(acc).sum(axis=1, dtype=np.int64)
|
|
|
|
pair_min = np.full(n_quartets, np.iinfo(np.int64).max)
|
|
for x in range(4):
|
|
for y in range(x + 1, 4):
|
|
pair_min = np.minimum(pair_min, overlap[quartets[:, x], quartets[:, y]])
|
|
ranks = tree_rank[quartets]
|
|
csize = clade[ranks.min(axis=1), ranks.max(axis=1)]
|
|
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
|
|
return counts, counts / np.maximum(pair_min, 1), stratum
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("alignment")
|
|
ap.add_argument("treefile")
|
|
ap.add_argument("-o", "--output", required=True, help="output file prefix")
|
|
ap.add_argument("--outgroup", help="tip label to root the tree on")
|
|
ap.add_argument("--absent", default="-?", help="absence symbols (default '-?')")
|
|
ap.add_argument("--chunk", type=int, default=131072,
|
|
help="columns per chunk, multiple of 8 (default 131072)")
|
|
ap.add_argument("--quartet-test", action="store_true",
|
|
help="measure exact 4-way coverage of random genome quartets")
|
|
ap.add_argument("--quartets", type=int, default=20000,
|
|
help="number of random quartets (default 20000)")
|
|
ap.add_argument("--lift-test", action="store_true",
|
|
help="run the margin-preserving lift test (see module doc)")
|
|
ap.add_argument("--lift-columns", type=int, default=200000,
|
|
help="random columns used by the lift test (default 200000)")
|
|
ap.add_argument("--lift-frac", type=float, default=0.1,
|
|
help="fraction of partners ranked rare / abundant (default 0.1)")
|
|
ap.add_argument("--null-reps", type=int, default=20)
|
|
ap.add_argument("--trades", type=int, default=5000,
|
|
help="curveball trades between replicates, also burn-in (default 5000)")
|
|
ap.add_argument("--seed", type=int, default=None)
|
|
args = ap.parse_args()
|
|
if args.chunk % 8:
|
|
sys.exit("--chunk must be a multiple of 8")
|
|
|
|
rng = np.random.default_rng(args.seed)
|
|
labels, packed, length = read_fasta_presence(args.alignment, args.absent)
|
|
keep_p = min(1.0, args.lift_columns / length) if args.lift_test else 0.0
|
|
sampled = []
|
|
with open(args.treefile) as fh:
|
|
names, parent, blen = parse_newick(fh.read())
|
|
leaf_labels, clade, patristic = leaf_order_matrices(names, parent, blen, args.outgroup)
|
|
if set(leaf_labels) != set(labels):
|
|
sys.exit("tree tips and alignment records differ")
|
|
n = len(labels)
|
|
tree_row = np.array([labels.index(l) for l in leaf_labels]) # tree rank -> alignment row
|
|
|
|
bin_edges = np.array([2, 3, 5, 9, 17, 33, 65, 129, n + 1]) # clade-size strata
|
|
nb = len(bin_edges) - 1
|
|
cnt = np.zeros(nb, dtype=np.int64)
|
|
closed = np.zeros(nb, dtype=np.int64)
|
|
sum_ratio = np.zeros(nb)
|
|
sum_missing = np.zeros(nb)
|
|
overlap = np.zeros((n, n), dtype=np.int64)
|
|
n_cols_lt2 = 0
|
|
|
|
for a in range(0, length, args.chunk):
|
|
b = min(a + args.chunk, length)
|
|
pres = np.unpackbits(packed[:, a // 8:(b + 7) // 8], axis=1)[:, :b - a]
|
|
if keep_p:
|
|
sampled.append(pres[:, rng.random(b - a) < keep_p].astype(bool))
|
|
overlap += np.rint(pres.astype(np.float32) @ pres.astype(np.float32).T).astype(np.int64)
|
|
|
|
t = pres[tree_row]
|
|
size = t.sum(axis=0)
|
|
keep = size >= 2
|
|
n_cols_lt2 += int((~keep).sum())
|
|
t, size = t[:, keep], size[keep]
|
|
lo = t.argmax(axis=0)
|
|
hi = n - 1 - t[::-1].argmax(axis=0)
|
|
csize = clade[lo, hi]
|
|
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
|
|
cnt += np.bincount(stratum, minlength=nb)
|
|
closed += np.bincount(stratum, weights=(size == csize), minlength=nb).astype(np.int64)
|
|
sum_ratio += np.bincount(stratum, weights=size / csize, minlength=nb)
|
|
sum_missing += np.bincount(stratum, weights=csize - size, minlength=nb)
|
|
|
|
np.fill_diagonal(overlap, 0)
|
|
with open(f"{args.output}_family_overlap.csv", "w") as fh:
|
|
fh.write("genome," + ",".join(labels) + "\n")
|
|
for l, row in zip(labels, overlap):
|
|
fh.write(l + "," + ",".join(map(str, row)) + "\n")
|
|
|
|
with open(f"{args.output}_nesting.csv", "w") as fh:
|
|
fh.write("clade_size_min,clade_size_max,n_families,mean_completeness,"
|
|
"frac_clade_closed,mean_missing_in_clade\n")
|
|
for k in range(nb):
|
|
if cnt[k]:
|
|
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{cnt[k]},"
|
|
f"{sum_ratio[k] / cnt[k]:.4f},{closed[k] / cnt[k]:.4f},"
|
|
f"{sum_missing[k] / cnt[k]:.3f}\n")
|
|
|
|
off = overlap[np.triu_indices(n, 1)]
|
|
print(f"genomes={n} columns={length} (columns with <2 calls skipped for nesting: {n_cols_lt2})")
|
|
print(f"overlap off-diagonal: min={off.min()} median={int(np.median(off))} "
|
|
f"max={off.max()} CV={off.std() / off.mean():.3f}")
|
|
tot = cnt.sum()
|
|
print(f"clade-closed families: {closed.sum()}/{tot} ({closed.sum() / tot:.3f}); "
|
|
f"mean completeness {sum_ratio.sum() / tot:.3f}")
|
|
print(f"wrote {args.output}_family_overlap.csv and {args.output}_nesting.csv")
|
|
|
|
if args.quartet_test:
|
|
tree_rank = np.empty(n, dtype=np.int64)
|
|
tree_rank[tree_row] = np.arange(n)
|
|
counts, ratio, stratum = quartet_test(
|
|
packed, overlap, tree_rank, clade, args.quartets, bin_edges, rng)
|
|
print(f"quartet test: {args.quartets} random quartets over {length} columns")
|
|
q = np.quantile(counts, [0, .01, .1, .5, .9, .99, 1])
|
|
print("4-way count quantiles 0/1/10/50/90/99/100 %: " + " ".join(f"{v:.0f}" for v in q)
|
|
+ f"; uncovered: {(counts == 0).mean():.4f}; CV={counts.std() / max(counts.mean(), 1e-9):.2f}")
|
|
with open(f"{args.output}_quartets.csv", "w") as fh:
|
|
fh.write("depth_clade_min,depth_clade_max,n_quartets,median_count,q10_count,"
|
|
"frac_uncovered,median_ratio_to_min_pair\n")
|
|
for k in range(nb):
|
|
m = stratum == k
|
|
if m.any():
|
|
c = counts[m]
|
|
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{int(m.sum())},{np.median(c):.0f},"
|
|
f"{np.quantile(c, .1):.0f},{(c == 0).mean():.4f},{np.median(ratio[m]):.4f}\n")
|
|
print(f"wrote {args.output}_quartets.csv")
|
|
|
|
if args.lift_test:
|
|
pres = np.ascontiguousarray(np.hstack(sampled))
|
|
print(f"lift test on {pres.shape[1]} columns, {args.null_reps} null replicates")
|
|
rows = [("observed",) + lift_stats(pres, args.lift_frac)]
|
|
curveball(pres, args.trades, rng) # burn-in
|
|
for r in range(args.null_reps):
|
|
rows.append((f"null{r + 1}",) + lift_stats(pres, args.lift_frac))
|
|
curveball(pres, args.trades, rng)
|
|
with open(f"{args.output}_lift_test.csv", "w") as fh:
|
|
fh.write("replicate,L,T\n")
|
|
for name, l, t in rows:
|
|
fh.write(f"{name},{l:.6f},{t:.6f}\n")
|
|
for idx, label in ((1, "L (lift)"), (2, "T = P(C|A,B)")):
|
|
obs = rows[0][idx]
|
|
null = np.array([r[idx] for r in rows[1:]])
|
|
p = (1 + (null >= obs).sum()) / (1 + len(null))
|
|
print(f"{label}: observed={obs:.4f} null mean={null.mean():.4f} "
|
|
f"sd={null.std(ddof=1):.4f} z={(obs - null.mean()) / null.std(ddof=1):.1f} "
|
|
f"p(null>=obs)={p:.3f}")
|
|
print(f"wrote {args.output}_lift_test.csv")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|