Refactor indexing, sampling, and entropy documentation

This commit introduces new theoretical frameworks for k-mer indexing, pair coverage sampling, and sibling partition unification. It also implements new algorithms for calculating family entropy and statistical tests for overlap analysis. Documentation is updated across the board to reflect these new features and theoretical concepts.
This commit is contained in:
Eric Coissac committed 2026-10-13 08:54:21 +02:00
1 parent 95bbc0b2ae
commit 0d12494f1a
12 files changed
+1182 -56

No files matched your search

+395
View File
@@ -0,0 +1,395 @@
#!/usr/bin/env python3
"""Family overlap and clade-closure ("nesting") of a SNP pseudo-alignment.
Each alignment column is one family; a genome "carries a call" at a column
when its character is not an absence symbol (default '-' and '?').
Outputs (prefix given by -o):
<prefix>_family_overlap.csv genome x genome matrix: number of columns where
both genomes carry a call (diagonal = 0, same
convention as `obikmer phylo --family-overlap`)
<prefix>_nesting.csv per clade-size stratum: how closed the families
are under the tree
Nesting measure: for a family carried by the genome set S, let C(S) be the
smallest tree clade containing S. completeness = |S| / |C(S)|. A value of 1
means the family is clade-closed (every genome of the clade carries it); low
values mean dropout or patchy conservation inside the clade.
The tree is rooted as written in the Newick file unless --outgroup is given
(rooted at the neighbour of that tip, which then becomes a child of the root).
Optional lift test (--lift-test): do families covering a rare pair (A,B) also
cover an abundant pair (A,C) more often than the margins alone imply? For each
genome A, B ranges over its --lift-frac rarest partners and C over its most
abundant ones (ranked on pair overlap). Statistics, on a random column subsample:
T = mean over triples of O_ABC / O_AB = P(C | A,B)
L = sum n_A * O_ABC / sum O_AB * O_AC (lift; 1 if B,C independent given A)
Null: curveball swap randomisation of the genome x family presence matrix, which
preserves every genome's family count and every family's size exactly. Each null
replicate re-selects the rare/abundant partners from its own overlaps, so the
selection procedure is identical to the observed one.
Optional quartet test (--quartet-test): does flattening pair overlaps also flatten
quartet coverage? --quartets N genome quartets are drawn uniformly; for each, the
number of columns where all four genomes carry a call is counted exactly. Reported
per stratum of the quartet's depth (size of the smallest tree clade containing the
four genomes): count distribution, share of uncovered quartets, and the ratio of the
4-way count to the smallest of the six pair overlaps (1 = perfect nesting; the pair
overlap is an upper bound of the 4-way count).
Usage
-----
pseudo_alignment_nesting.py ALIGNMENT.fasta TREEFILE -o prefix
[--outgroup LABEL] [--absent '-?'] [--chunk 131072]
[--quartet-test [--quartets 20000]]
[--lift-test [--lift-columns 200000] [--lift-frac 0.1]
[--null-reps 20] [--trades 5000] [--seed N]]
"""
import argparse
import sys
import numpy as np
def read_fasta_presence(path, absent):
"""Return (labels, packed presence bits per genome, alignment length)."""
absent_lut = np.zeros(256, dtype=bool)
for ch in absent:
absent_lut[ord(ch)] = True
labels, packed, length = [], [], None
def flush(parts):
nonlocal length
seq = np.frombuffer(b"".join(parts), dtype=np.uint8)
if length is None:
length = len(seq)
elif len(seq) != length:
sys.exit(f"{labels[-1]}: length {len(seq)} != {length}")
packed.append(np.packbits(~absent_lut[seq]))
parts = None
with open(path, "rb") as fh:
for line in fh:
if line.startswith(b">"):
if parts is not None:
flush(parts)
labels.append(line[1:].split()[0].decode())
parts = []
else:
parts.append(line.strip())
if parts is not None:
flush(parts)
return labels, np.vstack(packed), length
def parse_newick(text):
"""Parse Newick into (names, parent, length) arrays; internal labels ignored."""
names, parent, length = [], [], []
stack, cur = [], None
i, n = 0, len(text)
def new_node():
names.append(None)
parent.append(-1)
length.append(0.0)
return len(names) - 1
while i < n:
c = text[i]
if c in " \n\r\t":
i += 1
elif c == "(":
if stack:
node = stack[-1] # the placeholder child becomes this internal node
else:
node = new_node()
stack.append(node)
cur = None
i += 1
child = new_node()
parent[child] = node
stack.append(child)
elif c == ",":
stack.pop()
child = new_node()
parent[child] = stack[-1]
stack.append(child)
i += 1
elif c == ")":
stack.pop()
cur = stack[-1]
i += 1
j = i
while j < n and text[j] not in ":,);":
j += 1
i = j # internal label (support values) discarded
elif c == ":":
j = i + 1
while j < n and text[j] not in ",);":
j += 1
length[stack[-1] if cur is None else cur] = float(text[i + 1:j])
cur = None
i = j
elif c == ";":
break
else:
j = i
while j < n and text[j] not in ":,);":
j += 1
names[stack[-1]] = text[i:j]
i = j
return names, parent, length
def leaf_order_matrices(names, parent, length, outgroup):
"""DFS leaf order plus LCA-clade-size and patristic matrices in that order."""
adj = [[] for _ in names]
for v, p in enumerate(parent):
if p >= 0:
adj[v].append((p, length[v]))
adj[p].append((v, length[v]))
root = next(v for v, p in enumerate(parent) if p < 0)
if outgroup is not None:
tip = names.index(outgroup)
root = adj[tip][0][0]
order, depth, children = [], {root: 0.0}, {}
stack, seen = [root], {root}
while stack:
v = stack.pop()
kids = [(u, w) for u, w in adj[v] if u not in seen]
for u, w in kids:
seen.add(u)
depth[u] = depth[v] + w
children[v] = [u for u, _ in kids]
stack.extend(reversed(children[v]))
order.append(v)
leaves = [v for v in order if not children[v]]
rank = {v: r for r, v in enumerate(leaves)}
n = len(leaves)
span = {}
for v in reversed(order):
span[v] = (rank[v], rank[v] + 1) if not children[v] else (
span[children[v][0]][0], span[children[v][-1]][1])
clade = np.ones((n, n), dtype=np.int32)
lca_depth = np.zeros((n, n))
for v in order:
kids = children[v]
for a in range(len(kids)):
for b in range(a + 1, len(kids)):
(a0, a1), (b0, b1) = span[kids[a]], span[kids[b]]
size = span[v][1] - span[v][0]
clade[a0:a1, b0:b1] = clade[b0:b1, a0:a1] = size
lca_depth[a0:a1, b0:b1] = lca_depth[b0:b1, a0:a1] = depth[v]
leaf_depth = np.array([depth[v] for v in leaves])
patristic = leaf_depth[:, None] + leaf_depth[None, :] - 2 * lca_depth
return [names[v] for v in leaves], clade, patristic
def lift_stats(pres, frac):
"""(L, T) for the rare/abundant partner triples of every genome (see module doc)."""
n = pres.shape[0]
pf = pres.astype(np.float32)
overlap = pf @ pf.T
k = max(1, round(frac * (n - 1)))
num = den = ratio_sum = 0.0
n_ratio = 0
for a in range(n):
order = [j for j in np.argsort(overlap[a], kind="stable") if j != a]
rare, abundant = order[:k], order[-k:]
cols = pres[a]
n_a = float(cols.sum())
if n_a == 0:
continue
o_abc = pf[rare][:, cols] @ pf[abundant][:, cols].T
o_ab, o_ac = overlap[a, rare].astype(np.float64), overlap[a, abundant].astype(np.float64)
num += n_a * float(o_abc.sum())
den += float((o_ab[:, None] * o_ac[None, :]).sum())
ok = o_ab > 0
ratio_sum += float((o_abc[ok] / o_ab[ok, None]).sum())
n_ratio += int(ok.sum()) * k
return num / den, ratio_sum / n_ratio
def curveball(pres, trades, rng):
"""In-place curveball trades: preserve every row sum and every column sum."""
n = pres.shape[0]
for _ in range(trades):
i, j = rng.choice(n, size=2, replace=False)
a, b = pres[i], pres[j]
diff = np.flatnonzero(a ^ b)
k = int((a & ~b).sum())
diff = rng.permutation(diff)
a[diff] = False
b[diff] = False
a[diff[:k]] = True
b[diff[k:]] = True
def quartet_test(packed, overlap, tree_rank, clade, n_quartets, bin_edges, rng):
"""Exact 4-way co-presence counts for random genome quartets.
Returns (counts, ratio_to_min_pair, depth_stratum), one entry per quartet.
"""
n = packed.shape[0]
quartets = np.argpartition(rng.random((n_quartets, n)), 3, axis=1)[:, :4]
counts = np.zeros(n_quartets, dtype=np.int64)
block = 2048 # bytes of packed columns per pass: bounds the (Q, block) gathers
for a in range(0, packed.shape[1], block):
blk = packed[:, a:a + block]
acc = blk[quartets[:, 0]] & blk[quartets[:, 1]]
acc &= blk[quartets[:, 2]]
acc &= blk[quartets[:, 3]]
counts += np.bitwise_count(acc).sum(axis=1, dtype=np.int64)
pair_min = np.full(n_quartets, np.iinfo(np.int64).max)
for x in range(4):
for y in range(x + 1, 4):
pair_min = np.minimum(pair_min, overlap[quartets[:, x], quartets[:, y]])
ranks = tree_rank[quartets]
csize = clade[ranks.min(axis=1), ranks.max(axis=1)]
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
return counts, counts / np.maximum(pair_min, 1), stratum
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("alignment")
ap.add_argument("treefile")
ap.add_argument("-o", "--output", required=True, help="output file prefix")
ap.add_argument("--outgroup", help="tip label to root the tree on")
ap.add_argument("--absent", default="-?", help="absence symbols (default '-?')")
ap.add_argument("--chunk", type=int, default=131072,
help="columns per chunk, multiple of 8 (default 131072)")
ap.add_argument("--quartet-test", action="store_true",
help="measure exact 4-way coverage of random genome quartets")
ap.add_argument("--quartets", type=int, default=20000,
help="number of random quartets (default 20000)")
ap.add_argument("--lift-test", action="store_true",
help="run the margin-preserving lift test (see module doc)")
ap.add_argument("--lift-columns", type=int, default=200000,
help="random columns used by the lift test (default 200000)")
ap.add_argument("--lift-frac", type=float, default=0.1,
help="fraction of partners ranked rare / abundant (default 0.1)")
ap.add_argument("--null-reps", type=int, default=20)
ap.add_argument("--trades", type=int, default=5000,
help="curveball trades between replicates, also burn-in (default 5000)")
ap.add_argument("--seed", type=int, default=None)
args = ap.parse_args()
if args.chunk % 8:
sys.exit("--chunk must be a multiple of 8")
rng = np.random.default_rng(args.seed)
labels, packed, length = read_fasta_presence(args.alignment, args.absent)
keep_p = min(1.0, args.lift_columns / length) if args.lift_test else 0.0
sampled = []
with open(args.treefile) as fh:
names, parent, blen = parse_newick(fh.read())
leaf_labels, clade, patristic = leaf_order_matrices(names, parent, blen, args.outgroup)
if set(leaf_labels) != set(labels):
sys.exit("tree tips and alignment records differ")
n = len(labels)
tree_row = np.array([labels.index(l) for l in leaf_labels]) # tree rank -> alignment row
bin_edges = np.array([2, 3, 5, 9, 17, 33, 65, 129, n + 1]) # clade-size strata
nb = len(bin_edges) - 1
cnt = np.zeros(nb, dtype=np.int64)
closed = np.zeros(nb, dtype=np.int64)
sum_ratio = np.zeros(nb)
sum_missing = np.zeros(nb)
overlap = np.zeros((n, n), dtype=np.int64)
n_cols_lt2 = 0
for a in range(0, length, args.chunk):
b = min(a + args.chunk, length)
pres = np.unpackbits(packed[:, a // 8:(b + 7) // 8], axis=1)[:, :b - a]
if keep_p:
sampled.append(pres[:, rng.random(b - a) < keep_p].astype(bool))
overlap += np.rint(pres.astype(np.float32) @ pres.astype(np.float32).T).astype(np.int64)
t = pres[tree_row]
size = t.sum(axis=0)
keep = size >= 2
n_cols_lt2 += int((~keep).sum())
t, size = t[:, keep], size[keep]
lo = t.argmax(axis=0)
hi = n - 1 - t[::-1].argmax(axis=0)
csize = clade[lo, hi]
stratum = np.searchsorted(bin_edges, csize, side="right") - 1
cnt += np.bincount(stratum, minlength=nb)
closed += np.bincount(stratum, weights=(size == csize), minlength=nb).astype(np.int64)
sum_ratio += np.bincount(stratum, weights=size / csize, minlength=nb)
sum_missing += np.bincount(stratum, weights=csize - size, minlength=nb)
np.fill_diagonal(overlap, 0)
with open(f"{args.output}_family_overlap.csv", "w") as fh:
fh.write("genome," + ",".join(labels) + "\n")
for l, row in zip(labels, overlap):
fh.write(l + "," + ",".join(map(str, row)) + "\n")
with open(f"{args.output}_nesting.csv", "w") as fh:
fh.write("clade_size_min,clade_size_max,n_families,mean_completeness,"
"frac_clade_closed,mean_missing_in_clade\n")
for k in range(nb):
if cnt[k]:
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{cnt[k]},"
f"{sum_ratio[k] / cnt[k]:.4f},{closed[k] / cnt[k]:.4f},"
f"{sum_missing[k] / cnt[k]:.3f}\n")
off = overlap[np.triu_indices(n, 1)]
print(f"genomes={n} columns={length} (columns with <2 calls skipped for nesting: {n_cols_lt2})")
print(f"overlap off-diagonal: min={off.min()} median={int(np.median(off))} "
f"max={off.max()} CV={off.std() / off.mean():.3f}")
tot = cnt.sum()
print(f"clade-closed families: {closed.sum()}/{tot} ({closed.sum() / tot:.3f}); "
f"mean completeness {sum_ratio.sum() / tot:.3f}")
print(f"wrote {args.output}_family_overlap.csv and {args.output}_nesting.csv")
if args.quartet_test:
tree_rank = np.empty(n, dtype=np.int64)
tree_rank[tree_row] = np.arange(n)
counts, ratio, stratum = quartet_test(
packed, overlap, tree_rank, clade, args.quartets, bin_edges, rng)
print(f"quartet test: {args.quartets} random quartets over {length} columns")
q = np.quantile(counts, [0, .01, .1, .5, .9, .99, 1])
print("4-way count quantiles 0/1/10/50/90/99/100 %: " + " ".join(f"{v:.0f}" for v in q)
+ f"; uncovered: {(counts == 0).mean():.4f}; CV={counts.std() / max(counts.mean(), 1e-9):.2f}")
with open(f"{args.output}_quartets.csv", "w") as fh:
fh.write("depth_clade_min,depth_clade_max,n_quartets,median_count,q10_count,"
"frac_uncovered,median_ratio_to_min_pair\n")
for k in range(nb):
m = stratum == k
if m.any():
c = counts[m]
fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{int(m.sum())},{np.median(c):.0f},"
f"{np.quantile(c, .1):.0f},{(c == 0).mean():.4f},{np.median(ratio[m]):.4f}\n")
print(f"wrote {args.output}_quartets.csv")
if args.lift_test:
pres = np.ascontiguousarray(np.hstack(sampled))
print(f"lift test on {pres.shape[1]} columns, {args.null_reps} null replicates")
rows = [("observed",) + lift_stats(pres, args.lift_frac)]
curveball(pres, args.trades, rng) # burn-in
for r in range(args.null_reps):
rows.append((f"null{r + 1}",) + lift_stats(pres, args.lift_frac))
curveball(pres, args.trades, rng)
with open(f"{args.output}_lift_test.csv", "w") as fh:
fh.write("replicate,L,T\n")
for name, l, t in rows:
fh.write(f"{name},{l:.6f},{t:.6f}\n")
for idx, label in ((1, "L (lift)"), (2, "T = P(C|A,B)")):
obs = rows[0][idx]
null = np.array([r[idx] for r in rows[1:]])
p = (1 + (null >= obs).sum()) / (1 + len(null))
print(f"{label}: observed={obs:.4f} null mean={null.mean():.4f} "
f"sd={null.std(ddof=1):.4f} z={(obs - null.mean()) / null.std(ddof=1):.1f} "
f"p(null>=obs)={p:.3f}")
print(f"wrote {args.output}_lift_test.csv")
if __name__ == "__main__":
main()