From 9da0876ee6d01ec6b72b47ff8d03cb8db18e0ff8 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Sat, 12 Sep 2026 12:44:22 +0200 Subject: [PATCH 01/10] Refactor indexing logic using hash-based minimizer selection. Updates indexing architecture to use hashing for minimizer distribution and introduces a new hash-based selection mechanism. Additionally, updates documentation references and implements a new script for converting MkDocs into a flat wiki structure. --- .gitignore | 1 + UserDocMD/references.bib | 86 ++++ UserDocMD/theory/indexing_architecture.md | 10 +- UserDocMD/theory/minimizer_selection.md | 84 +++- scripts/mk_wiki.py | 528 ++++++++++++++++++++++ 5 files changed, 686 insertions(+), 23 deletions(-) create mode 100755 scripts/mk_wiki.py diff --git a/.gitignore b/.gitignore index 00fa7f0e..f0bf8a7c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ biblio *.parstree *.ckp.gz *.model +scripts/__pycache__/mk_wiki.cpython-314.pyc diff --git a/UserDocMD/references.bib b/UserDocMD/references.bib index fca4c8bc..ce2b810f 100644 --- a/UserDocMD/references.bib +++ b/UserDocMD/references.bib @@ -259,3 +259,89 @@ url = {https://doi.org/10.1186/s12864-015-1647-5}, volume = 16, year = 2015} + +@article{Roberts2004-rz, + title = "{Reducing storage requirements for biological sequence + comparison}", + author = "Roberts, Michael and Hayes, Wayne and Hunt, Brian R and Mount, + Stephen M and Yorke, James A", + journal = "Bioinformatics (Oxford, England)", + publisher = "Oxford University Press (OUP)", + volume = 20, + number = 18, + pages = "3363--3369", + abstract = "MOTIVATION: Comparison of nucleic acid and protein sequences is a + fundamental tool of modern bioinformatics. A dominant method of + such string matching is the 'seed-and-extend' approach, in which + occurrences of short subsequences called 'seeds' are used to + search for potentially longer matches in a large database of + sequences. Each such potential match is then checked to see if it + extends beyond the seed. To be effective, the seed-and-extend + approach needs to catalogue seeds from virtually every substring + in the database of search strings. Projects such as mammalian + genome assemblies and large-scale protein matching, however, have + such large sequence databases that the resulting list of seeds + cannot be stored in RAM on a single computer. This significantly + slows the matching process. RESULTS: We present a simple and + elegant method in which only a small fraction of seeds, called + 'minimizers', needs to be stored. Using minimizers can speed up + string-matching computations by a large factor while missing only + a small fraction of the matches found using all seeds.", + month = dec, + year = 2004, + url = "https://dx.doi.org/10.1093/bioinformatics/bth408", + doi = "10.1093/bioinformatics/bth408", + pmid = 15256412, + issn = "1367-4803,1367-4811", + language = "en" + } + +@INPROCEEDINGS{Steele2014-cz, + title = "{Fast splittable pseudorandom number generators}", + author = "Steele, Jr, Guy L and Lea, Doug and Flood, Christine H", + booktitle = "{Proceedings of the 2014 ACM International Conference on Object + Oriented Programming Systems Languages \& Applications}", + publisher = "ACM", + address = "New York, NY, USA", + abstract = "We describe a new algorithm SplitMix for an object-oriented and + splittable pseudorandom number generator (PRNG) that is quite + fast: 9 64-bit arithmetic/logical operations per 64 bits + generated. A conventional linear PRNG object provides a generate + method that returns one pseudorandom value and updates the state + of the PRNG, but a splittable PRNG object also has a second + operation, split, that replaces the original PRNG object with two + (seemingly) independent PRNG objects, by creating and returning a + new such object and updating the state of the original object. + Splittable PRNG objects make it easy to organize the use of + pseudorandom numbers in multithreaded programs structured using + fork-join parallelism. No locking or synchronization is required + (other than the usual memory fence immediately after object + creation). Because the generate method has no loops or + conditionals, it is suitable for SIMD or GPU implementation. We + derive SplitMix from the DotMix algorithm of Leiserson, Schardl, + and Sukha by making a series of program transformations and + engineering improvements. The end result is an object-oriented + version of the purely functional API used in the Haskell library + for over a decade, but SplitMix is faster and produces + pseudorandom sequences of higher quality; it is also far superior + in quality and speed to java.util.Random, and has been included + in Java JDK8 as the class java.util.SplittableRandom. We have + tested the pseudorandom sequences produced by SplitMix using two + standard statistical test suites (DieHarder and TestU01) and they + appear to be adequate for ``everyday'' use, such as in Monte + Carlo algorithms and randomized data structures where speed is + important.", + month = oct, + year = 2014, + url = "http://dx.doi.org/10.1145/2660193.2660195", + doi = "10.1145/2660193.2660195", + isbn = 9781450325851 + } + +@online{Stafford2011, + author = {Stafford, David}, + title = {Better Bit Mixing - Improving on MurmurHash3's 64-bit Finalizer}, + date = {2011-09-28}, + url = {https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html}, + urldate = {2026-09-12} + } diff --git a/UserDocMD/theory/indexing_architecture.md b/UserDocMD/theory/indexing_architecture.md index bd402519..43c72fab 100644 --- a/UserDocMD/theory/indexing_architecture.md +++ b/UserDocMD/theory/indexing_architecture.md @@ -10,18 +10,12 @@ The canonical minimizer of a super-kmer (see [Minimizer selection](minimizer_sel canonical minimizer → hash(minimizer) → p-bit value → partition index ``` -The routing value is recomputed whenever it is needed (during construction and again at query time) rather than stored — it is not part of the on-disk super-kmer representation. - Within a partition, kmers are indexed as plain values via a minimal perfect hash function (see [On-disk storage](../formats/index_layout.md)); the minimizer plays no further role once a super-kmer has reached its partition. -## Why hashing is necessary - -A canonical minimizer is an m-mer ($m \in \{9, 11, 13, 15\}$), and its distribution over all possible m-mer values is not uniform — as the lexicographic minimum of a window, small values are systematically over-represented [@Zheng2020-ji; @Zheng2021-cc; @Pan2024-hb; @Kille2023-px; @Golan2025-xf]. Routing directly on the raw minimizer value would therefore produce badly unbalanced partitions. - -Hashing the minimizer before routing redistributes this skewed distribution uniformly across partitions. This works reliably because the number of partition-index bits $p$ is chosen well below the number of bits available in the minimizer ($2m$): even with strong bias in the minimizer distribution, the hash has enough entropy margin to absorb it, provided the number of distinct minimizers actually observed is much larger than the number of partitions. - ## Parameter guidance +Even though $H$ already makes minimizer values well-distributed (see [Minimizer selection](minimizer_selection.md)), choosing $p$ well below the number of bits available in the minimizer ($2m$) leaves a comfortable entropy margin, provided the number of distinct minimizers actually observed is much larger than the number of partitions. + | Minimizer size $m$ | Minimizer bits ($2m$) | Typical partition-index bits $p$ | Partitions | |----|-----------|-----------|------------| | 9 | 18 | 6–8 | 64–256 | diff --git a/UserDocMD/theory/minimizer_selection.md b/UserDocMD/theory/minimizer_selection.md index 21940f34..430610ec 100644 --- a/UserDocMD/theory/minimizer_selection.md +++ b/UserDocMD/theory/minimizer_selection.md @@ -2,27 +2,58 @@ ## Definition -A **minimizer** of a kmer window is the m-mer ($m < k$) that is smallest, among all $k - m + 1$ overlapping m-mers in the window, under a chosen ordering. The minimizer is always taken in canonical form (lexicographic minimum of forward and reverse complement) so that selection is strand-independent. +The **minimizer** of a k-mer is the canonical form of the m-mer that is smallest according to a chosen ordering among the $k-m+1$ overlapping m-mers contained in the k-mer, with $m> 30) x ← x × 0xbf58476d1ce4e5b9 x ← x ⊕ (x >> 27) @@ -30,13 +61,36 @@ H(x): return x ⊕ (x >> 31) ``` -The XOR seed avoids the finalizer's fixed point at 0 ($\text{mix64}(0) = 0$), which would otherwise make an all-A m-mer (canonical value 0) win every window comparison. +The hash uses a fixed non-zero seed -## Partition routing is independent of minimizer selection +$$ \sigma=\texttt{0x9e3779b97f4a7c15}. $$ -The hash used to select a minimizer within a window (the minimum of several hash values) and the hash used to route a super-kmer to a storage partition are computed separately: +The choice of $\sigma$ is not arbitrary. Low-complexity m-mers (homopolymers and short tandem repeats) are disproportionately abundant in real genomes. If one of them happened to be the global argmin of $H$, it could therefore occur as the minimizer in far more windows than expected from the composition-uniform behavior described above. This is not a bias of the mixing function itself, but a consequence of the repeated occurrence of the same input in real sequence data. -- **Selection** uses $H$ applied to every candidate m-mer in the window, keeping the minimum. -- **Partition routing** recomputes $H$ on the single selected minimizer only, once its position is fixed. This is a hash of one specific value, not the minimum of several, so it is uniformly distributed and safe to use directly for routing. +In particular, with $\sigma=0$, the all-A m-mer is encoded as $0$, and -See [Partitioning and indexing architecture](indexing_architecture.md) for how the routing value is turned into a partition index. +$$ \operatorname{Mix13}(0)=0, $$ + +This makes it the global argmin of the hash function. The non-zero seed eliminates the particular pathological correspondence between the all-A m-mer and the zero output of the mixer. + +Exhaustive enumeration of all $4^m$ m-mers confirms that, with $\sigma=\texttt{0x9e3779b97f4a7c15}$, the global argmin is neither a homopolymer nor a periodic repeat for any tested \(m\): + +| $m$ | argmin (canonical) | decoded sequence | minimal period | +|---|---|---|---| +| 3 | 16 | `CAA` | 3 | +| 5 | 78 | `ACATG` | 5 | +| 7 | 5512 | `CCCGAGA` | 7 | +| 9 | 108760 | `CGGGATCGA` | 9 | +| 11 | 179014 | `AAGGTGTCACG` | 11 | +| 13 | 33759044 | `GAAATACTTCACA` | 13 | +| 15 | 29869313 | `AACTACTTACCAAAC` | 15 | + +When the minimal period of the sequence is $m$, as observed in the table above, the sequence is primitive: it cannot be represented as repetitions of a shorter sequence. + +## Partition routing + +A super-kmer's partition is the low $p$ bits of its minimizer's hash: + +$$\text{partition} = H(\text{minimizer}) \bmod 2^p$$ + +See [Partitioning and indexing architecture](indexing_architecture.md) for more details. diff --git a/scripts/mk_wiki.py b/scripts/mk_wiki.py new file mode 100755 index 00000000..b395027d --- /dev/null +++ b/scripts/mk_wiki.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +"""Convert the MkDocs user documentation (UserDocMD/) into a Gitea/Forgejo wiki tree. + +Gitea/Forgejo wikis use a different link syntax than MkDocs and cannot render +multi-line `$$...$$` display math (a known Goldmark/KaTeX limitation: the +parser only recognizes the delimiters when the whole block sits on one +physical line). This script rewrites the three constructs that would +otherwise break, flattens the tree onto page basenames, and copies everything +else verbatim: + + * Markdown links [text](path.md) -> [text](PageName) + Relative paths (including MkDocs root-relative `/path.md` links) are + resolved against the source tree and then flattened to a wiki page id; + only the target is rewritten, the label is left untouched. Gitea/Forgejo's + own `[[page|label]]` shortlink syntax is deliberately not used: it is + post-processed on rendered HTML text nodes, so it cannot match when the + label contains inline formatting (a `` `code` `` span splits the text + into separate DOM nodes around it) — confirmed empirically on the target + instance, where such labels never became links at all, and even a plain + label put target and display text in the position swapped from what the + MediaWiki-style `[[target|label]]` convention would suggest. A native + Markdown link is a real AST link node, so it has neither problem. + External links (http(s)://, mailto:) and links pointing outside the + source tree are left untouched. + * Pandoc citations [@key; @key] -> formatted, CSL-styled citation + Gitea/Forgejo's Goldmark does not support the [@key] citation syntax, and + a bare `(@key)` fallback is meaningless to a reader without the .bib file + open. Files containing citations are instead piped through `pandoc + --citeproc` with the project's own `references.bib` and `ecology-letters + .csl` — the same files `mkdocs-bibtex` uses for the real site — so the + wiki gets the identical citation style, e.g. `[@Zheng2020-ji]` becomes + `(Zheng et al. 2020)`, with a formatted bibliography (one `
` per entry, real title/journal/DOI) appended to that page. This + duplicates entries across pages that cite the same source rather than + centralizing them on one wiki page — an acceptable tradeoff for a + handful of references, and far simpler than inventing a cross-page + anchor scheme. Files with no `[@key]` at all skip pandoc entirely. + * Multi-line display math $$\\n...\\n$$ -> single-line $$...$$ + Collapsed onto one line (interior newlines replaced by spaces; `\\\\` + row separators are preserved) so Gitea/Forgejo's renderer recognizes it. + Inline `$...$` and already single-line `$$...$$` need no change and pass + through as-is. + +None of these rewrites touch fenced code blocks (```` ``` ```` or `~~~`, +including openings longer than three characters, e.g. ```` ```` ```` wrapping +a literal triple-backtick example). + +Wiki page ids are flat (no directories), so a source file living in a +sub-directory gets its directory path folded into the page id: a file at +`theory/encoding.md` becomes the page `theory-encoding`. Files at the root keep +their bare stem, except `index.md` which maps to `Home` — the page name +Gitea/Forgejo treats as the wiki's landing page. The same mapping is used to +resolve internal links, so cross-directory references stay consistent. If +folding still produces a collision the script refuses to write rather than +silently dropping a page. + +The `nav:` section of the MkDocs config is reproduced as a `_Sidebar.md` so the +wiki keeps the same top-level grouping (Home / Installation / Theory / Usage / +Formats / Architecture). Nav entries are usually `{label: path}` mappings, but +a bare `path` entry (no explicit label) is also honored: the label is taken +from the target file's first `# ` heading, falling back to a prettified +filename. The sidebar is static and shown on every page, which is the only +way Gitea/Forgejo expose a structured menu; the file is hidden from the page +list because its name starts with `_`. + +Usage: + mk_wiki.py OUTPUT [OPTIONS] + +Options: + --source DIR Source tree (default: UserDocMD next to this script's parent) + --config FILE MkDocs user config (default: /mkdocs-user.yml) + --csl FILE CSL citation style (default: /ecology-letters.csl) + --bib FILE BibTeX bibliography (default: /references.bib) + --no-sidebar do not generate _Sidebar.md + --verbose print every conversion +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +import yaml + + +class ConversionError(Exception): + """Raised when a source file cannot be converted faithfully.""" + + +def _make_lenient_loader() -> type[yaml.SafeLoader]: + """A SafeLoader that ignores unknown YAML tags instead of erroring. + + The MkDocs config uses `!!python/name:pymdownx.superfences.fence_code_format` + on a custom-fence entry; PyYAML cannot resolve that tag without an unsafe + loader, but the script only needs the `nav:` section, which is plain YAML. + Treating unknown tags as their literal scalar value is enough. + """ + + class _LenientLoader(yaml.SafeLoader): + pass + + def _unknown_constructor(loader: yaml.Loader, node: yaml.Node): # type: ignore[no-untyped-def] + if isinstance(node, yaml.ScalarNode): + return loader.construct_scalar(node) + if isinstance(node, yaml.SequenceNode): + return loader.construct_sequence(node) + if isinstance(node, yaml.MappingNode): + return loader.construct_mapping(node) + return None + + _LenientLoader.add_constructor(None, _unknown_constructor) + return _LenientLoader + + +def find_repo_root(start: Path) -> Path: + """Walk up until we find a directory containing UserDocMD/.""" + for candidate in [start.resolve(), *start.resolve().parents]: + if (candidate / "UserDocMD").is_dir(): + return candidate + return start.resolve() + + +def is_external(target: str) -> bool: + return bool(re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", target)) or target.startswith("mailto:") + + +def split_anchor(target: str) -> tuple[str, str]: + if "#" in target: + path, anchor = target.split("#", 1) + return path, "#" + anchor + return target, "" + + +def wiki_page_id(src_root: Path, abs_target: Path) -> str: + """Map a source file to its wiki page id. + + Wikis are flat, so the directory path is folded into the id: a file at + `theory/encoding.md` becomes `theory-encoding`, while a root file keeps its + bare stem, except `index.md` which maps to `Home` — the page name + Gitea/Forgejo renders as the wiki's landing page. + """ + rel = abs_target.relative_to(src_root.resolve()) + parts = list(rel.parts) + if len(parts) > 1: + # Drop the .md suffix from the last part, join the rest with '-'. + return "-".join([*parts[:-1], parts[-1][:-3]]) + stem = parts[0][:-3] + return "Home" if stem == "index" else stem + + +def resolve_wiki_path(src_root: Path, src_file: Path, target: str) -> str | None: + """Turn a relative markdown target into a wiki page id. + + Returns None for targets we should leave as plain Markdown links + (external URLs, non-markdown files, missing files, or targets outside the + source tree). + """ + path, anchor = split_anchor(target) + if is_external(path) or path == "": + return None + if path.startswith("/"): + # MkDocs root-relative link: resolved against the docs root, not the + # current file's directory (Path("a") / "/b" would silently discard + # "a" and resolve "/b" from the filesystem root, which is wrong here). + abs_target = (src_root / path.lstrip("/")).resolve() + else: + abs_target = (src_file.parent / path).resolve() + try: + abs_target.relative_to(src_root.resolve()) + except ValueError: + return None + if abs_target.suffix != ".md": + return None + if not abs_target.is_file(): + return None + return wiki_page_id(src_root, abs_target) + anchor + + +LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)\s]+)\)") +FENCE_RE = re.compile(r"^(`{3,}|~{3,})") +# Pandoc citation syntax, e.g. [@Zheng2020-ji; @Golan2025-xf] — just used to +# detect whether a file has any citation worth running through pandoc at all. +HAS_CITATION_RE = re.compile(r"\[@[^\]]+\]") + +# pandoc's markdown writer must be told, relative to its own defaults, to: +# -smart leave dashes/quotes as written (no em-dash -> "---") +# -fenced_divs emit the bibliography as raw HTML
(Goldmark +# has no ``:::`` fenced-div extension) +# -simple_tables, -multiline_tables, -grid_tables, +pipe_tables +# keep GFM pipe-table syntax, the only kind Goldmark +# parses, instead of pandoc's space-aligned tables +# +fenced_code_blocks (pandoc still emits an indented block for a fence +# with no language tag regardless of this extension — +# such fences need a language hint in the source) +# -header_attributes Goldmark has no `{#id .class}` heading-attribute +# syntax; without this the "Bibliography" heading pandoc +# adds (see BIBLIOGRAPHY_TITLE below) would render with +# a literal `{#bibliography .unnumbered}` after it +PANDOC_MARKDOWN_FORMAT = ( + "markdown-smart-fenced_divs-simple_tables-multiline_tables-grid_tables" + "-header_attributes+pipe_tables+fenced_code_blocks" +) + +# Title pandoc gives the generated bibliography section (always emitted as a +# level-1 `# ` heading, with no option to pick the level) — demoted to `##` +# below to match the rest of this doc set's heading hierarchy. +BIBLIOGRAPHY_TITLE = "Bibliography" + + +class FenceState: + """Tracks whether the current line is inside a fenced code block. + + A closing fence must use the same character and be at least as long as + the opening one (CommonMark rule) — this is what lets a source file wrap + a literal ``` example inside a longer ```` fence. + """ + + def __init__(self) -> None: + self.marker = "" + + def toggle(self, line: str) -> bool: + """Update state for a fence delimiter line; return True if it is one.""" + m = FENCE_RE.match(line.rstrip("\n")) + if not m: + return False + run = m.group(1) + if not self.marker: + self.marker = run + elif run[0] == self.marker[0] and len(run) >= len(self.marker): + self.marker = "" + return True + + @property + def active(self) -> bool: + return bool(self.marker) + + +def convert_links(text: str, src_root: Path, src_file: Path, verbose: bool) -> str: + out_lines: list[str] = [] + fence = FenceState() + + for line in text.splitlines(keepends=True): + if fence.toggle(line) or fence.active: + out_lines.append(line) + continue + + def repl(m: re.Match) -> str: + label, target = m.group(1), m.group(2) + wiki = resolve_wiki_path(src_root, src_file, target) + if wiki is None: + return m.group(0) # external or non-md: leave as Markdown + if verbose: + print(f" {src_file.relative_to(src_root)}: [{label}]({target}) -> [{label}]({wiki})") + return f"[{label}]({wiki})" + + out_lines.append(LINK_RE.sub(repl, line)) + + return "".join(out_lines) + + +def convert_citations( + text: str, src_root: Path, src_file: Path, csl_path: Path, bib_path: Path, verbose: bool +) -> str: + """Render Pandoc `[@key; @key]` citations with the project's real CSL style. + + Skips files with no citation entirely, so a missing `--csl`/`--bib` or a + missing `pandoc` binary is only an error for files that actually need it. + """ + if not HAS_CITATION_RE.search(text): + return text + + if not csl_path.is_file(): + raise ConversionError(f"{src_file.relative_to(src_root)}: CSL style not found: {csl_path}") + if not bib_path.is_file(): + raise ConversionError(f"{src_file.relative_to(src_root)}: bibliography not found: {bib_path}") + + try: + result = subprocess.run( + [ + "pandoc", + "--citeproc", + f"--csl={csl_path}", + f"--bibliography={bib_path}", + "--wrap=none", + f"--metadata=link-citations=true", + f"--metadata=reference-section-title={BIBLIOGRAPHY_TITLE}", + "-f", + "markdown-smart", + "-t", + PANDOC_MARKDOWN_FORMAT, + ], + input=text, + capture_output=True, + text=True, + ) + except FileNotFoundError: + raise ConversionError( + f"{src_file.relative_to(src_root)}: 'pandoc' not found on PATH, needed to render citations" + ) from None + + if result.returncode != 0: + raise ConversionError( + f"{src_file.relative_to(src_root)}: pandoc failed to render citations:\n{result.stderr}" + ) + + # pandoc always emits the bibliography title as a level-1 heading; demote + # it to `##` to match the rest of this doc set's hierarchy. + out = result.stdout.replace(f"\n# {BIBLIOGRAPHY_TITLE}\n", f"\n## {BIBLIOGRAPHY_TITLE}\n") + + if verbose: + n = len(HAS_CITATION_RE.findall(text)) + print(f" {src_file.relative_to(src_root)}: rendered {n} citation group(s) via pandoc/{csl_path.name}") + + return out + + +def convert_math(text: str, src_root: Path, src_file: Path, verbose: bool) -> str: + """Collapse multi-line `$$...$$` display blocks onto a single line. + + Gitea/Forgejo's math renderer only recognizes `$$...$$` when it sits on + one physical line; a block opened and closed on their own bare `$$` line + (the natural Pandoc/MkDocs style) renders as literal text otherwise. + Interior line breaks are replaced by spaces (insignificant in math mode); + `\\\\` row separators inside `aligned`/`cases` environments are untouched. + Already single-line `$$...$$` and inline `$...$` need no change. + """ + lines = text.split("\n") + out: list[str] = [] + fence = FenceState() + i = 0 + n = len(lines) + while i < n: + line = lines[i] + if fence.toggle(line) or fence.active: + out.append(line) + i += 1 + continue + + if line.strip() == "$$": + body: list[str] = [] + i += 1 + closed = False + while i < n: + if lines[i].strip() == "$$": + closed = True + i += 1 + break + body.append(lines[i].strip()) + i += 1 + if not closed: + raise ConversionError( + f"{src_file.relative_to(src_root)}: unterminated $$ display math block" + ) + collapsed = "$$" + " ".join(part for part in body if part) + "$$" + if verbose: + print(f" {src_file.relative_to(src_root)}: collapsed {len(body)}-line $$ block") + out.append(collapsed) + continue + + out.append(line) + i += 1 + + return "\n".join(out) + + +def convert_file( + src_root: Path, src_file: Path, csl_path: Path, bib_path: Path, verbose: bool +) -> str: + text = src_file.read_text(encoding="utf-8") + # Citations first, before pandoc has seen any of our own wiki-specific + # rewrites: it needs to parse plain, unmodified Markdown. + text = convert_citations(text, src_root, src_file, csl_path, bib_path, verbose) + text = convert_math(text, src_root, src_file, verbose) + text = convert_links(text, src_root, src_file, verbose) + return text + + +def _default_label(src_root: Path, rel_path: str) -> str: + """Derive a nav label for a bare `path.md` entry (no explicit label). + + Mirrors MkDocs' own fallback: use the page's first `# ` heading if it has + one, otherwise prettify the filename. + """ + target = src_root / rel_path + if target.is_file(): + for line in target.read_text(encoding="utf-8").splitlines(): + if line.startswith("# "): + return line[2:].strip() + return Path(rel_path).stem.replace("_", " ").replace("-", " ").title() + + +def build_sidebar(src_root: Path, config_path: Path) -> str | None: + """Render the MkDocs `nav:` section as a Gitea/Forgejo `_Sidebar.md`. + + Returns None if the config is missing or has no usable `nav:` section. + Each nav entry is either a bare `path` string or a `{label: path}` pair + (or a nested list for a group); the path is mapped through `wiki_page_id` + so the sidebar links to the flattened page ids. + """ + if not config_path.is_file(): + print(f"warning: config not found, skipping sidebar: {config_path}") + return None + + try: + with config_path.open(encoding="utf-8") as fh: + cfg = yaml.safe_load(fh) + except yaml.YAMLError: + # The MkDocs config carries `!!python/name:...` tags that SafeLoader + # cannot resolve; fall back to a lenient loader that treats unknown + # tags as plain scalars so the `nav:` section is still readable. + try: + with config_path.open(encoding="utf-8") as fh: + cfg = yaml.load(fh, Loader=_make_lenient_loader()) + except yaml.YAMLError as exc2: + print(f"warning: could not parse {config_path}: {exc2}") + return None + + nav = cfg.get("nav") if isinstance(cfg, dict) else None + if not nav: + print(f"warning: no `nav:` section in {config_path}, skipping sidebar") + return None + + def link_line(depth: int, rel_path: str, label: str) -> str: + page = wiki_page_id(src_root, (src_root / rel_path).resolve()) + return f"{' ' * depth}- [{label}]({page})" + + def render_entry(entry: object, depth: int) -> list[str]: + if isinstance(entry, str): + return [link_line(depth, entry, _default_label(src_root, entry))] + if not isinstance(entry, dict): + return [] + lines: list[str] = [] + for label, target in entry.items(): + if isinstance(target, str): + lines.append(link_line(depth, target, label)) + elif isinstance(target, list): + # Headings must never be indented: CommonMark treats a line + # indented by 4+ spaces as a code block, not an ATX heading, + # which a depth-scaled indent would trigger from depth 2 on. + lines.append(f"## {label}") + lines.extend(render_subtree(target, depth + 1)) + return lines + + def render_subtree(nodes: list[object], depth: int) -> list[str]: + out: list[str] = [] + for node in nodes: + out.extend(render_entry(node, depth)) + return out + + body = render_subtree(nav, 0) + if not body: + return None + + return "# Wiki sidebar\n\n" + "\n".join(body) + "\n" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("output", help="destination wiki directory (created if missing)") + ap.add_argument("--source", default=None, help="source Markdown tree (default: /UserDocMD)") + ap.add_argument("--config", default=None, + help="MkDocs user config (default: /mkdocs-user.yml)") + ap.add_argument("--csl", default=None, + help="CSL citation style (default: /ecology-letters.csl)") + ap.add_argument("--bib", default=None, + help="BibTeX bibliography (default: /references.bib)") + ap.add_argument("--no-sidebar", action="store_true", help="do not generate _Sidebar.md") + ap.add_argument("--verbose", action="store_true", help="print every conversion") + args = ap.parse_args() + + script_dir = Path(__file__).resolve().parent + repo_root = find_repo_root(script_dir) + src_root = Path(args.source).resolve() if args.source else (repo_root / "UserDocMD") + out_root = Path(args.output).resolve() + config_path = Path(args.config).resolve() if args.config else (repo_root / "mkdocs-user.yml") + csl_path = Path(args.csl).resolve() if args.csl else (src_root / "ecology-letters.csl") + bib_path = Path(args.bib).resolve() if args.bib else (src_root / "references.bib") + + if not src_root.is_dir(): + print(f"error: source tree not found: {src_root}", file=sys.stderr) + return 2 + + md_files = sorted(p for p in src_root.rglob("*.md") if p.is_file()) + if not md_files: + print(f"error: no Markdown files under {src_root}", file=sys.stderr) + return 2 + + # A wiki is flat, so the destination filename is the source path folded into + # a page id. Detect collisions up front so a silent overwrite does not drop + # a page. + by_name: dict[str, list[Path]] = {} + for src in md_files: + by_name.setdefault(wiki_page_id(src_root, src), []).append(src) + collisions = {name: paths for name, paths in by_name.items() if len(paths) > 1} + if collisions: + for name, paths in sorted(collisions.items()): + print(f"error: wiki page id collision '{name}':", file=sys.stderr) + for p in paths: + print(f" {p.relative_to(src_root)}", file=sys.stderr) + return 2 + + out_root.mkdir(parents=True, exist_ok=True) + written = 0 + for src in md_files: + dst = out_root / (wiki_page_id(src_root, src) + ".md") + try: + content = convert_file(src_root, src, csl_path, bib_path, args.verbose) + except ConversionError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + dst.write_text(content, encoding="utf-8") + written += 1 + + if not args.no_sidebar: + sidebar = build_sidebar(src_root, config_path) + if sidebar is not None: + (out_root / "_Sidebar.md").write_text(sidebar, encoding="utf-8") + written += 1 + + print(f"converted {written} Markdown file(s) from {src_root} -> {out_root}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) -- 2.54.1 From e715528d344bf71a5eff6670d57f5879361db4bc Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Mon, 14 Sep 2026 13:06:14 +0200 Subject: [PATCH 02/10] Refactor documentation for kmer indexing and phylogenetic models This commit updates documentation across the board, introducing definitions for kmer constraints, super-kmers, k-mer set distances, SNP models, and sampling design principles. It also clarifies the index construction pipeline and the underlying mathematical models used for genome pair calibration. --- UserDocMD/architecture.md | 4 +- UserDocMD/formats/index_layout.md | 4 +- UserDocMD/index.md | 8 +- UserDocMD/references.bib | 221 ++++++++ .../theory/{ => kmer_indexing}/encoding.md | 6 +- .../{ => kmer_indexing}/entropy_filter.md | 0 .../indexing_architecture.md | 6 +- UserDocMD/theory/kmer_indexing/kmers.md | 6 + .../minimizer_selection.md | 20 +- UserDocMD/theory/kmer_indexing/superkmers.md | 15 + UserDocMD/theory/kmers_and_superkmers.md | 24 - .../kmer_based/kmer_set_distances.md | 140 +++++ .../phylogeny/snp_based/central_snp_model.md | 85 +++ .../snp_based/gamma_rate_heterogeneity.md | 27 + .../phylogeny/snp_based/sampling_design.md | 24 + .../phylogeny/snp_based/sankoff_model.md | 22 + UserDocMD/usage/phylo.md | 134 +---- mkdocs-user.yml | 20 +- scripts/mk_wiki.py | 528 ------------------ 19 files changed, 582 insertions(+), 712 deletions(-) rename UserDocMD/theory/{ => kmer_indexing}/encoding.md (60%) rename UserDocMD/theory/{ => kmer_indexing}/entropy_filter.md (100%) rename UserDocMD/theory/{ => kmer_indexing}/indexing_architecture.md (83%) create mode 100644 UserDocMD/theory/kmer_indexing/kmers.md rename UserDocMD/theory/{ => kmer_indexing}/minimizer_selection.md (88%) create mode 100644 UserDocMD/theory/kmer_indexing/superkmers.md delete mode 100644 UserDocMD/theory/kmers_and_superkmers.md create mode 100644 UserDocMD/theory/phylogeny/kmer_based/kmer_set_distances.md create mode 100644 UserDocMD/theory/phylogeny/snp_based/central_snp_model.md create mode 100644 UserDocMD/theory/phylogeny/snp_based/gamma_rate_heterogeneity.md create mode 100644 UserDocMD/theory/phylogeny/snp_based/sampling_design.md create mode 100644 UserDocMD/theory/phylogeny/snp_based/sankoff_model.md delete mode 100755 scripts/mk_wiki.py diff --git a/UserDocMD/architecture.md b/UserDocMD/architecture.md index a3da8b95..2e7a205b 100644 --- a/UserDocMD/architecture.md +++ b/UserDocMD/architecture.md @@ -1,6 +1,6 @@ # Architecture notes for advanced use -This page describes execution-level behavior relevant to sizing and running `obikmer` on large datasets or multi-socket machines. It complements the [index format](formats/index_layout.md) and [theory](theory/indexing_architecture.md) pages. +This page describes execution-level behavior relevant to sizing and running `obikmer` on large datasets or multi-socket machines. It complements the [index format](formats/index_layout.md) and [theory](theory/kmer_indexing/indexing_architecture.md) pages. ## Sequence invariant @@ -8,7 +8,7 @@ Every input sequence is treated purely as a compact representation of a set of o - Only the `A`/`C`/`G`/`T` alphabet (case-insensitive) is recognized; a sequence is cut at any other character (including IUPAC ambiguity codes), so runs containing them are not represented in the index. - Sequences are internally processed in chunks of at most 256 nucleotides; a chunk shorter than k is dropped. This is invisible to the user beyond the ACGT-only, minimum-length-k constraints above. -- Kmers are always handled in canonical form (see [DNA encoding](theory/encoding.md)), so the tool is strand-agnostic throughout: a kmer and its reverse complement are always the same entry. +- Kmers are always handled in canonical form (see [DNA encoding](theory/kmer_indexing/encoding.md)), so the tool is strand-agnostic throughout: a kmer and its reverse complement are always the same entry. ## Index dimensioning diff --git a/UserDocMD/formats/index_layout.md b/UserDocMD/formats/index_layout.md index b590364f..46303b39 100644 --- a/UserDocMD/formats/index_layout.md +++ b/UserDocMD/formats/index_layout.md @@ -2,9 +2,9 @@ ## Construction pipeline -Building an index ([`index`](../usage/index_command.md)) proceeds through a fixed sequence of phases, each operating independently per partition (see [Partitioning and indexing architecture](../theory/indexing_architecture.md)): +Building an index ([`index`](../usage/index_command.md)) proceeds through a fixed sequence of phases, each operating independently per partition (see [Partitioning and indexing architecture](../theory/kmer_indexing/indexing_architecture.md)): -1. **Scatter.** A single streaming pass over the input. Each sequence fragment is cut at non-ACGT bases, passed through the low-complexity entropy filter (see [Low-complexity kmer filter](../theory/entropy_filter.md)), and any resulting segment shorter than k is dropped. Surviving segments are decomposed into super-kmers, canonicalized, and routed by `hash(minimizer) mod n_partitions` into one file per partition. +1. **Scatter.** A single streaming pass over the input. Each sequence fragment is cut at non-ACGT bases, passed through the low-complexity entropy filter (see [Low-complexity kmer filter](../theory/kmer_indexing/entropy_filter.md)), and any resulting segment shorter than k is dropped. Surviving segments are decomposed into super-kmers, canonicalized, and routed by `hash(minimizer) mod n_partitions` into one file per partition. 2. **Dereplication.** Within each partition, identical super-kmer sequences are merged and their occurrence counts summed. This count is per super-kmer, not per kmer — a kmer's true abundance is the sum of the counts of every super-kmer containing it. 3. **Exact counting.** Every kmer in every dereplicated super-kmer is enumerated and its exact total count computed. A per-genome kmer frequency spectrum is produced at this stage. 4. **Quorum filtering.** Kmers outside the `--min-abundance`/`--max-abundance` range are dropped, and super-kmers are recompacted around the surviving kmer set. diff --git a/UserDocMD/index.md b/UserDocMD/index.md index aa1fcc7b..b050433d 100644 --- a/UserDocMD/index.md +++ b/UserDocMD/index.md @@ -6,11 +6,11 @@ All functionality is exposed through a single binary, `obikmer`, organized as su ## Core principles -- Kmers are of fixed, odd length $k$, chosen at index-construction time in the range $[11, 31]$ (see [Kmers and super-kmers](theory/kmers_and_superkmers.md)). -- Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see [DNA encoding](theory/encoding.md)). +- Kmers are of fixed, odd length $k$, chosen at index-construction time in the range $[11, 31]$ (see [Kmers](theory/kmer_indexing/kmers.md)). +- Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see [DNA encoding](theory/kmer_indexing/encoding.md)). - Kmers are handled in **canonical form** ($\text{canonical}(kmer) = \min(kmer, \text{revcomp}(kmer))$), making counting strand-independent. -- Sequences are decomposed into **super-kmers** before storage, anchored on a hash-selected **minimizer** (see [Minimizer selection](theory/minimizer_selection.md)), then routed to one of several **partitions** for parallel, memory-bounded processing (see [Partitioning and indexing architecture](theory/indexing_architecture.md)). -- Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see [Low-complexity kmer filter](theory/entropy_filter.md)). +- Sequences are decomposed into **super-kmers** before storage, anchored on a hash-selected **minimizer** (see [Minimizer selection](theory/kmer_indexing/minimizer_selection.md)), then routed to one of several **partitions** for parallel, memory-bounded processing (see [Partitioning and indexing architecture](theory/kmer_indexing/indexing_architecture.md)). +- Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see [Low-complexity kmer filter](theory/kmer_indexing/entropy_filter.md)). ## Commands diff --git a/UserDocMD/references.bib b/UserDocMD/references.bib index ce2b810f..1dbf1ef4 100644 --- a/UserDocMD/references.bib +++ b/UserDocMD/references.bib @@ -345,3 +345,224 @@ url = {https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html}, urldate = {2026-09-12} } + + @ARTICLE{Hamming1950-cw, + title = "{Error detecting and error correcting codes}", + author = "Hamming, R W", + journal = "The Bell System Technical Journal", + publisher = "Institute of Electrical and Electronics Engineers (IEEE)", + volume = 29, + number = 2, + pages = "147--160", + abstract = "The author was led to the study given in this paper from a + consideration of large scale computing machines in which a large + number of operations must be performed without a single error in + the end result. This problem of “doing things right” on a large + scale is not essentially new; in a telephone central office, for + example, a very large number of operations are performed while + the errors leading to wrong numbers are kept well under control, + though they have not been completely eliminated. This has been + achieved, in part, through the use of self-checking circuits. The + occasional failure that escapes routine checking is still + detected by the customer and will, if it persists, result in + customer complaint, while if it is transient it will produce only + occasional wrong numbers. At the same time the rest of the + central office functions satisfactorily. In a digital computer, + on the other hand, a single failure usually means the complete + failure, in the sense that if it is detected no more computing + can be done until the failure is located and corrected, while if + it escapes detection then it invalidates all subsequent + operations of the machine. Put in other words, in a telephone + central office there are a number of parallel paths which are + more or less independent of each other; in a digital machine + there is usually a single long path which passes through the same + piece of equipment many, many times before the answer is + obtained.", + month = apr, + year = 1950, + url = "http://dx.doi.org/10.1002/j.1538-7305.1950.tb00463.x", + doi = "10.1002/j.1538-7305.1950.tb00463.x", + issn = "0005-8580" + } + + @ARTICLE{Ondov2016-dq, + title = "{Mash: fast genome and metagenome distance estimation using + {MinHash}}", + author = "Ondov, Brian D and Treangen, Todd J and Melsted, Páll and + Mallonee, Adam B and Bergman, Nicholas H and Koren, Sergey and + Phillippy, Adam M", + journal = "Genome biology", + volume = 17, + number = 1, + pages = 132, + abstract = "Mash extends the MinHash dimensionality-reduction technique to + include a pairwise mutation distance and P value significance + test, enabling the efficient clustering and search of massive + sequence collections. Mash reduces large sequences and sequence + sets to small, representative sketches, from which global mutation + distances can be rapidly estimated. We demonstrate several use + cases, including the clustering of all 54,118 NCBI RefSeq genomes + in 33 CPU h; real-time database search using assembled or + unassembled Illumina, Pacific Biosciences, and Oxford Nanopore + data; and the scalable clustering of hundreds of metagenomic + samples by composition. Mash is freely released under a BSD + license ( https://github.com/marbl/mash ).", + month = jun, + year = 2016, + url = "http://dx.doi.org/10.1186/s13059-016-0997-x", + keywords = "Alignment; Comparative genomics; Genomic distance; Metagenomics; + Nanopore; Sequencing", + doi = "10.1186/s13059-016-0997-x", + pmc = "PMC4915045", + pmid = 27323842, + issn = "1465-6906", + language = "en" + } + @ARTICLE{Bray1957-vv, + title = "{An ordination of the upland forest communities of southern + Wisconsin}", + author = "Bray, J Roger and Curtis, J T", + journal = "Ecological Monographs", + publisher = "Wiley", + volume = 27, + number = 4, + pages = "325--349", + month = oct, + year = 1957, + url = "http://dx.doi.org/10.2307/1942268", + doi = "10.2307/1942268", + issn = "0012-9615,1557-7015", + language = "en" + } + @ARTICLE{Gower1986-ri, + title = "{Metric and Euclidean properties of dissimilarity coefficients}", + author = "Gower, J C and Legendre, P", + journal = "Journal of Classification", + publisher = "Springer Science and Business Media LLC", + volume = 3, + number = 1, + pages = "5--48", + abstract = "We assemble here properties of certain dissimilarity coefficients + and are specially concerned with their metric and Euclidean + status. No attempt is made to be exhaustive as far as + coefficients are concerned, but certain mathematical results that + we have found useful are presented and should help establish + similar properties for other coefficients. The response to + different types of data is investigated, leading to guidance on + the choice of an appropriate coefficient.", + month = mar, + year = 1986, + url = "http://dx.doi.org/10.1007/bf01896809", + doi = "10.1007/bf01896809", + issn = "0176-4268,1432-1343", + language = "en" + } + @ARTICLE{Legendre2001-cq, + title = "{Ecologically meaningful transformations for ordination of species + data}", + author = "Legendre, Pierre and Gallagher, Eugene D", + journal = "Oecologia", + volume = 129, + number = 2, + pages = "271--280", + abstract = "This paper examines how to obtain species biplots in unconstrained + or constrained ordination without resorting to the Euclidean + distance [used in principal-component analysis (PCA) and + redundancy analysis (RDA)] or the chi-square distance [preserved + in correspondence analysis (CA) and canonical correspondence + analysis (CCA)] which are not always appropriate for the analysis + of community composition data. To achieve this goal, + transformations are proposed for species data tables. They allow + ecologists to use ordination methods such as PCA and RDA, which + are Euclidean-based, for the analysis of community data, while + circumventing the problems associated with the Euclidean distance, + and avoiding CA and CCA which present problems of their own in + some cases. This allows the use of the original (transformed) + species data in RDA carried out to test for relationships with + explanatory variables (i.e. environmental variables, or factors of + a multifactorial analysis-of-variance model); ecologists can then + draw biplots displaying the relationships of the species to the + explanatory variables. Another application allows the use of + species data in other methods of multivariate data analysis which + optimize a least-squares loss function; an example is K-means + partitioning.", + month = oct, + year = 2001, + url = "http://dx.doi.org/10.1007/s004420100716", + keywords = "Biplot diagram; Canonical correspondence analysis; Correspondence + analysis; Principal-component analysis; Redundancy analysis", + doi = "10.1007/s004420100716", + pmid = 28547606, + issn = "0029-8549,1432-1939", + language = "en" + } + @ARTICLE{Anderson2011-rq, + title = "{Navigating the multiple meanings of β diversity: a roadmap for + the practicing ecologist}", + author = "Anderson, Marti J and Crist, Thomas O and Chase, Jonathan M and + Vellend, Mark and Inouye, Brian D and Freestone, Amy L and + Sanders, Nathan J and Cornell, Howard V and Comita, Liza S and + Davies, Kendi F and Harrison, Susan P and Kraft, Nathan J B and + Stegen, James C and Swenson, Nathan G", + journal = "Ecology Letters", + publisher = "Wiley", + volume = 14, + number = 1, + pages = "19--28", + abstract = "A recent increase in studies of β diversity has yielded a + confusing array of concepts, measures and methods. Here, we + provide a roadmap of the most widely used and ecologically + relevant approaches for analysis through a series of mission + statements. We distinguish two types of β diversity: directional + turnover along a gradient vs. non-directional variation. + Different measures emphasize different properties of ecological + data. Such properties include the degree of emphasis on + presence/absence vs. relative abundance information and the + inclusion vs. exclusion of joint absences. Judicious use of + multiple measures in concert can uncover the underlying nature of + patterns in β diversity for a given dataset. A case study of + Indonesian coral assemblages shows the utility of a multi-faceted + approach. We advocate careful consideration of relevant + questions, matched by appropriate analyses. The rigorous + application of null models will also help to reveal potential + processes driving observed patterns in β diversity.", + month = jan, + year = 2011, + url = "http://dx.doi.org/10.1111/j.1461-0248.2010.01552.x", + doi = "10.1111/j.1461-0248.2010.01552.x", + pmid = 21070562, + issn = "1461-023X,1461-0248", + language = "en" + } + @ARTICLE{Jaccard1901-mx, + title = "{Étude comparative de la distribution florale dans une portion + des Alpes et du Jura}", + author = "Jaccard, Paul", + journal = "Bulletin de la Société vaudoise des sciences naturelles", + publisher = "Imprimerie Corbaz \& Comp.", + volume = 37, + number = 142, + pages = "547--579", + abstract = "Schweizer Zeitschriften online - Ein Service der ETH-Bibliothek. + Revues suisses en ligne - Un service de ETH Library. Swiss + journals online - A service by ETH Library.", + year = 1901, + url = "http://dx.doi.org/10.5169/seals-266450", + doi = "10.5169/seals-266450", + issn = "0037-9603", + language = "de" + } + @ARTICLE{Kosub2019-xt, + title = "{A note on the triangle inequality for the Jaccard distance}", + author = "Kosub, Sven", + journal = "Pattern Recognition Letters", + publisher = "Elsevier BV", + volume = 120, + pages = "36--38", + month = apr, + year = 2019, + url = "http://dx.doi.org/10.1016/j.patrec.2018.12.007", + doi = "10.1016/j.patrec.2018.12.007", + issn = "0167-8655,1872-7344", + language = "en" + } diff --git a/UserDocMD/theory/encoding.md b/UserDocMD/theory/kmer_indexing/encoding.md similarity index 60% rename from UserDocMD/theory/encoding.md rename to UserDocMD/theory/kmer_indexing/encoding.md index 80230a92..a9fcefb7 100644 --- a/UserDocMD/theory/encoding.md +++ b/UserDocMD/theory/kmer_indexing/encoding.md @@ -21,8 +21,8 @@ Reverse complement is computed by bit manipulation directly on the packed word, ## Canonical form -The canonical form of a kmer is the lexicographic minimum of the kmer and its reverse complement: +The canonical form of a packed 2-bit sequence is the lexicographic minimum of the sequence and its reverse complement: -$$\text{canonical}(kmer) = \min\big(kmer,\ \text{revcomp}(kmer)\big)$$ +$$\text{canonical}(x) = \min\big(x,\ \text{revcomp}(x)\big)$$ -Using the canonical form halves the kmer space and makes counting strand-independent: a kmer and its reverse complement are always treated as the same entity, regardless of which DNA strand was sequenced. +This is a single operation on the packed representation, so it applies identically regardless of what `x` represents — a kmer (see [Kmers](kmers.md)), an m-mer (see [Minimizer selection](minimizer_selection.md)), or a super-kmer (see [Super-kmers](superkmers.md)). Using the canonical form halves the relevant space and makes counting/selection strand-independent: a sequence and its reverse complement are always treated as the same entity, regardless of which DNA strand was sequenced. diff --git a/UserDocMD/theory/entropy_filter.md b/UserDocMD/theory/kmer_indexing/entropy_filter.md similarity index 100% rename from UserDocMD/theory/entropy_filter.md rename to UserDocMD/theory/kmer_indexing/entropy_filter.md diff --git a/UserDocMD/theory/indexing_architecture.md b/UserDocMD/theory/kmer_indexing/indexing_architecture.md similarity index 83% rename from UserDocMD/theory/indexing_architecture.md rename to UserDocMD/theory/kmer_indexing/indexing_architecture.md index 43c72fab..f95514fa 100644 --- a/UserDocMD/theory/indexing_architecture.md +++ b/UserDocMD/theory/kmer_indexing/indexing_architecture.md @@ -4,13 +4,15 @@ An index is split into a fixed number of **partitions**, each handling an indepe ## Routing -The canonical minimizer of a super-kmer (see [Minimizer selection](minimizer_selection.md)) is hashed to produce a $p$-bit routing value that selects the destination partition: +The canonical minimizer of a super-kmer (see [Minimizer selection](minimizer_selection.md)) is hashed to produce a $p$-bit routing value that selects the destination partition — the low $p$ bits of the minimizer's hash: ``` canonical minimizer → hash(minimizer) → p-bit value → partition index ``` -Within a partition, kmers are indexed as plain values via a minimal perfect hash function (see [On-disk storage](../formats/index_layout.md)); the minimizer plays no further role once a super-kmer has reached its partition. +$$\text{partition} = H(\text{minimizer}) \bmod 2^p$$ + +Within a partition, kmers are indexed as plain values via a minimal perfect hash function (see [On-disk storage](../../formats/index_layout.md)); the minimizer plays no further role once a super-kmer has reached its partition. ## Parameter guidance diff --git a/UserDocMD/theory/kmer_indexing/kmers.md b/UserDocMD/theory/kmer_indexing/kmers.md new file mode 100644 index 00000000..dae3acd7 --- /dev/null +++ b/UserDocMD/theory/kmer_indexing/kmers.md @@ -0,0 +1,6 @@ +# Kmers + +A **kmer** is a DNA subsequence of fixed length $k$. Two constraints apply to $k$, both enforced when a command starts (an invalid value exits immediately with an error): + +- $k \in [11, 31]$: long enough to be specific, short enough to fit in a single 64-bit word at 2 bits/base ($k \le 32$ is the hard limit; $k < 11$ gives insufficient specificity). +- $k$ **is odd**: an odd-length sequence can never equal its own reverse complement, so the two orientations of any kmer are always distinct. This is required for the canonical form (see [DNA encoding](encoding.md)) to be well defined. diff --git a/UserDocMD/theory/minimizer_selection.md b/UserDocMD/theory/kmer_indexing/minimizer_selection.md similarity index 88% rename from UserDocMD/theory/minimizer_selection.md rename to UserDocMD/theory/kmer_indexing/minimizer_selection.md index 430610ec..3dec6fe4 100644 --- a/UserDocMD/theory/minimizer_selection.md +++ b/UserDocMD/theory/kmer_indexing/minimizer_selection.md @@ -2,15 +2,7 @@ ## Definition -The **minimizer** of a k-mer is the canonical form of the m-mer that is smallest according to a chosen ordering among the $k-m+1$ overlapping m-mers contained in the k-mer, with $m_entropy.csv`: per-family Shannon entropy, one row per family, full unsampled scan | | `--pseudo-alignment` | Write `_alignment.fasta`: a SNP-only pseudo-alignment. Requires `--subsample N` | -A family is eligible for a genome pair $(i,j)$ only if both genomes carry exactly one of its observed forms (single-copy, unambiguous). - ### `--sibling-stats` `_siblings.csv` — family size = number of distinct central bases observed at a family (1-4). @@ -219,7 +103,7 @@ A `--sankoff`-family run and a `snp-*` `--distance` run share the same cached sa If a run using `--session` is interrupted (crash, kill, `Ctrl-C`), the next run against the same `--session DIR` resumes from the last automatic checkpoint (roughly every 8 partitions'-worth of sampling progress) instead of starting over. The resumed run's sample is **not** guaranteed to be identical to what an uninterrupted run would have produced past that checkpoint — each process draws its own independent random sequence, same as any two separate invocations do — but nothing already checkpointed is lost, and no work needs redoing beyond that point. -Without `--subsample`, every variable family (family size ≥ 2) is used. With `--subsample N`, roughly `N` families are kept instead, drawn in proportion to how many candidate families each part of the index actually holds, so the sample stays representative of the whole index. If the index has fewer than `N` candidate families, `--subsample` has no effect. +Without `--subsample`, every variable family (family size ≥ 2) is used. With `--subsample N`, roughly `N` families are kept instead (see [Sampling design](../theory/phylogeny/snp_based/sampling_design.md) for how the draw stays representative of the whole index). If the index has fewer than `N` candidate families, `--subsample` has no effect. ### `--shannon`: measuring how informative a family is @@ -238,7 +122,7 @@ Run with `--subsample N --shannon` to get a bounded diagnostic sample instead of ### `--entropy MU` / `--entropy-sd SIGMA`: biasing the sample toward informative families -By default, `--subsample` draws families uniformly. With `--entropy`/`--entropy-sd`, each family's chance of being kept is instead weighted by how close its own entropy (`entropy15`) is to `MU`, using a Gaussian curve of width `SIGMA` — no hard cutoff. The filter activates as soon as either flag is given; the other defaults to `1.0`/`0.5`. Combine with `--subsample N` (expect somewhat fewer than `N` families kept in practice) or use alone (a soft filter over the whole index, no size target). +By default, `--subsample` draws families uniformly. With `--entropy`/`--entropy-sd`, each family's chance of being kept is instead weighted by how close its own entropy (`entropy15`) is to `MU`, using a Gaussian curve of width `SIGMA` — no hard cutoff (see [Sampling design](../theory/phylogeny/snp_based/sampling_design.md) for the weighting formula). The filter activates as soon as either flag is given; the other defaults to `1.0`/`0.5`. Combine with `--subsample N` (expect somewhat fewer than `N` families kept in practice) or use alone (a soft filter over the whole index, no size target). The first `phylo` run on a given index that uses `--entropy`/`--entropy-sd` pays a one-time extra cost (every candidate family's entropy is computed once and saved alongside the index); later runs, even with different `MU`/`SIGMA`, reuse that saved data. @@ -257,7 +141,7 @@ The first `phylo` run on a given index that uses `--entropy`/`--entropy-sd` pays ### The 16-state model -Each family is a character with 16 possible states: one per subset of the 4 possible central bases (including the empty subset). Calibration combines a $5 \times 5$ transition matrix over family cardinality (0-4 observed forms) and a $4 \times 4$ base-substitution matrix from unambiguous single-copy loci, both restricted to genome pairs at or below `--sankoff-ratio-ceiling`, into a row-normalized $16 \times 16$ transition probability matrix $P$, converted to a symmetric cost matrix via $\text{cost}(a,b) = -\ln P(a,b)$. +Each family is a character with 16 possible states: one per subset of the 4 possible central bases (including the empty subset). See [The 16-state Sankoff model](../theory/phylogeny/snp_based/sankoff_model.md) for how the calibrated cost matrix is built from cardinality and base-composition transitions. `--sankoff` alone writes the cost matrix, the calibration parameters, and a pseudo-alignment recoded so the empty state uses the symbol `0` (never a gap character). It does not run any external tool. diff --git a/mkdocs-user.yml b/mkdocs-user.yml index cfe95afb..838d1e7c 100644 --- a/mkdocs-user.yml +++ b/mkdocs-user.yml @@ -31,11 +31,21 @@ nav: - Home: index.md - Installation: installation.md - Theory: - - Kmers and super-kmers: theory/kmers_and_superkmers.md - - DNA encoding: theory/encoding.md - - Low-complexity kmer filter: theory/entropy_filter.md - - Minimizer selection: theory/minimizer_selection.md - - Partitioning and indexing architecture: theory/indexing_architecture.md + - Kmer indexing: + - DNA encoding: theory/kmer_indexing/encoding.md + - Kmers: theory/kmer_indexing/kmers.md + - Minimizer selection: theory/kmer_indexing/minimizer_selection.md + - Super-kmers: theory/kmer_indexing/superkmers.md + - Partitioning and indexing architecture: theory/kmer_indexing/indexing_architecture.md + - Low-complexity kmer filter: theory/kmer_indexing/entropy_filter.md + - Phylogeny: + - Kmer-based: + - Whole k-mer set distances: theory/phylogeny/kmer_based/kmer_set_distances.md + - SNP-based: + - Central-position SNP model: theory/phylogeny/snp_based/central_snp_model.md + - Gamma rate heterogeneity: theory/phylogeny/snp_based/gamma_rate_heterogeneity.md + - Sankoff model: theory/phylogeny/snp_based/sankoff_model.md + - Sampling design: theory/phylogeny/snp_based/sampling_design.md - Usage: - superkmer: usage/superkmer.md - index: usage/index_command.md diff --git a/scripts/mk_wiki.py b/scripts/mk_wiki.py deleted file mode 100755 index b395027d..00000000 --- a/scripts/mk_wiki.py +++ /dev/null @@ -1,528 +0,0 @@ -#!/usr/bin/env python3 -"""Convert the MkDocs user documentation (UserDocMD/) into a Gitea/Forgejo wiki tree. - -Gitea/Forgejo wikis use a different link syntax than MkDocs and cannot render -multi-line `$$...$$` display math (a known Goldmark/KaTeX limitation: the -parser only recognizes the delimiters when the whole block sits on one -physical line). This script rewrites the three constructs that would -otherwise break, flattens the tree onto page basenames, and copies everything -else verbatim: - - * Markdown links [text](path.md) -> [text](PageName) - Relative paths (including MkDocs root-relative `/path.md` links) are - resolved against the source tree and then flattened to a wiki page id; - only the target is rewritten, the label is left untouched. Gitea/Forgejo's - own `[[page|label]]` shortlink syntax is deliberately not used: it is - post-processed on rendered HTML text nodes, so it cannot match when the - label contains inline formatting (a `` `code` `` span splits the text - into separate DOM nodes around it) — confirmed empirically on the target - instance, where such labels never became links at all, and even a plain - label put target and display text in the position swapped from what the - MediaWiki-style `[[target|label]]` convention would suggest. A native - Markdown link is a real AST link node, so it has neither problem. - External links (http(s)://, mailto:) and links pointing outside the - source tree are left untouched. - * Pandoc citations [@key; @key] -> formatted, CSL-styled citation - Gitea/Forgejo's Goldmark does not support the [@key] citation syntax, and - a bare `(@key)` fallback is meaningless to a reader without the .bib file - open. Files containing citations are instead piped through `pandoc - --citeproc` with the project's own `references.bib` and `ecology-letters - .csl` — the same files `mkdocs-bibtex` uses for the real site — so the - wiki gets the identical citation style, e.g. `[@Zheng2020-ji]` becomes - `(Zheng et al. 2020)`, with a formatted bibliography (one `
` per entry, real title/journal/DOI) appended to that page. This - duplicates entries across pages that cite the same source rather than - centralizing them on one wiki page — an acceptable tradeoff for a - handful of references, and far simpler than inventing a cross-page - anchor scheme. Files with no `[@key]` at all skip pandoc entirely. - * Multi-line display math $$\\n...\\n$$ -> single-line $$...$$ - Collapsed onto one line (interior newlines replaced by spaces; `\\\\` - row separators are preserved) so Gitea/Forgejo's renderer recognizes it. - Inline `$...$` and already single-line `$$...$$` need no change and pass - through as-is. - -None of these rewrites touch fenced code blocks (```` ``` ```` or `~~~`, -including openings longer than three characters, e.g. ```` ```` ```` wrapping -a literal triple-backtick example). - -Wiki page ids are flat (no directories), so a source file living in a -sub-directory gets its directory path folded into the page id: a file at -`theory/encoding.md` becomes the page `theory-encoding`. Files at the root keep -their bare stem, except `index.md` which maps to `Home` — the page name -Gitea/Forgejo treats as the wiki's landing page. The same mapping is used to -resolve internal links, so cross-directory references stay consistent. If -folding still produces a collision the script refuses to write rather than -silently dropping a page. - -The `nav:` section of the MkDocs config is reproduced as a `_Sidebar.md` so the -wiki keeps the same top-level grouping (Home / Installation / Theory / Usage / -Formats / Architecture). Nav entries are usually `{label: path}` mappings, but -a bare `path` entry (no explicit label) is also honored: the label is taken -from the target file's first `# ` heading, falling back to a prettified -filename. The sidebar is static and shown on every page, which is the only -way Gitea/Forgejo expose a structured menu; the file is hidden from the page -list because its name starts with `_`. - -Usage: - mk_wiki.py OUTPUT [OPTIONS] - -Options: - --source DIR Source tree (default: UserDocMD next to this script's parent) - --config FILE MkDocs user config (default: /mkdocs-user.yml) - --csl FILE CSL citation style (default: /ecology-letters.csl) - --bib FILE BibTeX bibliography (default: /references.bib) - --no-sidebar do not generate _Sidebar.md - --verbose print every conversion -""" - -from __future__ import annotations - -import argparse -import re -import subprocess -import sys -from pathlib import Path - -import yaml - - -class ConversionError(Exception): - """Raised when a source file cannot be converted faithfully.""" - - -def _make_lenient_loader() -> type[yaml.SafeLoader]: - """A SafeLoader that ignores unknown YAML tags instead of erroring. - - The MkDocs config uses `!!python/name:pymdownx.superfences.fence_code_format` - on a custom-fence entry; PyYAML cannot resolve that tag without an unsafe - loader, but the script only needs the `nav:` section, which is plain YAML. - Treating unknown tags as their literal scalar value is enough. - """ - - class _LenientLoader(yaml.SafeLoader): - pass - - def _unknown_constructor(loader: yaml.Loader, node: yaml.Node): # type: ignore[no-untyped-def] - if isinstance(node, yaml.ScalarNode): - return loader.construct_scalar(node) - if isinstance(node, yaml.SequenceNode): - return loader.construct_sequence(node) - if isinstance(node, yaml.MappingNode): - return loader.construct_mapping(node) - return None - - _LenientLoader.add_constructor(None, _unknown_constructor) - return _LenientLoader - - -def find_repo_root(start: Path) -> Path: - """Walk up until we find a directory containing UserDocMD/.""" - for candidate in [start.resolve(), *start.resolve().parents]: - if (candidate / "UserDocMD").is_dir(): - return candidate - return start.resolve() - - -def is_external(target: str) -> bool: - return bool(re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", target)) or target.startswith("mailto:") - - -def split_anchor(target: str) -> tuple[str, str]: - if "#" in target: - path, anchor = target.split("#", 1) - return path, "#" + anchor - return target, "" - - -def wiki_page_id(src_root: Path, abs_target: Path) -> str: - """Map a source file to its wiki page id. - - Wikis are flat, so the directory path is folded into the id: a file at - `theory/encoding.md` becomes `theory-encoding`, while a root file keeps its - bare stem, except `index.md` which maps to `Home` — the page name - Gitea/Forgejo renders as the wiki's landing page. - """ - rel = abs_target.relative_to(src_root.resolve()) - parts = list(rel.parts) - if len(parts) > 1: - # Drop the .md suffix from the last part, join the rest with '-'. - return "-".join([*parts[:-1], parts[-1][:-3]]) - stem = parts[0][:-3] - return "Home" if stem == "index" else stem - - -def resolve_wiki_path(src_root: Path, src_file: Path, target: str) -> str | None: - """Turn a relative markdown target into a wiki page id. - - Returns None for targets we should leave as plain Markdown links - (external URLs, non-markdown files, missing files, or targets outside the - source tree). - """ - path, anchor = split_anchor(target) - if is_external(path) or path == "": - return None - if path.startswith("/"): - # MkDocs root-relative link: resolved against the docs root, not the - # current file's directory (Path("a") / "/b" would silently discard - # "a" and resolve "/b" from the filesystem root, which is wrong here). - abs_target = (src_root / path.lstrip("/")).resolve() - else: - abs_target = (src_file.parent / path).resolve() - try: - abs_target.relative_to(src_root.resolve()) - except ValueError: - return None - if abs_target.suffix != ".md": - return None - if not abs_target.is_file(): - return None - return wiki_page_id(src_root, abs_target) + anchor - - -LINK_RE = re.compile(r"\[([^\]]*)\]\(([^)\s]+)\)") -FENCE_RE = re.compile(r"^(`{3,}|~{3,})") -# Pandoc citation syntax, e.g. [@Zheng2020-ji; @Golan2025-xf] — just used to -# detect whether a file has any citation worth running through pandoc at all. -HAS_CITATION_RE = re.compile(r"\[@[^\]]+\]") - -# pandoc's markdown writer must be told, relative to its own defaults, to: -# -smart leave dashes/quotes as written (no em-dash -> "---") -# -fenced_divs emit the bibliography as raw HTML
(Goldmark -# has no ``:::`` fenced-div extension) -# -simple_tables, -multiline_tables, -grid_tables, +pipe_tables -# keep GFM pipe-table syntax, the only kind Goldmark -# parses, instead of pandoc's space-aligned tables -# +fenced_code_blocks (pandoc still emits an indented block for a fence -# with no language tag regardless of this extension — -# such fences need a language hint in the source) -# -header_attributes Goldmark has no `{#id .class}` heading-attribute -# syntax; without this the "Bibliography" heading pandoc -# adds (see BIBLIOGRAPHY_TITLE below) would render with -# a literal `{#bibliography .unnumbered}` after it -PANDOC_MARKDOWN_FORMAT = ( - "markdown-smart-fenced_divs-simple_tables-multiline_tables-grid_tables" - "-header_attributes+pipe_tables+fenced_code_blocks" -) - -# Title pandoc gives the generated bibliography section (always emitted as a -# level-1 `# ` heading, with no option to pick the level) — demoted to `##` -# below to match the rest of this doc set's heading hierarchy. -BIBLIOGRAPHY_TITLE = "Bibliography" - - -class FenceState: - """Tracks whether the current line is inside a fenced code block. - - A closing fence must use the same character and be at least as long as - the opening one (CommonMark rule) — this is what lets a source file wrap - a literal ``` example inside a longer ```` fence. - """ - - def __init__(self) -> None: - self.marker = "" - - def toggle(self, line: str) -> bool: - """Update state for a fence delimiter line; return True if it is one.""" - m = FENCE_RE.match(line.rstrip("\n")) - if not m: - return False - run = m.group(1) - if not self.marker: - self.marker = run - elif run[0] == self.marker[0] and len(run) >= len(self.marker): - self.marker = "" - return True - - @property - def active(self) -> bool: - return bool(self.marker) - - -def convert_links(text: str, src_root: Path, src_file: Path, verbose: bool) -> str: - out_lines: list[str] = [] - fence = FenceState() - - for line in text.splitlines(keepends=True): - if fence.toggle(line) or fence.active: - out_lines.append(line) - continue - - def repl(m: re.Match) -> str: - label, target = m.group(1), m.group(2) - wiki = resolve_wiki_path(src_root, src_file, target) - if wiki is None: - return m.group(0) # external or non-md: leave as Markdown - if verbose: - print(f" {src_file.relative_to(src_root)}: [{label}]({target}) -> [{label}]({wiki})") - return f"[{label}]({wiki})" - - out_lines.append(LINK_RE.sub(repl, line)) - - return "".join(out_lines) - - -def convert_citations( - text: str, src_root: Path, src_file: Path, csl_path: Path, bib_path: Path, verbose: bool -) -> str: - """Render Pandoc `[@key; @key]` citations with the project's real CSL style. - - Skips files with no citation entirely, so a missing `--csl`/`--bib` or a - missing `pandoc` binary is only an error for files that actually need it. - """ - if not HAS_CITATION_RE.search(text): - return text - - if not csl_path.is_file(): - raise ConversionError(f"{src_file.relative_to(src_root)}: CSL style not found: {csl_path}") - if not bib_path.is_file(): - raise ConversionError(f"{src_file.relative_to(src_root)}: bibliography not found: {bib_path}") - - try: - result = subprocess.run( - [ - "pandoc", - "--citeproc", - f"--csl={csl_path}", - f"--bibliography={bib_path}", - "--wrap=none", - f"--metadata=link-citations=true", - f"--metadata=reference-section-title={BIBLIOGRAPHY_TITLE}", - "-f", - "markdown-smart", - "-t", - PANDOC_MARKDOWN_FORMAT, - ], - input=text, - capture_output=True, - text=True, - ) - except FileNotFoundError: - raise ConversionError( - f"{src_file.relative_to(src_root)}: 'pandoc' not found on PATH, needed to render citations" - ) from None - - if result.returncode != 0: - raise ConversionError( - f"{src_file.relative_to(src_root)}: pandoc failed to render citations:\n{result.stderr}" - ) - - # pandoc always emits the bibliography title as a level-1 heading; demote - # it to `##` to match the rest of this doc set's hierarchy. - out = result.stdout.replace(f"\n# {BIBLIOGRAPHY_TITLE}\n", f"\n## {BIBLIOGRAPHY_TITLE}\n") - - if verbose: - n = len(HAS_CITATION_RE.findall(text)) - print(f" {src_file.relative_to(src_root)}: rendered {n} citation group(s) via pandoc/{csl_path.name}") - - return out - - -def convert_math(text: str, src_root: Path, src_file: Path, verbose: bool) -> str: - """Collapse multi-line `$$...$$` display blocks onto a single line. - - Gitea/Forgejo's math renderer only recognizes `$$...$$` when it sits on - one physical line; a block opened and closed on their own bare `$$` line - (the natural Pandoc/MkDocs style) renders as literal text otherwise. - Interior line breaks are replaced by spaces (insignificant in math mode); - `\\\\` row separators inside `aligned`/`cases` environments are untouched. - Already single-line `$$...$$` and inline `$...$` need no change. - """ - lines = text.split("\n") - out: list[str] = [] - fence = FenceState() - i = 0 - n = len(lines) - while i < n: - line = lines[i] - if fence.toggle(line) or fence.active: - out.append(line) - i += 1 - continue - - if line.strip() == "$$": - body: list[str] = [] - i += 1 - closed = False - while i < n: - if lines[i].strip() == "$$": - closed = True - i += 1 - break - body.append(lines[i].strip()) - i += 1 - if not closed: - raise ConversionError( - f"{src_file.relative_to(src_root)}: unterminated $$ display math block" - ) - collapsed = "$$" + " ".join(part for part in body if part) + "$$" - if verbose: - print(f" {src_file.relative_to(src_root)}: collapsed {len(body)}-line $$ block") - out.append(collapsed) - continue - - out.append(line) - i += 1 - - return "\n".join(out) - - -def convert_file( - src_root: Path, src_file: Path, csl_path: Path, bib_path: Path, verbose: bool -) -> str: - text = src_file.read_text(encoding="utf-8") - # Citations first, before pandoc has seen any of our own wiki-specific - # rewrites: it needs to parse plain, unmodified Markdown. - text = convert_citations(text, src_root, src_file, csl_path, bib_path, verbose) - text = convert_math(text, src_root, src_file, verbose) - text = convert_links(text, src_root, src_file, verbose) - return text - - -def _default_label(src_root: Path, rel_path: str) -> str: - """Derive a nav label for a bare `path.md` entry (no explicit label). - - Mirrors MkDocs' own fallback: use the page's first `# ` heading if it has - one, otherwise prettify the filename. - """ - target = src_root / rel_path - if target.is_file(): - for line in target.read_text(encoding="utf-8").splitlines(): - if line.startswith("# "): - return line[2:].strip() - return Path(rel_path).stem.replace("_", " ").replace("-", " ").title() - - -def build_sidebar(src_root: Path, config_path: Path) -> str | None: - """Render the MkDocs `nav:` section as a Gitea/Forgejo `_Sidebar.md`. - - Returns None if the config is missing or has no usable `nav:` section. - Each nav entry is either a bare `path` string or a `{label: path}` pair - (or a nested list for a group); the path is mapped through `wiki_page_id` - so the sidebar links to the flattened page ids. - """ - if not config_path.is_file(): - print(f"warning: config not found, skipping sidebar: {config_path}") - return None - - try: - with config_path.open(encoding="utf-8") as fh: - cfg = yaml.safe_load(fh) - except yaml.YAMLError: - # The MkDocs config carries `!!python/name:...` tags that SafeLoader - # cannot resolve; fall back to a lenient loader that treats unknown - # tags as plain scalars so the `nav:` section is still readable. - try: - with config_path.open(encoding="utf-8") as fh: - cfg = yaml.load(fh, Loader=_make_lenient_loader()) - except yaml.YAMLError as exc2: - print(f"warning: could not parse {config_path}: {exc2}") - return None - - nav = cfg.get("nav") if isinstance(cfg, dict) else None - if not nav: - print(f"warning: no `nav:` section in {config_path}, skipping sidebar") - return None - - def link_line(depth: int, rel_path: str, label: str) -> str: - page = wiki_page_id(src_root, (src_root / rel_path).resolve()) - return f"{' ' * depth}- [{label}]({page})" - - def render_entry(entry: object, depth: int) -> list[str]: - if isinstance(entry, str): - return [link_line(depth, entry, _default_label(src_root, entry))] - if not isinstance(entry, dict): - return [] - lines: list[str] = [] - for label, target in entry.items(): - if isinstance(target, str): - lines.append(link_line(depth, target, label)) - elif isinstance(target, list): - # Headings must never be indented: CommonMark treats a line - # indented by 4+ spaces as a code block, not an ATX heading, - # which a depth-scaled indent would trigger from depth 2 on. - lines.append(f"## {label}") - lines.extend(render_subtree(target, depth + 1)) - return lines - - def render_subtree(nodes: list[object], depth: int) -> list[str]: - out: list[str] = [] - for node in nodes: - out.extend(render_entry(node, depth)) - return out - - body = render_subtree(nav, 0) - if not body: - return None - - return "# Wiki sidebar\n\n" + "\n".join(body) + "\n" - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("output", help="destination wiki directory (created if missing)") - ap.add_argument("--source", default=None, help="source Markdown tree (default: /UserDocMD)") - ap.add_argument("--config", default=None, - help="MkDocs user config (default: /mkdocs-user.yml)") - ap.add_argument("--csl", default=None, - help="CSL citation style (default: /ecology-letters.csl)") - ap.add_argument("--bib", default=None, - help="BibTeX bibliography (default: /references.bib)") - ap.add_argument("--no-sidebar", action="store_true", help="do not generate _Sidebar.md") - ap.add_argument("--verbose", action="store_true", help="print every conversion") - args = ap.parse_args() - - script_dir = Path(__file__).resolve().parent - repo_root = find_repo_root(script_dir) - src_root = Path(args.source).resolve() if args.source else (repo_root / "UserDocMD") - out_root = Path(args.output).resolve() - config_path = Path(args.config).resolve() if args.config else (repo_root / "mkdocs-user.yml") - csl_path = Path(args.csl).resolve() if args.csl else (src_root / "ecology-letters.csl") - bib_path = Path(args.bib).resolve() if args.bib else (src_root / "references.bib") - - if not src_root.is_dir(): - print(f"error: source tree not found: {src_root}", file=sys.stderr) - return 2 - - md_files = sorted(p for p in src_root.rglob("*.md") if p.is_file()) - if not md_files: - print(f"error: no Markdown files under {src_root}", file=sys.stderr) - return 2 - - # A wiki is flat, so the destination filename is the source path folded into - # a page id. Detect collisions up front so a silent overwrite does not drop - # a page. - by_name: dict[str, list[Path]] = {} - for src in md_files: - by_name.setdefault(wiki_page_id(src_root, src), []).append(src) - collisions = {name: paths for name, paths in by_name.items() if len(paths) > 1} - if collisions: - for name, paths in sorted(collisions.items()): - print(f"error: wiki page id collision '{name}':", file=sys.stderr) - for p in paths: - print(f" {p.relative_to(src_root)}", file=sys.stderr) - return 2 - - out_root.mkdir(parents=True, exist_ok=True) - written = 0 - for src in md_files: - dst = out_root / (wiki_page_id(src_root, src) + ".md") - try: - content = convert_file(src_root, src, csl_path, bib_path, args.verbose) - except ConversionError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - dst.write_text(content, encoding="utf-8") - written += 1 - - if not args.no_sidebar: - sidebar = build_sidebar(src_root, config_path) - if sidebar is not None: - (out_root / "_Sidebar.md").write_text(sidebar, encoding="utf-8") - written += 1 - - print(f"converted {written} Markdown file(s) from {src_root} -> {out_root}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) -- 2.54.1 From 0d1ecac2338d2180a795fdccb0f9fc0be86e3b72 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 18 Sep 2026 06:45:42 +0200 Subject: [PATCH 03/10] Refactor k-mer indexing, enhance session features, and update documentation This commit introduces significant refinements across the codebase. Core indexing logic is updated with hard constraints on k-mer length, statistical modeling clarifications, and the implementation of the Karlin & Ost theory for chance-collision statistics. Session management features are enhanced with robust memoization capabilities, atomic file operations, per-key locking, and memory mapping support. Additionally, documentation is updated, including a new citation style, and new mathematical notebook support is added. --- UserDocMD/molecular-ecology-modified.csl | 157 ++++++ UserDocMD/references.bib | 202 +++++++ UserDocMD/theory/kmer_indexing/kmers.md | 224 +++++++- .../Untitled-checkpoint.ipynb | 6 + math/Untitled.ipynb | 126 +++++ mkdocs-user.yml | 2 +- src/Cargo.lock | 11 + src/obikphylo/src/karlin.rs | 331 ++++++++++++ src/obikphylo/src/lib.rs | 8 + src/obiksession/Cargo.toml | 2 + src/obiksession/src/lib.rs | 15 + src/obiksession/src/memoize.rs | 511 ++++++++++++++++++ src/obiksession/src/session.rs | 106 +++- 13 files changed, 1683 insertions(+), 18 deletions(-) create mode 100644 UserDocMD/molecular-ecology-modified.csl create mode 100644 math/.ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100644 math/Untitled.ipynb create mode 100644 src/obikphylo/src/karlin.rs create mode 100644 src/obiksession/src/memoize.rs diff --git a/UserDocMD/molecular-ecology-modified.csl b/UserDocMD/molecular-ecology-modified.csl new file mode 100644 index 00000000..ed2b1ce9 --- /dev/null +++ b/UserDocMD/molecular-ecology-modified.csl @@ -0,0 +1,157 @@ + + diff --git a/UserDocMD/references.bib b/UserDocMD/references.bib index 1dbf1ef4..f547a3f4 100644 --- a/UserDocMD/references.bib +++ b/UserDocMD/references.bib @@ -566,3 +566,205 @@ issn = "0167-8655,1872-7344", language = "en" } + @ARTICLE{Fleischmann2014-ed, + title = {Evolution of genome size and chromosome number in the + carnivorous plant genus Genlisea (Lentibulariaceae), with a new + estimate of the minimum genome size in angiosperms}, + author = "Fleischmann, Andreas and Michael, Todd P and Rivadavia, Fernando + and Sousa, Aretuza and Wang, Wenqin and Temsch, Eva M and + Greilhuber, Johann and Müller, Kai F and Heubl, Günther", + journal = "Annals of Botany", + publisher = "Oxford University Press (OUP)", + volume = 114, + number = 8, + pages = "1651--1663", + abstract = "BACKGROUND AND AIMS: Some species of Genlisea possess ultrasmall + nuclear genomes, the smallest known among angiosperms, and some + have been found to have chromosomes of diminutive size, which may + explain why chromosome numbers and karyotypes are not known for + the majority of species of the genus. However, other members of + the genus do not possess ultrasmall genomes, nor do most taxa + studied in related genera of the family or order. This study + therefore examined the evolution of genome sizes and chromosome + numbers in Genlisea in a phylogenetic context. The correlations + of genome size with chromosome number and size, with the + phylogeny of the group and with growth forms and habitats were + also examined. METHODS: Nuclear genome sizes were measured from + cultivated plant material for a comprehensive sampling of taxa, + including nearly half of all species of Genlisea and representing + all major lineages. Flow cytometric measurements were conducted + in parallel in two laboratories in order to compare the + consistency of different methods and controls. Chromosome counts + were performed for the majority of taxa, comparing different + staining techniques for the ultrasmall chromosomes. KEY RESULTS: + Genome sizes of 15 taxa of Genlisea are presented and interpreted + in a phylogenetic context. A high degree of congruence was found + between genome size distribution and the major phylogenetic + lineages. Ultrasmall genomes with 1C values of <100 Mbp were + almost exclusively found in a derived lineage of South American + species. The ancestral haploid chromosome number was inferred to + be n = 8. Chromosome numbers in Genlisea ranged from 2n = 2x = 16 + to 2n = 4x = 32. Ascendant dysploid series (2n = 36, 38) are + documented for three derived taxa. The different ploidy levels + corresponded to the two subgenera, but were not directly + correlated to differences in genome size; the three different + karyotype ranges mirrored the different sections of the genus. + The smallest known plant genomes were not found in G. margaretae, + as previously reported, but in G. tuberosa (1C ≈ 61 Mbp) and some + strains of G. aurea (1C ≈ 64 Mbp). CONCLUSIONS: Genlisea is an + ideal candidate model organism for the understanding of genome + reduction as the genus includes species with both relatively + large (∼1700 Mbp) and ultrasmall (∼61 Mbp) genomes. This + comparative, phylogeny-based analysis of genome sizes and + karyotypes in Genlisea provides essential data for selection of + suitable species for comparative whole-genome analyses, as well + as for further studies on both the molecular and cytogenetic + basis of genome reduction in plants.", + month = dec, + year = 2014, + url = "http://dx.doi.org/10.1093/aob/mcu189", + keywords = "Bladderwort; Genlisea; Lamiales; Lentibulariaceae; carnivorous + plant; chromosome number; flow cytometry; genome miniaturization; + genome size", + doi = "10.1093/aob/mcu189", + pmc = "PMC4649684", + pmid = 25274549, + issn = "0305-7364,1095-8290", + language = "en" + } + @ARTICLE{Fernandez2024-kd, + title = "A 160 Gbp fork fern genome shatters size record for eukaryotes", + author = "Fernández, Pol and Amice, Rémy and Bruy, David and Christenhusz, + Maarten J M and Leitch, Ilia J and Leitch, Andrew L and Pokorny, + Lisa and Hidalgo, Oriane and Pellicer, Jaume", + journal = "iScience", + publisher = "Elsevier BV", + volume = 27, + number = 6, + pages = 109889, + abstract = "Vascular plants are exceptional among eukaryotes due to their + outstanding genome size diversity which ranges ∼2,400-fold, + including the largest genome so far recorded in the angiosperm + Paris japonica (148.89 Gbp/1C). Despite available data showing + that giant genomes are restricted across the Tree of Life, the + biological limits to genome size expansion remain to be + established. Here, we report the discovery of an even larger + eukaryotic genome in Tmesipteris oblanceolata, a New Caledonian + fork fern. At 160.45 Gbp/1C, this record-breaking genome + challenges current understanding and opens new avenues to explore + the evolutionary dynamics of genomic gigantism.", + month = jun, + year = 2024, + url = "http://dx.doi.org/10.1016/j.isci.2024.109889", + keywords = "Plant evolution; Plant genetics", + doi = "10.1016/j.isci.2024.109889", + pmc = "PMC11270024", + pmid = 39055604, + issn = "2589-0042", + language = "en" + } + @ARTICLE{Fraser1995-jc, + title = "The minimal gene complement of Mycoplasma genitalium", + author = "Fraser, C M and Gocayne, J D and White, O and Adams, M D and + Clayton, R A and Fleischmann, R D and Bult, C J and Kerlavage, A + R and Sutton, G and Kelley, J M and Fritchman, R D and Weidman, J + F and Small, K V and Sandusky, M and Fuhrmann, J and Nguyen, D + and Utterback, T R and Saudek, D M and Phillips, C A and Merrick, + J M and Tomb, J F and Dougherty, B A and Bott, K F and Hu, P C + and Lucier, T S and Peterson, S N and Smith, H O and Hutchison, + 3rd, C A and Venter, J C", + journal = "Science (New York, N.Y.)", + publisher = "American Association for the Advancement of Science (AAAS)", + volume = 270, + number = 5235, + pages = "397--403", + abstract = "The complete nucleotide sequence (580,070 base pairs) of the + Mycoplasma genitalium genome, the smallest known genome of any + free-living organism, has been determined by whole-genome random + sequencing and assembly. A total of only 470 predicted coding + regions were identified that include genes required for DNA + replication, transcription and translation, DNA repair, cellular + transport, and energy metabolism. Comparison of this genome to + that of Haemophilus influenzae suggests that differences in + genome content are reflected as profound differences in + physiology and metabolic capacity between these two organisms.", + month = oct, + year = 1995, + url = "http://dx.doi.org/10.1126/science.270.5235.397", + doi = "10.1126/science.270.5235.397", + pmid = 7569993, + issn = "0036-8075,1095-9203", + language = "en" + } + @ARTICLE{Schneiker2007-gs, + title = "Complete genome sequence of the myxobacterium Sorangium + cellulosum", + author = "Schneiker, Susanne and Perlova, Olena and Kaiser, Olaf and Gerth, + Klaus and Alici, Aysel and Altmeyer, Matthias O and Bartels, + Daniela and Bekel, Thomas and Beyer, Stefan and Bode, Edna and + Bode, Helge B and Bolten, Christoph J and Choudhuri, Jomuna V and + Doss, Sabrina and Elnakady, Yasser A and Frank, Bettina and + Gaigalat, Lars and Goesmann, Alexander and Groeger, Carolin and + Gross, Frank and Jelsbak, Lars and Jelsbak, Lotte and Kalinowski, + Jörn and Kegler, Carsten and Knauber, Tina and Konietzny, + Sebastian and Kopp, Maren and Krause, Lutz and Krug, Daniel and + Linke, Bukhard and Mahmud, Taifo and Martinez-Arias, Rosa and + McHardy, Alice C and Merai, Michelle and Meyer, Folker and + Mormann, Sascha and Muñoz-Dorado, Jose and Perez, Juana and + Pradella, Silke and Rachid, Shwan and Raddatz, Günter and + Rosenau, Frank and Rückert, Christian and Sasse, Florenz and + Scharfe, Maren and Schuster, Stephan C and Suen, Garret and + Treuner-Lange, Anke and Velicer, Gregory J and Vorhölter, + Frank-Jörg and Weissman, Kira J and Welch, Roy D and Wenzel, + Silke C and Whitworth, David E and Wilhelm, Susanne and Wittmann, + Christoph and Blöcker, Helmut and Pühler, Alfred and Müller, Rolf", + journal = "Nature Biotechnology", + publisher = "Springer Science and Business Media LLC", + volume = 25, + number = 11, + pages = "1281--1289", + abstract = "The genus Sorangium synthesizes approximately half of the + secondary metabolites isolated from myxobacteria, including the + anti-cancer metabolite epothilone. We report the complete genome + sequence of the model Sorangium strain S. cellulosum So ce56, + which produces several natural products and has morphological and + physiological properties typical of the genus. The circular + genome, comprising 13,033,779 base pairs, is the largest + bacterial genome sequenced to date. No global synteny with the + genome of Myxococcus xanthus is apparent, revealing an + unanticipated level of divergence between these myxobacteria. A + large percentage of the genome is devoted to regulation, + particularly post-translational phosphorylation, which probably + supports the strain's complex, social lifestyle. This regulatory + network includes the highest number of eukaryotic protein + kinase-like kinases discovered in any organism. Seventeen + secondary metabolite loci are encoded in the genome, as well as + many enzymes with potential utility in industry.", + month = nov, + year = 2007, + url = "http://dx.doi.org/10.1038/nbt1354", + doi = "10.1038/nbt1354", + pmid = 17965706, + issn = "1087-0156,1546-1696", + language = "en" + } + @incollection{Karlin1985, + author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric}, + title = {Maximal segmental match length among random sequences from a finite alphabet}, + booktitle = {Proceedings of the Berkeley Conference in Honor of Jerzy Neyman and Jack Kiefer}, + editor = {Le Cam, Lucien M. and Olshen, Richard A.}, + volume = {1}, + pages = {225--243}, + year = {1985}, + publisher = {Wadsworth}, + address = {Monterey, CA} + } + @article{Karlin1988, + author = {Karlin, Samuel and Ost, Fr{\'e}d{\'e}ric}, + title = {Maximal length of common words among random letter sequences}, + journal = {The Annals of Probability}, + volume = {16}, + number = {2}, + pages = {535--563}, + year = {1988} + } diff --git a/UserDocMD/theory/kmer_indexing/kmers.md b/UserDocMD/theory/kmer_indexing/kmers.md index dae3acd7..36befb07 100644 --- a/UserDocMD/theory/kmer_indexing/kmers.md +++ b/UserDocMD/theory/kmer_indexing/kmers.md @@ -1,6 +1,222 @@ -# Kmers +# K-mer size -A **kmer** is a DNA subsequence of fixed length $k$. Two constraints apply to $k$, both enforced when a command starts (an invalid value exits immediately with an error): +A k-mer is a DNA sequence of length $k$. OBIKmer is designed to index large k-mers. Here, *large* refers to a k-mer length for which the probability of observing the same k-mer in several genomes by chance alone is sufficiently small. The k-mer length must therefore be large enough that, when a k-mer is observed in several genomes of the dataset, the probability that this shared occurrence is due to chance alone is sufficiently low for the match to be considered evidence of homology. -- $k \in [11, 31]$: long enough to be specific, short enough to fit in a single 64-bit word at 2 bits/base ($k \le 32$ is the hard limit; $k < 11$ gives insufficient specificity). -- $k$ **is odd**: an odd-length sequence can never equal its own reverse complement, so the two orientations of any kmer are always distinct. This is required for the canonical form (see [DNA encoding](encoding.md)) to be well defined. +OBIKmer restricts $k$ to odd values in the range + +$$ +11 \leq k \leq 31. +$$ + +These limits result from two different considerations. The upper bound is imposed by the representation used by OBIKmer, whereas the lower bound defines a practical range of k-mer lengths for genome-scale sequence comparisons. This range covers most practical applications: the lower limit is well below the k-mer size that can be considered large, even for the smallest bacterial genomes, whereas the upper limit provides sufficient specificity for most applications, including those involving very large genomes. + +## Representation limit + +OBIKmer uses two bits to encode each nucleotide. Therefore, a k-mer requires $2k$ bits. To ensure computational efficiency, OBIKmer limits the representation of k-mers to a single 64-bit word, thereby restricting their length. + +$$ +2k \leq 64, +$$ + +This gives the hard representation limit: $k \leq 32.$ Furthermore, OBIKmer only considers odd values of $k$, meaning the largest supported k-mer size is $31$. + +### K-mer specificity + +A simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are + +$$ +4^k +$$ + +possible k-mers, and a particular k-mer is therefore expected to occur once every $4^k$ positions on average. In a sequence of length $G$, the expected number of occurrences of a particular k-mer is approximately + +$$ +E[n_K] \simeq \frac{G}{4^k}. +$$ + +When $4^k\gg G$, this quantity is also a good approximation of the probability that an observed k-mer will occur at least once by chance elsewhere in a sequence of length $G$. It therefore provides a simple measure of the risk of a chance recurrence of an observed k-mer: + +$$ +P_{\mathrm{chance}}(k,G) \simeq \frac{G}{4^k}. +$$ + +This probability decreases by a factor of four for every additional nucleotide in the k-mer. Conversely, it increases linearly with the amount of sequence searched. + +This simple model already shows why very short k-mers provide limited specificity for genome-scale analyses. With $k=11$, + +$$ +4^{11}\simeq4.2\times10^6, +$$ + +which is a small combinatorial space even compared with the size of very small bacterial genomes. In contrast, + +$$ +4^{31}\simeq4.6\times10^{18}, +$$ + +which is vastly larger than the number of k-mer positions even in very large genomes. Thus, increasing $k$ from 11 to 31 provides an enormous increase in sequence specificity. + +The estimate above is intentionally simplistic. Real genomes do not consist of independent, equiprobable nucleotides: nucleotide composition and sequence dependencies affect the probability of observing particular words. More accurate estimates of the probability of observing matches by chance therefore take sequence length and nucleotide composition into account. + +Karlin and Ost provide an analytical framework for this problem based on the distribution of the longest exact match expected between random sequences. If $L_{\max}$ denotes the length of the longest seed observed in random sequences of given size and nucleotide composition, the corresponding seed-level probability is + +$$ +P_{\mathrm{seeds}}(L)=P(L_{\max}\geq L). +$$ + +Conversely, for a chosen false-positive probability $P$, one can determine the minimum seed length $L_{\min}$ such that matches of length $L_{\min}$ or greater are not expected to occur by chance. This is the approach used by Repseek to determine statistically significant seed lengths. + +The simple $G/4^k$ estimate and the Karlin–Ost statistics therefore describe the same fundamental effect at different levels of approximation: the probability of a chance match decreases rapidly with k-mer length, while the amount of sequence being searched increases the opportunity for such matches. + +Consequently, the appropriate value of $k$ depends on the total amount of sequence being compared and its nucleotide composition. Choosing $k$ appropriately allows the false-positive rate of homology inference to be controlled. + + + +Considering that bacterial genomes are typically a few megabases long, with the smallesr plublished today *Mycoplasma genitalium* having a size $G=580\,Mb$ [@Fraser1995-jc], and the largest $13\,Mb$ in *Sorangium cellulosum* [@Schneiker2007-gs]. For such genomes, the space of 11-mers is already to small to get a reasonable false positive rate: + +$$ +\begin{aligned} +P(C\geq1) &= 1- e^{-\frac{580.000^2}{2 \cdot 4^{11}}} \\ +&\simeq 1 +\end{aligned} +$$ + +, whereas 21-mers already provide a space of several trillion possible sequences. + +Plant genomes illustrate the opposite situation. Plant genome sizes span several orders of magnitude, from the smallest angiosperm genomes present in the carnivorous plans genus [@Fleischmann2014-ed] about 60 Mb to the largest one 160 Gb in Tmesipteris oblanceolata, a New Caledonian fork fern [@Fernandez2024-kd], and large genomes often contain substantial amounts of repeated sequence. Even at $k=31$, however, the theoretical k-mer space remains vastly larger than any individual genome. At these lengths, the limiting factor is therefore not exhaustion of the theoretical k-mer space but the biological repetitiveness and non-random composition of the genome. + +For example, the number of possible k-mers grows as follows: + +| $k$ | $4^k$ | $4^k/2$ after canonicalization | +| ----: | --------------------: | -------------------------------: | +| 11 | $4.19\times10^6$ | $2.10\times10^6$ | +| 15 | $1.07\times10^9$ | $5.37\times10^8$ | +| 21 | $4.40\times10^{12}$ | $2.20\times10^{12}$ | +| 25 | $1.13\times10^{15}$ | $5.63\times10^{14}$ | +| 31 | $4.61\times10^{18}$ | $2.31\times10^{18}$ | + +Canonicalization approximately halves the number of possible representations because a k-mer and its reverse complement are represented by the same canonical form. + + + +The distinction between the simple $4^k$ model and the Karlin--Ost framework is important. The former describes the size of the combinatorial k-mer space; the latter describes the probability of observing long matches in sequences whose composition may depart from the uniform random model. Consequently, the value of $k$ cannot be selected from genome size alone. + + +These scales have very different implications for different genome sizes. + +The random-sequence calculation is only a null model. Real genomes exhibit strong and reproducible oligonucleotide composition biases. Karlin and colleagues showed that relative abundances of short oligonucleotides, in particular dinucleotides, form characteristic genomic signatures that remain relatively stable within genomes and differ between organisms [@Karlin1995; @Karlin1997]. More generally, short k-mer frequencies can therefore capture compositional properties of genomes rather than provide highly specific sequence identifiers. + +This distinction is important for whole-genome k-mer comparison. Increasing $k$ increases the theoretical diversity of the k-mer space exponentially and generally increases the proportion of k-mers that occur at unique genomic positions. Conversely, shorter k-mers occur more frequently and are more strongly influenced by genomic composition and repeated sequence. Empirical studies of k-mer spectra show this transition clearly: for example, k-mers of length $k\leq11$ can behave primarily as genomic-composition signatures, whereas longer k-mers progressively provide greater sequence specificity. + +The supported range $11\leq k\leq31$ should therefore be understood as an engineering compromise rather than as a universal biological optimum. Smaller values increase the probability that unrelated genomic regions share the same k-mers, while larger values increase specificity but also make the representation more sensitive to substitutions, indels, and incomplete homology. The appropriate value within this range consequently depends on the biological scale and comparison task. + +## Odd k-mer lengths + +OBIKmer additionally requires $k$ to be odd. This constraint is independent of the resolution argument above. + +For an odd-length DNA sequence, a k-mer cannot be identical to its own reverse complement. Consequently, every k-mer has two distinct orientations, which can be unambiguously collapsed into a canonical representation. For even $k$, self-reverse-complementary sequences exist, so a k-mer can coincide with its own reverse complement. + +Odd $k$ is therefore **not required for canonicalization itself**: the canonical form + +$$ +s^c=\min_{\mathrm{lex}}\left(s,\operatorname{RC}(s)\right) +$$ + +remains well defined for even $k$. The odd-$k$ restriction instead guarantees that the two orientations are always distinct, which simplifies the representation and orientation handling used by OBIKmer. + +## Karlin--Ost (1988) formula + +[@Karlin1988] considers $s$ independent stationary letter sequences over a finite alphabet and studies $K_{r,s}(N)$, the length of the longest word common to $r$ of the $s$ sequences, each of length $N$. For i.i.d. equiprobable letters over an alphabet of size $m$ (here $m=4$ for DNA), the characteristic local match parameter reduces to + +$$ +\lambda^{[r]} = m^{1-r}, +$$ + +so for $r=2$ (a word shared by two occurrences), $\lambda = m^{-1} = 1/4$. + +The expected number of common words of length $k$ among the $\binom{s}{r}$ relevant groups of sequences is + +$$ +S_1(N,k) = N^{r}\binom{s}{r}\gamma\,\lambda^{k}\,(1+o(1)), +$$ + +with $\gamma=1$ exactly in the i.i.d. equiprobable case. Theorem 2.2 gives the limiting distribution of $K_{r,s}(N)$, with $k$ set on the scale $k=\left[\dfrac{\log N^{r}}{-\log\lambda^{*}}+x+1\right]$: + +$$ +\lim_{N\to\infty}\left[\Pr\{K_{r,s}(N)oo} F(k) / lambda^k`, where +//! `F(k) = sum_w Pr{w}^r` over all `m^k` possible words `w`. For the i.i.d. +//! equiprobable case, `Pr{w} = m^-k` for every word, so `F(k) = m^k * +//! m^(-k*r) = m^(k*(1-r)) = lambda^k` **exactly**, for every `k` — not just +//! asymptotically. Hence `gamma = 1` exactly and the `(1 + o(1))` error term +//! in the general formula vanishes identically in this case; no correction +//! is dropped. +//! +//! Theorem 2.2 (eq. 2.12) gives the limiting distribution of the longest +//! `r`-fold common word length `K_{r,s}(N)` as `N -> oo`: +//! +//! ```text +//! Pr{K_{r,s}(N) < k} ~= exp{ -(1 - lambda) * S_1(N, k) } +//! ``` +//! +//! so the probability of at least one chance `r`-fold match of length `k` +//! is +//! +//! ```text +//! P(k) = Pr{K_{r,s}(N) >= k} ~= 1 - exp{ -(1 - lambda) * S_1(N, k) } +//! ``` +//! +//! The `(1 - lambda)` factor is the de-clumping correction: it accounts for +//! the fact that a single long chance match of length `L` is otherwise +//! counted once for every `k <= L` it contains, which would overstate the +//! rate of genuinely distinct matches. +//! +//! # Unequal genome lengths +//! +//! The `N^r` factor in `S_1(N, k)` is a shortcut for a more basic fact: for +//! one fixed group of `r` sequences, the number of candidate position +//! combinations is the *product* of that group's individual lengths (one +//! independent position choice per sequence in the group), not `N^r` — the +//! latter only holds because all `s` sequences share the same length `N`. +//! Dropping that assumption and summing over all `C(s, r)` groups of `s` +//! sequences with individual lengths `N_1, ..., N_s` gives +//! +//! ```text +//! S_1(k) = lambda^k * e_r(N_1, ..., N_s) +//! ``` +//! +//! where `e_r` is the degree-`r` [elementary symmetric polynomial] in the +//! `s` lengths, `e_r(N_1,...,N_s) = sum_{i_1 < ... < i_r} N_i1 * N_i2 * ... +//! * N_ir`. Setting all `N_i = N` collapses `e_r` to `C(s, r) * N^r`, +//! recovering the equal-length formula above exactly — this is a strict +//! generalization, not a different approximation. +//! +//! [elementary symmetric polynomial]: https://en.wikipedia.org/wiki/Elementary_symmetric_polynomial +//! +//! The Remark following Theorem 2.2 in Karlin & Ost (1988) notes that for +//! the *limiting distribution* itself (not just its `S_1` parameter) to +//! converge, the `N_i` must stay of comparable order as they all grow +//! (formally, `(N_{sigma_1} * ... * N_{sigma_r}) / N^r` must converge to +//! some `alpha(sigma)` for a common reference scale `N`) — in practice this +//! is a non-issue unless genome sizes differ by several orders of +//! magnitude. +//! +//! `N_i` here is the sequence length proper; as a first approximation it +//! can be taken as the genome's k-mer set cardinality (or total k-mer +//! occurrence count), which is already what OBIKmer computes per genome +//! and tracks the true sequence length closely enough for this purpose. +//! +//! # Validity +//! +//! - The `exp{-(1-lambda) S_1}` form (Poisson approximation) is valid +//! whenever the per-position match probability `lambda^k` is small, which +//! holds throughout OBIKmer's supported range (`k in [11, 31]`) — it does +//! **not** require `S_1` itself to be small. +//! - The further linearisation `1 - exp(-x) ~= x` used by callers that want +//! `P(k) ~= (1 - lambda) * S_1(N, k)` **does** require `(1-lambda) * +//! S_1(N, k) << 1`; callers must check this themselves (or just use +//! [`p_at_least_one`], which always uses the exponential form and is +//! valid regardless of scale). +//! - This module only covers the *unrestricted-position* case (`K_{r,s}`, +//! matches anywhere in each sequence, not aligned at a common start) and +//! *distinct* sequences (`s` independent realizations). The single +//! sequence self-repeat case (a genome compared to itself, positions +//! drawn from the same realization) needs the reverse-complement variant +//! `R(N)` treated separately in Karlin & Ost (1987); it is not +//! implemented here. +//! +//! # References +//! +//! - Karlin, S. and Ost, F. (1988). "Maximal length of common words among +//! random letter sequences." *The Annals of Probability*, 16(2):535-563. +//! [`Theorem 2.2`] — the unrestricted-position `K_{r,s}(N)` result +//! implemented here. DOI: 10.1214/aop/1176991772. +//! - Karlin, S. and Ost, F. (1987). "Counts of long aligned word matches +//! among random letter sequences." *Advances in Applied Probability*, +//! 19(2):293-351. Companion paper for the *aligned* variant `C_{r,s}(N)` +//! (fixed common start position across sequences, relevant to local +//! alignment significance, not to unrestricted repeat-finding) — not +//! implemented here. +//! - Karlin, S. and Ost, F. (1985). "Maximal segmental match length among +//! random sequences from a finite alphabet." In *Proceedings of the +//! Berkeley Conference in Honor of Jerzy Neyman and Jack Kiefer*, vol. 1, +//! pp. 225-243. Wadsworth. No DOI found (pre-DOI conference proceedings). +//! Evaluates the reverse-complement match parameter needed for the +//! self-repeat case `R(N)` mentioned above. + +/// DNA alphabet size (`A`, `C`, `G`, `T`), assumed equiprobable and +/// independent — the same null-model assumption used throughout +/// `UserDocMD/theory/kmer_indexing/kmers.md`. +const DNA_ALPHABET_SIZE: f64 = 4.0; + +/// `C(n, k)`, the binomial coefficient, computed as `f64` since `S_1` is +/// itself only ever an asymptotic approximation — exact integer precision +/// is not meaningful here. +fn binomial(n: u32, k: u32) -> f64 { + if k > n { + return 0.0; + } + let k = k.min(n - k); + let mut result = 1.0f64; + for i in 0..k { + result *= (n - i) as f64 / (i + 1) as f64; + } + result +} + +/// The Karlin-Ost characteristic local match parameter `lambda^[r] = m^(1-r)` +/// for `r` i.i.d. equiprobable sequences over an alphabet of size `m` +/// (eq. 2.5-2.6 of Karlin & Ost 1988). +/// +/// `alphabet_size` defaults to [`DNA_ALPHABET_SIZE`] for the DNA case via +/// [`dna_lambda`]; this general form is kept for documentation fidelity to +/// the source formula and for non-DNA alphabets (e.g. amino acids). +fn lambda(alphabet_size: f64, r: u32) -> f64 { + alphabet_size.powi(1 - r as i32) +} + +/// [`lambda`] specialized to the DNA alphabet (`m = 4`). +pub fn dna_lambda(r: u32) -> f64 { + lambda(DNA_ALPHABET_SIZE, r) +} + +/// Degree-`r` elementary symmetric polynomial `e_r(lengths)` — see module +/// docs, "Unequal genome lengths". Computed by the standard `O(r * s)` DP: +/// `e[j]` accumulates the sum over all `j`-subsets of the lengths seen so +/// far, updated one length at a time from high `j` to low so each length is +/// used at most once per subset. +/// +/// Returns `0.0` if `r > lengths.len()`. +fn elementary_symmetric(lengths: &[f64], r: u32) -> f64 { + let r = r as usize; + if r > lengths.len() { + return 0.0; + } + let mut e = vec![0.0f64; r + 1]; + e[0] = 1.0; + for &length in lengths { + for j in (1..=r).rev() { + e[j] += length * e[j - 1]; + } + } + e[r] +} + +/// Expected number of chance `r`-fold common k-words among `C(s, r)` groups +/// of `s` independent i.i.d. equiprobable DNA sequences of individual +/// lengths `lengths` (Karlin & Ost 1988, eq. 2.9, generalized to unequal +/// lengths via the degree-`r` elementary symmetric polynomial — see module +/// docs, "Unequal genome lengths" — with the i.i.d. `gamma = 1` +/// simplification derived in the module docs). +/// +/// `lengths` can be exact sequence lengths, or, as a first approximation, +/// each genome's k-mer set cardinality (or total k-mer occurrence count). +/// +/// # Panics +/// +/// Does not panic; returns `0.0` if `r > lengths.len()`. +pub fn expected_shared_kmers_varying(lengths: &[f64], k: u32, r: u32) -> f64 { + elementary_symmetric(lengths, r) * dna_lambda(r).powi(k as i32) +} + +/// Expected number of chance `r`-fold common k-words among `C(s, r)` groups +/// of `s` independent i.i.d. equiprobable DNA sequences of length `n` +/// (Karlin & Ost 1988, eq. 2.9, with the i.i.d. `gamma = 1` simplification +/// derived in the module docs). +/// +/// `r = 2, s = 2` recovers the plain pairwise chance-collision count for +/// two sequences; `r = s` recovers Karlin-Ost's within-a-single-genome +/// growth rate scaled by the number of independent sequences considered. +/// +/// Closed-form fast path for the equal-length case — equivalent to +/// [`expected_shared_kmers_varying`] called with `s` copies of `n`, since +/// `e_r(n,...,n) = C(s, r) * n^r` (see module docs, "Unequal genome +/// lengths"). +/// +/// # Panics +/// +/// Does not panic; returns `0.0` if `r > s`. +pub fn expected_shared_kmers(n: f64, k: u32, r: u32, s: u32) -> f64 { + n.powi(r as i32) * binomial(s, r) * dna_lambda(r).powi(k as i32) +} + +/// Probability that at least one k-mer of length `k` is shared, purely by +/// chance, by at least `r` of `s` independent i.i.d. equiprobable DNA +/// sequences of individual lengths `lengths` (Karlin & Ost 1988, Theorem +/// 2.2 / eq. 2.12, exponential form, generalized to unequal lengths). +/// +/// Valid across the full range relevant to OBIKmer (`k` in `[11, 31]`) +/// regardless of whether [`expected_shared_kmers_varying`] itself is small +/// or large — unlike the linear approximation `P ~= (1 - lambda) * +/// expected_shared_kmers_varying(..)`, which additionally requires the +/// latter to be `<< 1` (see module docs, "Validity"). +pub fn p_at_least_one_varying(lengths: &[f64], k: u32, r: u32) -> f64 { + let s1 = expected_shared_kmers_varying(lengths, k, r); + let lambda = dna_lambda(r); + -f64::exp_m1(-(1.0 - lambda) * s1) +} + +/// Probability that at least one k-mer of length `k` is shared, purely by +/// chance, by at least `r` of `s` independent i.i.d. equiprobable DNA +/// sequences of length `n` (Karlin & Ost 1988, Theorem 2.2 / eq. 2.12, +/// exponential form). +/// +/// Valid across the full range relevant to OBIKmer (`k` in `[11, 31]`) +/// regardless of whether [`expected_shared_kmers`] itself is small or +/// large — unlike the linear approximation `P ~= (1 - lambda) * +/// expected_shared_kmers(..)`, which additionally requires the latter to +/// be `<< 1` (see module docs, "Validity"). +pub fn p_at_least_one(n: f64, k: u32, r: u32, s: u32) -> f64 { + let s1 = expected_shared_kmers(n, k, r, s); + let lambda = dna_lambda(r); + -f64::exp_m1(-(1.0 - lambda) * s1) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Reproduces the r=2, s=2, N=1Gb, k=31 example worked out by hand: + /// S_1 ~= 0.217, P ~= 0.150. + #[test] + fn two_genomes_one_gb_k31() { + let s1 = expected_shared_kmers(1e9, 31, 2, 2); + assert!((s1 - 0.217).abs() < 0.01, "S_1 = {s1}"); + + let p = p_at_least_one(1e9, 31, 2, 2); + assert!((p - 0.150).abs() < 0.01, "P = {p}"); + } + + /// Reproduces the r=5, s=10, N=10Gb, k=31 example: S_1 ~= 5.6e-23. + #[test] + fn five_of_ten_genomes_ten_gb_k31() { + let s1 = expected_shared_kmers(1e10, 31, 5, 10); + assert!(s1 < 1e-20, "S_1 = {s1}"); + assert!(s1 > 1e-25, "S_1 = {s1}"); + } + + /// Reproduces the r=3, s=10, N=10Gb, k=31 example: S_1 ~= 5.6e-6. + #[test] + fn three_of_ten_genomes_ten_gb_k31() { + let s1 = expected_shared_kmers(1e10, 31, 3, 10); + assert!((s1 - 5.6e-6).abs() < 1e-6, "S_1 = {s1}"); + } + + /// `expected_shared_kmers_varying` with `s` equal lengths must agree + /// with the closed-form `expected_shared_kmers`, since `e_r(n,...,n) = + /// C(s, r) * n^r`. + #[test] + fn varying_matches_closed_form_when_lengths_equal() { + let lengths = vec![1e10; 10]; + let general = expected_shared_kmers_varying(&lengths, 31, 3); + let closed_form = expected_shared_kmers(1e10, 31, 3, 10); + assert!( + (general - closed_form).abs() / closed_form < 1e-9, + "general = {general}, closed_form = {closed_form}" + ); + } + + /// `e_2(a, b) = a * b` for two sequences — direct check of the + /// unequal-length case against hand computation, using an 8 Gb and a + /// 12 Gb genome at k=21. + #[test] + fn two_genomes_unequal_lengths_k21() { + let lengths = [8e9, 12e9]; + let s1 = expected_shared_kmers_varying(&lengths, 21, 2); + let expected = 8e9 * 12e9 * dna_lambda(2).powi(21); + assert!((s1 - expected).abs() / expected < 1e-9, "S_1 = {s1}"); + } +} diff --git a/src/obikphylo/src/lib.rs b/src/obikphylo/src/lib.rs index 08b8d4cd..7e80a1bc 100644 --- a/src/obikphylo/src/lib.rs +++ b/src/obikphylo/src/lib.rs @@ -7,6 +7,13 @@ //! cardinality, pseudo-alignment); further phylo-domain functionality //! (currently [`distance`]) moves here incrementally. //! +//! [`karlin`] holds standalone chance-collision statistics (Karlin & Ost) +//! for judging whether a k-mer shared by several genomes/layers is +//! statistically significant; kept here for now as it has no other +//! consumer yet, but it depends on nothing else in this crate and may move +//! to a lower-level crate if an indexing-side consumer (e.g. default-`k` +//! guidance) appears. +//! //! `siblings` is being reconnected piece by piece while `obikindex`'s //! stateless audit is caught up with — see the project memory on this. So //! far it only carries the annex-construction path @@ -15,6 +22,7 @@ //! still disconnected. mod distance; +pub mod karlin; mod tree; pub mod siblings; diff --git a/src/obiksession/Cargo.toml b/src/obiksession/Cargo.toml index 3ee4a5a2..1845cf5b 100644 --- a/src/obiksession/Cargo.toml +++ b/src/obiksession/Cargo.toml @@ -8,6 +8,8 @@ obisys = { path = "../obisys", default-features = false } memmap2 = "0.9.11" xxhash-rust = { version = "0.8.18", features = ["xxh3"] } tracing = "0.1" +serde = { version = "1", optional = true } [dev-dependencies] +approx = "0.5.1" tempfile = "3" diff --git a/src/obiksession/src/lib.rs b/src/obiksession/src/lib.rs index 1c7b8b31..a7d445f1 100644 --- a/src/obiksession/src/lib.rs +++ b/src/obiksession/src/lib.rs @@ -13,7 +13,22 @@ //! already used elsewhere in this project between a generic mechanism //! crate and the domain crate that plugs into it (`obisys`/`obikindex`, //! `obicompactvec`/`obikindexer`). +//! +//! ## Memoization +//! +//! [`Session::memoize`] returns a [`Memoize`] handle that lets callers +//! memoize expensive computations on top of the same checksummed, +//! persist-on-disk artifact store. Two API styles are provided: +//! +//! * **Manual** — [`Memoize::get_or_compute_with_key`] with an explicit +//! string key, or [`Memoize::get_or_compute`] with auto-derived keys +//! via the [`MemoKey`] trait. +//! * **Declarative** — the `session_memoize!` macro wraps a function so +//! that recursive calls transparently consult the cache, similar to +//! `kash`/`cached`. +mod memoize; mod session; +pub use memoize::{MemoKey, Memoize, MemoizeValue}; pub use session::{ParamsConflict, Session, SessionOutcome}; diff --git a/src/obiksession/src/memoize.rs b/src/obiksession/src/memoize.rs new file mode 100644 index 00000000..8271bc15 --- /dev/null +++ b/src/obiksession/src/memoize.rs @@ -0,0 +1,511 @@ +use std::fmt; +use std::io; +use std::path::PathBuf; + +use memmap2::Mmap; + +use crate::Session; +use crate::session::{MEMO_DIR, checksum_path_for, verify_mmap, write_checked}; + +/// Trait for types that can be used as memoization keys. +/// +/// The default implementation uses `Debug` + `xxh3` to produce a +/// stable string key from any `fmt::Debug` type. Callers can +/// override this to supply a custom key (e.g. a pre-hashed value +/// or a semantic key). +pub trait MemoKey: fmt::Debug { + fn memo_key(&self, function_name: &str) -> String; +} + +impl MemoKey for T { + fn memo_key(&self, function_name: &str) -> String { + let payload = format!("{}:{:?}", function_name, self); + xxhash_rust::xxh3::xxh3_64(payload.as_bytes()).to_string() + } +} + +/// Marker trait for types that can be stored in the memoization +/// cache. The default blanket impl covers all `Vec` values. +pub trait MemoizeValue: Sized { + fn to_bytes(&self) -> Vec; + fn from_bytes(bytes: &[u8]) -> io::Result; +} + +impl MemoizeValue for Vec { + fn to_bytes(&self) -> Vec { + self.clone() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + Ok(bytes.to_vec()) + } +} + +// Blanket impl for serde types when the serde feature is enabled +#[cfg(feature = "serde")] +impl MemoizeValue for T +where + T: serde::Serialize + serde::de::DeserializeOwned, +{ + fn to_bytes(&self) -> Vec { + bincode::serialize(self).expect("serialization failed") + } + fn from_bytes(bytes: &[u8]) -> io::Result { + bincode::deserialize(bytes).map_err(io::Error::other) + } +} + +// Blanket impls for common primitive types (no serde needed) +impl MemoizeValue for u8 { + fn to_bytes(&self) -> Vec { + vec![*self] + } + fn from_bytes(bytes: &[u8]) -> io::Result { + bytes.first().copied().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "invalid u8 bytes") + }) + } +} + +impl MemoizeValue for u16 { + fn to_bytes(&self) -> Vec { + self.to_le_bytes().to_vec() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + let arr = <[u8; 2]>::try_from(bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid u16 bytes"))?; + Ok(u16::from_le_bytes(arr)) + } +} + +impl MemoizeValue for u32 { + fn to_bytes(&self) -> Vec { + self.to_le_bytes().to_vec() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + let arr = <[u8; 4]>::try_from(bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid u32 bytes"))?; + Ok(u32::from_le_bytes(arr)) + } +} + +impl MemoizeValue for u64 { + fn to_bytes(&self) -> Vec { + self.to_le_bytes().to_vec() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + let arr = <[u8; 8]>::try_from(bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid u64 bytes"))?; + Ok(u64::from_le_bytes(arr)) + } +} + +impl MemoizeValue for i32 { + fn to_bytes(&self) -> Vec { + self.to_le_bytes().to_vec() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + let arr = <[u8; 4]>::try_from(bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid i32 bytes"))?; + Ok(i32::from_le_bytes(arr)) + } +} + +impl MemoizeValue for i64 { + fn to_bytes(&self) -> Vec { + self.to_le_bytes().to_vec() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + let arr = <[u8; 8]>::try_from(bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid i64 bytes"))?; + Ok(i64::from_le_bytes(arr)) + } +} + +impl MemoizeValue for f64 { + fn to_bytes(&self) -> Vec { + self.to_le_bytes().to_vec() + } + fn from_bytes(bytes: &[u8]) -> io::Result { + let arr = <[u8; 8]>::try_from(bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid f64 bytes"))?; + Ok(f64::from_le_bytes(arr)) + } +} + +/// A memoization handle backed by a `Session` directory. +#[derive(Debug)] +pub struct Memoize<'a> { + session: &'a Session, +} + +impl<'a> Memoize<'a> { + pub(crate) fn new(session: &'a Session) -> Self { + Self { session } + } + + /// Retrieve a cached value by its explicit key, or compute and + /// store it via `compute_fn`. + /// + /// Concurrent calls for the *same* `key` (from different threads + /// sharing this session) don't race to recompute: the first caller + /// runs `compute_fn` while the others block on + /// [`Session::with_key_lock`], then each re-checks the cache — which + /// the first caller has by then populated — instead of recomputing. + pub fn get_or_compute_with_key( + &self, + key: &str, + compute_fn: impl FnOnce() -> io::Result>, + ) -> io::Result> { + if let Some(cached) = self.restore_raw(key)? { + return Ok(cached.to_vec()); + } + self.session.with_key_lock(key, || { + if let Some(cached) = self.restore_raw(key)? { + return Ok(cached.to_vec()); + } + let value = compute_fn()?; + self.store_raw(key, &value)?; + Ok(value) + }) + } + + /// Retrieve a cached value whose key is auto-derived from + /// `args` via [`MemoKey`], or compute and store it. + pub fn get_or_compute( + &self, + function_name: &str, + args: &K, + compute_fn: F, + ) -> io::Result> + where + K: MemoKey, + V: MemoizeValue, + F: FnOnce() -> io::Result, + { + let key = args.memo_key(function_name); + let raw = self.get_or_compute_with_key(&key, || { + compute_fn().map(|v| v.to_bytes()) + })?; + Ok(raw) + } + + /// Restore a raw (byte) artifact by name, `mmap`ed read-only. + fn restore_raw(&self, name: &str) -> io::Result> { + let path = self.memo_path(name); + verify_mmap(&path) + } + + /// Store raw bytes under `name`, atomically with a checksum sidecar. + fn store_raw(&self, name: &str, bytes: &[u8]) -> io::Result<()> { + let path = self.memo_path(name); + write_checked(&path, bytes) + } + + /// Remove a single memoized entry by key. Returns `true` if the + /// entry existed and was deleted. + pub fn invalidate(&self, key: &str) -> io::Result { + let path = self.memo_path(key); + let checksum_path = checksum_path_for(&path); + let existed = path.exists(); + if path.exists() { + std::fs::remove_file(&path)?; + } + if checksum_path.exists() { + let _ = std::fs::remove_file(&checksum_path); + } + Ok(existed) + } + + pub fn cached_keys(&self) -> io::Result> { + let dir = self.memo_dir(); + if !dir.exists() { + return Ok(Vec::new()); + } + let mut keys = Vec::new(); + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy().to_string(); + if name.ends_with(".xxh3") || name.ends_with(".tmp") { + continue; + } + keys.push(name); + } + Ok(keys) + } + + fn memo_dir(&self) -> PathBuf { + self.session.dir().join(MEMO_DIR) + } + + fn memo_path(&self, name: &str) -> PathBuf { + self.memo_dir().join(name) + } +} + + +/// Declarative memoization macro, similar in spirit to `kash`/`cached`. +/// +/// Wraps a function so that every call transparently consults the +/// session's persistent cache. On a cache miss the function body +/// is executed and its return value is stored; on a hit the stored +/// value is returned without re-executing the body. +/// +/// # Syntax +/// +/// ```ignore +/// session_memoize! { +/// fn my_func(session: &Session, a: u64, b: &str) -> MyResult { +/// // expensive computation +/// } +/// } +/// ``` +/// +/// After expansion, call `my_func(&session, 5, "hello")` — the function +/// returns `io::Result`. Check the result with `.unwrap()` or +/// `.expect()`. +/// +/// Recursive calls within the body to `my_func` also pass through the +/// cache (they must also include the session as the first argument). +/// +/// # Requirements +/// +/// * The first parameter must be `session: &Session`. +/// * All argument types must implement [`MemoKey`] (provided for `Debug` types). +/// * The return type must implement [`MemoizeValue`] (provided for +/// `Vec`, primitive numeric types, and, when the `serde` +/// feature is enabled, for `Serialize + DeserializeOwned` types). +#[macro_export] +macro_rules! session_memoize { + ( + fn $name:ident(session: &Session, $($arg:ident : $arg_ty:ty),*) -> $ret:ty $body:block + ) => { + fn $name(session: &Session, $($arg: $arg_ty),*) -> ::std::io::Result<$ret> { + let memo = session.memoize(); + let args = (&($($arg),*),); + let raw = memo.get_or_compute( + stringify!($name), + &args, + || -> ::std::io::Result<$ret> { + let __result: $ret = $body; + Ok(__result) + }, + )?; + <$ret as $crate::MemoizeValue>::from_bytes(&raw) + .map_err(|e| ::std::io::Error::new(::std::io::ErrorKind::Other, e)) + } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memoize_get_or_compute_miss_then_hit() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + let memo = Memoize::new(&session); + let computed = memo.get_or_compute_with_key("key1", || Ok(b"value1".to_vec())).unwrap(); + assert_eq!(&computed[..], b"value1"); + let cached = memo.get_or_compute_with_key("key1", || panic!("should not be called")).unwrap(); + assert_eq!(&cached[..], b"value1"); + } + + #[test] + fn memoize_invalidate() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + let memo = Memoize::new(&session); + memo.get_or_compute_with_key("key1", || Ok(b"value1".to_vec())).unwrap(); + assert!(memo.invalidate("key1").unwrap()); + assert!(memo.restore_raw("key1").unwrap().is_none()); + assert!(!memo.invalidate("key1").unwrap()); + } + + #[test] + fn memoize_cached_keys() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + let memo = Memoize::new(&session); + assert!(memo.cached_keys().unwrap().is_empty()); + memo.get_or_compute_with_key("a", || Ok(vec![1])).unwrap(); + memo.get_or_compute_with_key("b", || Ok(vec![2])).unwrap(); + let keys = memo.cached_keys().unwrap(); + assert!(keys.contains(&"a".to_string())); + assert!(keys.contains(&"b".to_string())); + assert_eq!(keys.len(), 2); + } + + #[test] + fn memo_key_from_debug() { + #[derive(Debug)] + struct Args { + x: u64, + y: String, + } + let args = Args { x: 42, y: "hello".into() }; + let key1 = args.memo_key("my_fn"); + let key2 = args.memo_key("my_fn"); + assert_eq!(key1, key2); + let key_other = args.memo_key("other_fn"); + assert_ne!(key1, key_other); + } + + #[test] + fn session_memoize_macro_u64() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + + session_memoize! { + fn double(session: &Session, x: u64) -> u64 { + x * 2 + } + } + + let result = double(&session, 7).unwrap(); + assert_eq!(result, 14); + + let result2 = double(&session, 7).unwrap(); + assert_eq!(result2, 14); + } + + #[test] + fn session_memoize_macro_i32() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + + session_memoize! { + fn add(session: &Session, a: i32, b: i32) -> i32 { + a + b + } + } + + let result = add(&session, 3, 4).unwrap(); + assert_eq!(result, 7); + + let result2 = add(&session, 3, 4).unwrap(); + assert_eq!(result2, 7); + } + + #[test] + fn session_memoize_macro_f64() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + + session_memoize! { + fn square(session: &Session, x: f64) -> f64 { + x * x + } + } + + let result = square(&session, 3.0).unwrap(); + approx::assert_relative_eq!(result, 9.0); + + let result2 = square(&session, 3.0).unwrap(); + approx::assert_relative_eq!(result2, 9.0); + } + + + #[test] + fn session_memoize_macro_persists_across_sessions() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + + // Define the memoized function once + session_memoize! { + fn cached_add(session: &Session, a: u64, b: u64) -> u64 { + a + b + } + } + + // First session: compute and cache + { + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + + assert_eq!(cached_add(&session, 3, 4).unwrap(), 7); + } + + // Second session with same params: should hit cache + { + let session = Session::open(&dir, b"params", false).unwrap().unwrap(); + + assert_eq!(cached_add(&session, 3, 4).unwrap(), 7); + } + } + + #[test] + fn session_memoize_macro_invalidated_by_force_reset() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + + session_memoize! { + fn tagged(session: &Session, x: u64) -> u64 { + x + } + } + + // First session: compute and cache under "params-v1". + { + let session = Session::open(&dir, b"params-v1", false).unwrap().unwrap(); + assert_eq!(tagged(&session, 1).unwrap(), 1); + assert_eq!(session.memoize().cached_keys().unwrap().len(), 1); + } + + // Reopen with different params and force: true — the whole + // session, including the memo cache, is reset, so the call + // below must recompute rather than reuse the stale entry. + { + let session = Session::open(&dir, b"params-v2", true).unwrap().unwrap(); + assert!(session.memoize().cached_keys().unwrap().is_empty()); + assert_eq!(tagged(&session, 1).unwrap(), 1); + } + } + + #[test] + fn concurrent_calls_for_same_key_compute_once() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("session"); + let session = std::sync::Arc::new(Session::open(&dir, b"params", false).unwrap().unwrap()); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + // Forces both threads to reach `get_or_compute_with_key` at + // essentially the same instant, so the second one is guaranteed + // to hit the "wait, don't recompute" path instead of just + // happening to run after the first one already finished. + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let handles: Vec<_> = (0..2) + .map(|_| { + let session = session.clone(); + let calls = calls.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + session + .memoize() + .get_or_compute_with_key("shared-key", || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(50)); + Ok(b"computed".to_vec()) + }) + .unwrap() + }) + }) + .collect(); + + for handle in handles { + assert_eq!(&handle.join().unwrap()[..], b"computed"); + } + + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + // The per-key lock must have been cleaned up once both threads + // are done with it, not left behind forever. + assert_eq!(session.memo_lock_table_len(), 0); + } +} diff --git a/src/obiksession/src/session.rs b/src/obiksession/src/session.rs index b057b03e..878929a2 100644 --- a/src/obiksession/src/session.rs +++ b/src/obiksession/src/session.rs @@ -1,12 +1,15 @@ +use std::collections::HashMap; use std::fs; use std::io; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use memmap2::Mmap; use obisys::DirLock; const PARAMS_FILE: &str = "params.bin"; const ARTIFACTS_DIR: &str = "artifacts"; +pub(crate) const MEMO_DIR: &str = "memo"; const CHECKSUM_EXT: &str = "xxh3"; /// Whether [`Session::open`] found a fresh (just-created, or just-reset by @@ -48,6 +51,9 @@ pub struct Session { dir: PathBuf, outcome: SessionOutcome, _lock: DirLock, + /// Per-key in-process locks backing [`Memoize`](crate::Memoize)'s + /// single-flight behavior — see [`Session::with_key_lock`]. + memo_locks: Mutex>>>, } impl Session { @@ -66,24 +72,31 @@ impl Session { dir: dir.to_path_buf(), outcome: SessionOutcome::Reused, _lock: lock, + memo_locks: Mutex::new(HashMap::new()), })), Some(saved) if !force => Ok(Err(ParamsConflict { saved, requested: params.to_vec() })), _ => { // No saved params yet, or a mismatch with `force: true`: - // start clean. Any artifact cached under different params - // is stale by construction — deleting the whole - // `artifacts/` directory is simpler and safer than trying - // to figure out which entries are still valid. + // start clean. Any artifact or memoized entry cached under + // different params is stale by construction — deleting the + // whole `artifacts/` and `memo/` directories is simpler and + // safer than trying to figure out which entries are still + // valid. let artifacts_dir = dir.join(ARTIFACTS_DIR); if artifacts_dir.exists() { fs::remove_dir_all(&artifacts_dir)?; } fs::create_dir_all(&artifacts_dir)?; + let memo_dir = dir.join(MEMO_DIR); + if memo_dir.exists() { + fs::remove_dir_all(&memo_dir)?; + } write_checked(¶ms_path, params)?; Ok(Ok(Session { dir: dir.to_path_buf(), outcome: SessionOutcome::Created, _lock: lock, + memo_locks: Mutex::new(HashMap::new()), })) } } @@ -123,20 +136,87 @@ impl Session { pub fn store(&self, name: &str, bytes: &[u8]) -> io::Result<()> { write_checked(&self.artifact_path(name), bytes) } + + pub fn memoize(&self) -> crate::Memoize<'_> { + crate::Memoize::new(self) + } + + /// Serialize concurrent calls that share the same `key`, within this + /// process: the first caller for a given `key` runs `f` while any + /// other caller for that *same* `key` blocks; once `f` returns, each + /// blocked caller runs in turn. Backs [`Memoize`](crate::Memoize)'s + /// single-flight behavior — combined with a cache re-check inside `f`, + /// it turns "N threads race to recompute and overwrite the same + /// artifact" into "one thread computes, the rest wait and then read + /// the cache". + /// + /// Purely an in-process mechanism: cross-process exclusivity is + /// already provided by the `DirLock` held for the `Session`'s entire + /// lifetime, so contention here only ever happens between threads of + /// this one process sharing the same `Session`. + /// + /// The per-key lock is garbage-collected: its entry is removed as + /// soon as no other thread is (or could still be) waiting on it, so a + /// long-running process doesn't accumulate one entry per distinct key + /// ever requested — only per key currently or recently contended. + pub(crate) fn with_key_lock(&self, key: &str, f: impl FnOnce() -> T) -> T { + let entry = { + let mut table = self.memo_locks.lock().unwrap(); + table.entry(key.to_string()).or_insert_with(|| Arc::new(Mutex::new(()))).clone() + }; + + let guard = entry.lock().unwrap(); + let result = f(); + + // Still holding `guard`: any other thread that already cloned + // `entry` out of the table is necessarily still blocked trying to + // acquire it below — it cannot have run `f` and returned yet — so + // `entry`'s strong count here is exact. `table` holds one + // reference and this call holds the other (`entry` itself); if + // that is still all of them, nobody else is or ever will be + // waiting on this particular entry, so it is safe to drop it from + // the table. A later call for the same `key` just starts a fresh + // entry. + { + let mut table = self.memo_locks.lock().unwrap(); + if Arc::strong_count(&entry) == 2 { + table.remove(key); + } + } + drop(guard); + + result + } + + /// Number of currently-contended per-key locks — exposed only for + /// tests, to check that [`with_key_lock`](Self::with_key_lock)'s + /// garbage collection actually removes entries once uncontended. + #[cfg(test)] + pub(crate) fn memo_lock_table_len(&self) -> usize { + self.memo_locks.lock().unwrap().len() + } } -fn checksum_path_for(path: &Path) -> PathBuf { +pub(crate) fn checksum_path_for(path: &Path) -> PathBuf { let mut os_str = path.as_os_str().to_owned(); os_str.push("."); os_str.push(CHECKSUM_EXT); PathBuf::from(os_str) } -/// `mmap` `path` and verify it against its checksum sidecar. `Ok(None)` -/// when `path` doesn't exist, its checksum sidecar doesn't exist/parse, or -/// the checksum doesn't match — every case here means "nothing usable is -/// cached," never an error the caller has to handle specially. -fn verify_mmap(path: &Path) -> io::Result> { +/// Append `.tmp` to `path`'s full file name (never replaces an existing +/// extension the way [`Path::with_extension`] would). Used to derive the +/// staging path for both the data file and the checksum sidecar in +/// [`write_checked`] — appending rather than replacing guarantees the two +/// staging paths can never collide with each other, nor with an unrelated +/// artifact whose name happens to contain a `.`. +fn tmp_path_for(path: &Path) -> PathBuf { + let mut os_str = path.as_os_str().to_owned(); + os_str.push(".tmp"); + PathBuf::from(os_str) +} + +pub(crate) fn verify_mmap(path: &Path) -> io::Result> { if !path.exists() { return Ok(None); } @@ -169,17 +249,17 @@ fn read_checked(path: &Path) -> io::Result>> { Ok(verify_mmap(path)?.map(|mmap| mmap[..].to_vec())) } -fn write_checked(path: &Path, bytes: &[u8]) -> io::Result<()> { +pub(crate) fn write_checked(path: &Path, bytes: &[u8]) -> io::Result<()> { if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; } - let tmp = path.with_extension("tmp"); + let tmp = tmp_path_for(path); fs::write(&tmp, bytes)?; fs::rename(&tmp, path)?; let checksum = xxhash_rust::xxh3::xxh3_64(bytes); let checksum_path = checksum_path_for(path); - let checksum_tmp = checksum_path.with_extension("tmp"); + let checksum_tmp = tmp_path_for(&checksum_path); fs::write(&checksum_tmp, checksum.to_string())?; fs::rename(&checksum_tmp, checksum_path)?; Ok(()) -- 2.54.1 From 1702addbf9b6089d14feb18629e2d8a853b4b52f Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 18 Sep 2026 11:24:53 +0200 Subject: [PATCH 04/10] Refactor algorithms for unified sampling and reduction Introduces a unified sampling trunk, new reducer traits, and refined logic for SNP distance calculation and session handling. Enforces strict parameter sharing across all algorithms to ensure consistent sampling. --- src/Cargo.lock | 1 - src/obikmer/src/cmd/phylo/mod.rs | 213 +++++++---------- src/obikphylo/Cargo.toml | 1 - .../src/siblings/algorithms/alignment.rs | 97 ++++---- src/obikphylo/src/siblings/algorithms/mod.rs | 12 +- .../src/siblings/algorithms/pairwise.rs | 45 +++- .../src/siblings/algorithms/sankoff.rs | 224 +++++------------- src/obikphylo/src/siblings/algorithms/scan.rs | 70 ++++++ .../src/siblings/algorithms/session_cache.rs | 101 -------- .../src/siblings/algorithms/snp_distance.rs | 160 ++----------- .../src/siblings/extensions/sibling_ext.rs | 149 +----------- src/obikphylo/src/siblings/mod.rs | 7 +- 12 files changed, 320 insertions(+), 760 deletions(-) create mode 100644 src/obikphylo/src/siblings/algorithms/scan.rs delete mode 100644 src/obikphylo/src/siblings/algorithms/session_cache.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index 2aac7e69..beb2b3ff 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1594,7 +1594,6 @@ dependencies = [ "obikindex", "obikindexer", "obikseq", - "obiksession", "obipipeline", "obiread", "obiskbuilder", diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 888c7367..e77ffa98 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -24,43 +24,6 @@ use phylip::write_phylip_relaxed; use sankoff::{write_sankoff_alignment_fasta, write_sankoff_matrix_csv, write_sankoff_params}; use tnt::write_sankoff_tnt; -/// Deterministic byte encoding of the `snp-*` `--distance` selection -/// parameters, for `obiksession::Session::open`'s params-conflict check. -/// `obiksession` treats this as an opaque blob (see its own crate docs) — -/// only this module needs to know the layout, and only well enough to -/// produce/compare it, never to pretty-print a per-field diff (v1 keeps -/// the conflict message generic rather than decoding both sides field by -/// field — see its use in `run` below). -fn session_params_blob( - n: Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, -) -> Vec { - let mut buf = Vec::new(); - match n { - Some(v) => { - buf.push(1); - buf.extend_from_slice(&(v as u64).to_le_bytes()); - } - None => buf.push(0), - } - buf.push(free_loss as u8); - buf.push(no_ambiguity as u8); - buf.extend_from_slice(&(excluded.len() as u64).to_le_bytes()); - buf.extend(excluded.iter().map(|&b| b as u8)); - match entropy_bias { - Some(EntropyBias { mu, sigma }) => { - buf.push(1); - buf.extend_from_slice(&mu.to_le_bytes()); - buf.extend_from_slice(&sigma.to_le_bytes()); - } - None => buf.push(0), - } - buf -} - pub use args::PhyloArgs; pub fn run(args: PhyloArgs) { @@ -273,63 +236,83 @@ pub fn run(args: PhyloArgs) { None }; - // ── `--session`: persist the snp-* sample across separate invocations ── - // Only covers `snp-*` `--distance` today (see `--session`'s own docs); - // `None` when `--session` isn't given, matching today's always-resample - // default exactly. - let session = args.session.as_ref().map(|dir| { - let params = session_params_blob( - args.subsample, - args.free_loss, - args.no_ambiguity, - &snp_exclude_mask, - entropy_bias, + // ── `--session` ────────────────────────────────────────────────────────── + // Temporarily unavailable: the per-driver checkpoint/resume plumbing + // it relied on (`algorithms::session_cache`) was removed along with + // the three independent sampling drivers it patched over (see + // `algorithms::scan`'s own module docs) — persistence needs to be + // rebuilt on top of the new single `sampled_scan` trunk instead of + // silently becoming a no-op. + if args.session.is_some() { + eprintln!( + "error: --session is temporarily unavailable (being rebuilt on top of the unified \ + sampling trunk) — rerun without --session/--session-force" ); - match obiksession::Session::open(dir, ¶ms, args.session_force) { - Ok(Ok(session)) => { - match session.outcome() { - obiksession::SessionOutcome::Created => { - info!("--session {}: starting fresh", dir.display()); - } - obiksession::SessionOutcome::Reused => { - info!("--session {}: found, will reuse its cached sample if present", dir.display()); - } - } - session - } - Ok(Err(_conflict)) => { - eprintln!( - "error: --session {} was built with different selection parameters \ - (--subsample/--free-loss/--no-ambiguity/--exclude-genome/--min-shared-family/\ - --entropy/--entropy-sd) than this run — pass --session-force to discard its \ - cached sample and start fresh, or point --session at a different directory", - dir.display() - ); - std::process::exit(1); - } - Err(e) => { - eprintln!("error opening --session {}: {e}", dir.display()); - std::process::exit(1); - } - } + std::process::exit(1); + } + + // ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ── + // All three consume exactly the same selection parameters + // (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/ + // `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from + // the *same* random sample when combined in one invocation — so + // whichever of them are requested here share exactly one + // `sampled_scan` call, each contributing only the reducer(s) it needs, + // instead of each running its own scan (which used to silently draw a + // different sample per flag — see `obikphylo::siblings::LayerReducer`'s + // own module docs). + let sankoff_requested = args.sankoff || args.tnt || args.phyg || args.iqtree; + let snp_kind = args.distance.as_snp(); + let need_alignment = args.pseudo_alignment || sankoff_requested; + let need_pairwise = sankoff_requested || snp_kind.is_some(); + + if args.pseudo_alignment && args.subsample.is_none() { + eprintln!("error: --pseudo-alignment requires --subsample "); + std::process::exit(1); + } + if sankoff_requested && args.subsample.is_none() { + eprintln!("error: --sankoff requires --subsample "); + std::process::exit(1); + } + + let mut alignment_reducer = need_alignment.then(|| { + obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity) }); + let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n)); - // ── SNP pseudo-alignment (`--pseudo-alignment`) ───────────────────────────── - if args.pseudo_alignment { - let Some(subsample_n) = args.subsample else { - eprintln!("error: --pseudo-alignment requires --subsample "); - std::process::exit(1); - }; + if need_alignment || need_pairwise { + let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new(); + if let Some(r) = alignment_reducer.as_mut() { + reducers.push(r); + } + if let Some(r) = pairwise_reducer.as_mut() { + reducers.push(r); + } - info!("sampling SNP pseudo-alignment (target {subsample_n} site(s))"); - let t = Stage::start("pseudo_alignment"); - let alignment = cache - .snp_pseudo_alignment(subsample_n, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias) + info!( + "sampling SNP sites{}", + match args.subsample { + Some(n) => format!(" (target {n} site(s))"), + None => " (exhaustive)".into(), + } + ); + let t = Stage::start("sampled_scan"); + obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers) .unwrap_or_else(|e| { - eprintln!("error building pseudo-alignment: {e}"); + eprintln!("error sampling SNP sites: {e}"); std::process::exit(1); }); rep.push(t.stop()); + } + + // Finished once here (not left as reducers) so `--pseudo-alignment` + // and the Sankoff group can each read it below without fighting over + // which one consumes it — see `SankoffBundle::finish`'s own docs. + let alignment = alignment_reducer.map(obikphylo::siblings::AlignmentReducer::finish); + + // ── SNP pseudo-alignment (`--pseudo-alignment`) ───────────────────────────── + if args.pseudo_alignment { + let alignment = alignment.as_ref().expect("scanned above: need_alignment"); let path = args.output.as_ref() .map(|p| format!("{}_alignment.fasta", p.display())) @@ -350,33 +333,17 @@ pub fn run(args: PhyloArgs) { // `snp-*` `--distance` branch below can reuse its already-sampled // tally instead of resampling — see `SankoffBundle::snp_distance`'s own // docs for why that's required for correctness, not just speed, when - // both are requested in the same invocation (they always share the - // same `n`/`free_loss`/`no_ambiguity`/excluded-set/`entropy_bias`: the - // CLI has no way to give them different values in one run). + // both are requested in the same invocation. let mut bundle = None; - if args.sankoff || args.tnt || args.phyg || args.iqtree { - let Some(subsample_n) = args.subsample else { - eprintln!("error: --sankoff requires --subsample "); - std::process::exit(1); - }; - - info!("sampling Sankoff calibration bundle (target {subsample_n} site(s))"); - let t = Stage::start("sankoff_bundle"); - let b = cache - .sankoff_bundle( - subsample_n, - args.free_loss, - args.no_ambiguity, - &snp_exclude_mask, - entropy_bias, - args.sankoff_ratio_ceiling, - session.as_ref(), - ) - .unwrap_or_else(|e| { - eprintln!("error computing Sankoff calibration bundle: {e}"); - std::process::exit(1); - }); - rep.push(t.stop()); + if sankoff_requested { + let alignment = alignment.clone().expect("scanned above: need_alignment"); + let pairwise = pairwise_reducer.take().expect("scanned above: need_pairwise"); + let b = obikphylo::siblings::SankoffBundle::finish( + alignment, + pairwise, + args.sankoff_ratio_ceiling, + &snp_exclude_mask, + ); let p_card = cardinality_transition_probs(&b.cardinality_tally); let p_comp = composition_transition_probs(&b.base_pair_tally); @@ -441,7 +408,7 @@ pub fn run(args: PhyloArgs) { eprintln!("error: --shared-kmers has no meaning for a snp-* --distance value"); std::process::exit(1); } - let kind = args.distance.as_snp().expect("DistanceArg is always classic or snp"); + let kind = snp_kind.expect("DistanceArg is always classic or snp"); let gamma_shape = args.gamma_shape.map_or(GammaShape::Disabled, Into::into); let t = Stage::start("snp_distance"); let matrix = match &bundle { @@ -453,23 +420,9 @@ pub fn run(args: PhyloArgs) { b.snp_distance(kind, gamma_shape) } None => { - info!( - "computing {kind:?} SNP distance for {n} genome(s){}", - match args.subsample { - Some(n) => format!(" (subsampled, target {n} site(s))"), - None => " (exhaustive)".into(), - } - ); - cache.snp_distance( - kind, - args.subsample, - args.free_loss, - args.no_ambiguity, - &snp_exclude_mask, - entropy_bias, - gamma_shape, - session.as_ref(), - ) + info!("computing {kind:?} SNP distance for {n} genome(s) (reusing the sampled scan above)"); + let (tally, dispersion) = pairwise_reducer.take().expect("scanned above: need_pairwise").finish(); + obikphylo::siblings::distance_matrix(&tally, &dispersion, kind, gamma_shape) } } .unwrap_or_else(|e| { diff --git a/src/obikphylo/Cargo.toml b/src/obikphylo/Cargo.toml index 08a6a0dd..a088f00d 100644 --- a/src/obikphylo/Cargo.toml +++ b/src/obikphylo/Cargo.toml @@ -12,7 +12,6 @@ obicompactvec = { path = "../obicompactvec" } obikidxcache = { path = "../obikidxcache" } obiskbuilder = { path = "../obiskbuilder" } obipipeline = { path = "../obipipeline" } -obiksession = { path = "../obiksession" } kodama = "0.3.0" memmap2 = "0.9" ndarray = "0.17" diff --git a/src/obikphylo/src/siblings/algorithms/alignment.rs b/src/obikphylo/src/siblings/algorithms/alignment.rs index 63339bc4..1371fc41 100644 --- a/src/obikphylo/src/siblings/algorithms/alignment.rs +++ b/src/obikphylo/src/siblings/algorithms/alignment.rs @@ -1,27 +1,25 @@ //! SNP-only pseudo-alignment: one row (IUPAC-coded) per genome, one column -//! per sampled family — built directly on [`sample_index`], so every family -//! this produces has already survived both its Bernoulli draw and post-hoc +//! per sampled family — built as a [`LayerReducer`] over +//! [`super::scan::sampled_scan`]'s per-layer batches, so every family this +//! produces has already survived both its Bernoulli draw and post-hoc //! masking ([`super::masking::survives_masking`]). use obikidxcache::index_cache::IndexCache; -use obikindex::OKIResult; use rkyv::{Archive, Deserialize, Serialize}; use super::masking::{iupac_code, masked_state}; -use super::subsample::{EntropyBias, SurvivingFamily, sample_index}; +use super::scan::LayerReducer; +use super::subsample::SurvivingFamily; /// `sequences[i]` is `genome_indices[i]`'s IUPAC-coded row — every row the -/// same length (one byte per sampled family, in the order [`sample_index`] -/// produced them: layer by layer, `family_idx` order within each). An -/// excluded genome (`excluded[g]` in -/// [`snp_pseudo_alignment`]) has no row at all, not a blanked one — its +/// same length (one byte per sampled family, in the order +/// [`super::scan::sampled_scan`] produced them: layer by layer, +/// `family_idx` order within each). An excluded genome (`excluded[g]` in +/// [`AlignmentReducer::new`]) has no row at all, not a blanked one — its /// index simply doesn't appear in `genome_indices`. `pub`, not /// `pub(crate)`: part of the public signature of -/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`]. -/// -/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): `--session` persistence -/// for `SankoffBundle` — same reasoning as `PairwiseTally`'s own derive. -#[derive(Archive, Serialize, Deserialize)] +/// [`crate::siblings::extensions::SiblingExt`]'s sampling-based methods. +#[derive(Clone, Archive, Serialize, Deserialize)] pub struct SnpAlignment { pub sequences: Vec>, /// This index's own genome numbering (matching `IndexCache::meta().genomes()`'s @@ -31,50 +29,43 @@ pub struct SnpAlignment { pub genome_indices: Vec, } -/// See [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`] -/// for the public-facing docs — this is its implementation. A single -/// "reduce" ([`reduce_alignment`]) over [`sample_index`]'s per-layer -/// batches — the only consumer today, but structured so a future sibling -/// reduce (e.g. the Sankoff cost-matrix calibration's cardinality/ -/// composition tallies) can run over the exact same batches, each cloning -/// its own `Arc`, without this function changing at all. -pub(crate) fn snp_pseudo_alignment( - cache: &IndexCache, - n: usize, +/// Accumulates a [`SnpAlignment`] over a [`super::scan::sampled_scan`] +/// pass — a thin [`LayerReducer`] wrapper around [`reduce_alignment`], so +/// it can run alongside any other reducer (e.g. +/// `algorithms::pairwise::PairwiseReducer`) over the exact same batches in +/// one shared scan, instead of each needing its own `sample_index` call. +pub struct AlignmentReducer { + genome_indices: Vec, + sequences: Vec>, free_loss: bool, no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, -) -> OKIResult { - let n_genomes = cache.meta().genomes().len(); - let genome_indices: Vec = (0..n_genomes) - .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) - .collect(); - let mut sequences: Vec> = vec![Vec::new(); genome_indices.len()]; - - sample_index( - cache, - n, - free_loss, - no_ambiguity, - excluded, - entropy_bias, - 0, - |_partition, _layer, _raw_index, survivors| { - reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences); - }, - )?; - - Ok(SnpAlignment { sequences, genome_indices }) } -/// Appends one layer's worth of columns onto `sequences` — a "reduce" over -/// [`sample_index`]'s per-layer batch, self-contained (no sampling/masking -/// logic of its own, just character encoding) so it can be called -/// independently of, and alongside, any other reduce over the same batch — -/// `pub(super)`: `algorithms::sankoff::sankoff_bundle` runs this one -/// alongside `pairwise::reduce_pairwise` over the same `on_layer` call. -pub(super) fn reduce_alignment( +impl AlignmentReducer { + pub fn new(cache: &IndexCache, excluded: &[bool], free_loss: bool, no_ambiguity: bool) -> Self { + let n_genomes = cache.meta().genomes().len(); + let genome_indices: Vec = (0..n_genomes) + .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) + .collect(); + let sequences = vec![Vec::new(); genome_indices.len()]; + Self { genome_indices, sequences, free_loss, no_ambiguity } + } + + pub fn finish(self) -> SnpAlignment { + SnpAlignment { sequences: self.sequences, genome_indices: self.genome_indices } + } +} + +impl LayerReducer for AlignmentReducer { + fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { + reduce_alignment(survivors, &self.genome_indices, self.free_loss, self.no_ambiguity, &mut self.sequences); + } +} + +/// Appends one layer's worth of columns onto `sequences` — self-contained +/// (no sampling/masking logic of its own, just character encoding), the +/// actual reduce [`AlignmentReducer`] wraps. +fn reduce_alignment( survivors: &[SurvivingFamily], genome_indices: &[usize], free_loss: bool, diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 3d832fd8..7eff4aff 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -22,29 +22,27 @@ mod masking; mod minorant_selection; mod pairwise; mod sankoff; -mod session_cache; +mod scan; mod snp_distance; mod stats; mod subsample; use obikidxcache::index_cache::IndexCache; -pub use alignment::SnpAlignment; +pub use alignment::{AlignmentReducer, SnpAlignment}; pub use cardcomp::{ cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix, }; -pub use pairwise::{BasePairTally, CardinalityTally, RawSnpDistanceOutput}; +pub use pairwise::{BasePairTally, CardinalityTally, PairwiseReducer, RawSnpDistanceOutput}; pub use sankoff::SankoffBundle; -pub use snp_distance::{GammaShape, SnpDistanceKind}; +pub use scan::{LayerReducer, sampled_scan}; +pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix}; pub use stats::SiblingAnnexStats; pub use subsample::{EntropyBias, SurvivingFamily}; -pub(crate) use alignment::snp_pseudo_alignment; pub(crate) use annex::build_layer_sibling_annex; pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4}; pub(crate) use family_scan::{Selection, scan_layer_families}; -pub(crate) use sankoff::sankoff_bundle; -pub(crate) use snp_distance::snp_distance; pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram}; /// Whether every layer number in `cache` fits in a `FamilyMask` field diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index a651428c..e5934b3e 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -44,16 +44,15 @@ struct PairStats { cardinality: [[u64; 5]; 5], } -/// See the module docs. `pub(crate)`: consumed by -/// `algorithms::sankoff`'s orchestration, not yet exposed past this crate. +/// See the module docs. `pub`: returned by [`PairwiseReducer::finish`]. /// -/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): lets `--session` dump/ -/// restore a tally across separate CLI invocations as opaque bytes handed -/// to `obiksession::Session::store`/`restore` — `obiksession` itself knows -/// nothing about this type, it only stores/checksums/mmaps the bytes this -/// crate produces (see `obiksession`'s own crate docs on that split). +/// `Archive`/`Serialize`/`Deserialize` (`rkyv`): kept from when this type +/// was dumped to `--session` as opaque bytes via `obiksession::Session`; +/// no active user right now (`--session` support was removed pending a +/// rebuild on top of `algorithms::scan::sampled_scan`), but cheap to keep +/// derived for when it returns. #[derive(Archive, Serialize, Deserialize)] -pub(crate) struct PairwiseTally { +pub struct PairwiseTally { n_genomes: usize, /// Upper triangle only (`i < j`), flat-indexed via [`Self::flat_index`]. pairs: Vec, @@ -314,7 +313,7 @@ pub struct CardinalityTally { /// (`PairStats` accumulates over every genome pair unconditionally; /// exclusion is a filter applied only in derived, post-hoc views). #[derive(Default, Clone, Archive, Serialize, Deserialize)] -pub(crate) struct PartitionDispersion { +pub struct PartitionDispersion { /// One entry per partition id seen so far (grows lazily — partitions /// are visited in increasing order in practice, but nothing here /// assumes that). @@ -462,6 +461,34 @@ pub(crate) fn reduce_pairwise( dispersion.record(partition, partition_subst, partition_eligible); } +/// Accumulates a [`PairwiseTally`]/[`PartitionDispersion`] pair over a +/// [`super::scan::sampled_scan`] pass — a thin +/// [`super::scan::LayerReducer`] wrapper around [`reduce_pairwise`], so it +/// can run alongside any other reducer (e.g. +/// `algorithms::alignment::AlignmentReducer`) over the exact same batches +/// in one shared scan, instead of each needing its own `sample_index` +/// call. +pub struct PairwiseReducer { + tally: PairwiseTally, + dispersion: PartitionDispersion, +} + +impl PairwiseReducer { + pub fn new(n_genomes: usize) -> Self { + Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() } + } + + pub fn finish(self) -> (PairwiseTally, PartitionDispersion) { + (self.tally, self.dispersion) + } +} + +impl super::scan::LayerReducer for PairwiseReducer { + fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { + reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion); + } +} + #[cfg(test)] mod dispersion_tests { use super::PartitionDispersion; diff --git a/src/obikphylo/src/siblings/algorithms/sankoff.rs b/src/obikphylo/src/siblings/algorithms/sankoff.rs index 1e3bdb22..c7117724 100644 --- a/src/obikphylo/src/siblings/algorithms/sankoff.rs +++ b/src/obikphylo/src/siblings/algorithms/sankoff.rs @@ -1,62 +1,36 @@ -//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` -//! pipeline: one [`sample_index`] call (one Bernoulli draw, shared by -//! everyone — the same family selection feeds the alignment and every -//! tally, never two independent samples of the same index) driving two -//! reduces over the same `on_layer` batches — [`reduce_alignment`] and -//! [`reduce_pairwise`] — so [`SnpAlignment`], [`RawSnpDistanceOutput`], -//! [`BasePairTally`] and [`CardinalityTally`] all come out of a single -//! scan. The latter three are cheap `O(n²)` post-processing over -//! [`PairwiseTally`] once the scan is done (see its own module docs for why -//! that removes the old two-pass `raw_snp_distance`-before-`base_pair_tally` -//! dependency entirely). +//! [`SankoffBundle`]: the pure post-processing glue for the +//! `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline — turns an already-run +//! [`super::scan::sampled_scan`] pass's [`AlignmentReducer`]/ +//! [`PairwiseReducer`] output into every value that pipeline needs +//! ([`SnpAlignment`], [`RawSnpDistanceOutput`], [`BasePairTally`], +//! [`CardinalityTally`]). No sampling of its own: the caller (`cmd/phylo`) +//! runs the shared scan itself, with whichever reducers a given CLI +//! invocation actually needs (alignment only, pairwise only, or both when +//! `--sankoff` and a `snp-*` `--distance` are requested together) — see +//! `algorithms::scan`'s own module docs for why that single shared call, +//! not a per-output driver, is what guarantees every output in one +//! invocation comes from the same site selection. use ndarray::Array2; -use obikidxcache::index_cache::IndexCache; -use obikindex::{OKIError, OKIResult}; -use obiksession::Session; +use obikindex::OKIResult; -use super::alignment::{SnpAlignment, reduce_alignment}; +use super::alignment::SnpAlignment; use super::pairwise::{ - BasePairTally, CardinalityTally, PairwiseTally, PartitionDispersion, RawSnpDistanceOutput, - reduce_pairwise, -}; -use super::session_cache::{ - CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint, - store_tally_checkpoint, + BasePairTally, CardinalityTally, PairwiseReducer, PairwiseTally, PartitionDispersion, + RawSnpDistanceOutput, }; use super::snp_distance::{GammaShape, SnpDistanceKind, distance_matrix}; -use super::subsample::{EntropyBias, sample_index}; -/// The alignment's own `--session` artifact — kept in lockstep with the -/// shared `pairwise_tally`/`partition_dispersion`/`progress` artifacts -/// (`session_cache`): both are folded from the *same* `on_layer` call, so -/// checkpointing them together, under the same `progress` marker, is what -/// keeps a resumed alignment's column count consistent with how many -/// layers the resumed tally actually reflects. -const ALIGNMENT_ARTIFACT: &str = "snp_alignment"; - -fn restore_alignment(session: &Session) -> Option { - let bytes = session.restore(ALIGNMENT_ARTIFACT).ok().flatten()?; - rkyv::from_bytes::(&bytes[..]).ok() -} - -fn store_alignment(session: &Session, alignment: &SnpAlignment) -> OKIResult<()> { - let bytes = rkyv::to_bytes::(alignment) - .map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?; - session.store(ALIGNMENT_ARTIFACT, &bytes).map_err(OKIError::Io) -} - -/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs, -/// computed together from one shared, possibly-subsampled/entropy-biased -/// selection — see the module docs. `pub`: part of the public signature of -/// [`crate::siblings::extensions::SiblingExt::sankoff_bundle`]. +/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline +/// needs, built together from one already-scanned +/// [`AlignmentReducer`]/[`PairwiseReducer`] pair. `pub`: part of the +/// public signature of [`crate::siblings::extensions::SiblingExt`]'s +/// sampling-based callers. /// /// `tally`/`dispersion` are kept (not just consumed into `raw`/ /// `base_pair_tally`/`cardinality_tally`) so [`Self::snp_distance`] can -/// compute a `snp-*` `--distance` matrix from this *exact* sample — see -/// that method's own docs for why reusing it, rather than a fresh -/// `SiblingExt::snp_distance` call, is a correctness requirement here, not -/// just a performance nicety. +/// compute a `snp-*` `--distance` matrix from this *exact* sample without +/// a second scan. pub struct SankoffBundle { pub alignment: SnpAlignment, pub raw: RawSnpDistanceOutput, @@ -67,124 +41,44 @@ pub struct SankoffBundle { } impl SankoffBundle { + /// Assemble a bundle from an already-finished [`SnpAlignment`] (see + /// `AlignmentReducer::finish`) plus a [`PairwiseReducer`] that both ran + /// through the same [`super::scan::sampled_scan`] call — pure + /// post-processing, `O(n²)` in genome count, never a second scan. + /// `alignment` is taken already-finished, not as an `AlignmentReducer`, + /// because a caller that *also* needs the alignment on its own (e.g. + /// `--pseudo-alignment` combined with `--sankoff` in one invocation) + /// has to read it before/alongside building this bundle — `SnpAlignment` + /// derives `Clone` for exactly that case. `ratio_ceiling` + /// (`--sankoff-ratio-ceiling`) gates [`Self::base_pair_tally`] only + /// (see [`PairwiseTally::cardinality_tally`]'s own docs for why + /// [`CardinalityTally`] uses a different — `excluded`-only — inclusion + /// rule). + pub fn finish( + alignment: SnpAlignment, + pairwise: PairwiseReducer, + ratio_ceiling: f64, + excluded: &[bool], + ) -> Self { + let (tally, dispersion) = pairwise.finish(); + let raw = tally.raw_snp_distance(); + let included = tally.included(ratio_ceiling, excluded); + let base_pair_tally = tally.base_pair_tally(&included); + let cardinality_tally = tally.cardinality_tally(excluded); + Self { alignment, raw, base_pair_tally, cardinality_tally, tally, dispersion } + } + /// A `snp-*` `--distance` matrix computed from this bundle's own - /// already-sampled tally — no new [`sample_index`] call. - /// - /// Exists specifically for the case where `--sankoff`/`--tnt`/`--phyg`/ - /// `--iqtree` and a `snp-*` `--distance` are requested in the same CLI - /// invocation: both consume exactly the same selection parameters - /// (`n`/`free_loss`/`no_ambiguity`/excluded set/`entropy_bias` — the - /// CLI has no way to give them different values in one run), so a - /// second independent `SiblingExt::snp_distance` call wouldn't just - /// waste the sampling work, it would silently sample a **different** - /// set of sites (`sample_layer`'s `rand::rng()` is a thread-local - /// generator that advances across calls, not reseeded each time) — - /// defeating the actual point of running several algorithms together, - /// which is comparing them on one identical site selection. + /// already-sampled tally — no new scan. Exists specifically for the + /// case where `--sankoff`/`--tnt`/`--phyg`/`--iqtree` and a `snp-*` + /// `--distance` are requested in the same CLI invocation: both consume + /// exactly the same selection parameters, so recomputing from a second + /// [`super::scan::sampled_scan`] call would silently draw a + /// **different** sample (`rand::rng()` is a thread-local generator + /// that advances across calls, not reseeded each time) — defeating the + /// actual point of running several algorithms together, which is + /// comparing them on one identical site selection. pub fn snp_distance(&self, kind: SnpDistanceKind, gamma_shape: GammaShape) -> OKIResult> { distance_matrix(&self.tally, &self.dispersion, kind, gamma_shape) } } - -/// See [`crate::siblings::extensions::SiblingExt::sankoff_bundle`] for the -/// public-facing docs — this is its implementation. `ratio_ceiling` gates -/// [`BasePairTally`] only (see [`PairwiseTally::cardinality_tally`]'s own -/// docs for why [`CardinalityTally`] uses a different — `excluded`-only — -/// inclusion rule). -/// -/// `session`: `--session`, `None` means no persistence (always resample). -/// `Some(session)` must already be open under the exact selection -/// parameters this call is about to use — the caller's job, same contract -/// as [`super::snp_distance::snp_distance`]'s own `session` parameter, -/// which this shares an artifact namespace with (see `session_cache`'s -/// module docs): a tally checkpointed here is restorable by a later -/// `--distance snp-*` call against the same session, and vice versa. -pub(crate) fn sankoff_bundle( - cache: &IndexCache, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ratio_ceiling: f64, - session: Option<&Session>, -) -> OKIResult { - let n_genomes = cache.meta().genomes().len(); - let genome_indices: Vec = (0..n_genomes) - .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) - .collect(); - - let restored = session.and_then(|s| Some((restore_tally_checkpoint(s)?, restore_alignment(s)?))); - let (mut tally, mut dispersion, mut sequences, skip_layers) = match restored { - Some(((tally, dispersion, skip_layers), alignment)) => { - (tally, dispersion, alignment.sequences, skip_layers) - } - None => { - let sequences = vec![Vec::new(); genome_indices.len()]; - (PairwiseTally::new(n_genomes), PartitionDispersion::default(), sequences, 0) - } - }; - - if skip_layers != usize::MAX { - // Same checkpoint cadence and rationale as `snp_distance`'s own - // resumable loop (see `session_cache::CHECKPOINT_INTERVAL*`'s own - // docs) — the alignment's `sequences` are folded in lockstep with - // the tally so a resumed run's column count always matches how - // many layers the restored tally actually reflects. - let mut last_checkpoint_layer = skip_layers; - let mut last_checkpoint_at = std::time::Instant::now(); - sample_index( - cache, - n, - free_loss, - no_ambiguity, - excluded, - entropy_bias, - skip_layers, - |partition, _layer, raw_index, survivors| { - reduce_alignment(&survivors, &genome_indices, free_loss, no_ambiguity, &mut sequences); - reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion); - let Some(session) = session else { return }; - let layers_done = raw_index + 1; - let due_by_count = - layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS; - let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL; - if !(due_by_count || due_by_time) { - return; - } - let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() }; - let stored = store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64) - .and_then(|()| store_alignment(session, &alignment)); - match stored { - Ok(()) => { - last_checkpoint_layer = layers_done; - last_checkpoint_at = std::time::Instant::now(); - } - Err(e) => { - tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}"); - } - } - }, - )?; - - if let Some(session) = session { - let alignment = SnpAlignment { sequences: sequences.clone(), genome_indices: genome_indices.clone() }; - store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?; - store_alignment(session, &alignment)?; - } - } - - let raw = tally.raw_snp_distance(); - let included = tally.included(ratio_ceiling, excluded); - let base_pair_tally = tally.base_pair_tally(&included); - let cardinality_tally = tally.cardinality_tally(excluded); - - Ok(SankoffBundle { - alignment: SnpAlignment { sequences, genome_indices }, - raw, - base_pair_tally, - cardinality_tally, - tally, - dispersion, - }) -} diff --git a/src/obikphylo/src/siblings/algorithms/scan.rs b/src/obikphylo/src/siblings/algorithms/scan.rs new file mode 100644 index 00000000..99112fcc --- /dev/null +++ b/src/obikphylo/src/siblings/algorithms/scan.rs @@ -0,0 +1,70 @@ +//! The single trunk every sampled sibling-family computation runs +//! through: one [`sample_index`] pass, fanned out to whichever +//! [`LayerReducer`]s the caller actually needs. Replaces three previously +//! independent drivers (a pseudo-alignment-only one, a tally-only one, and +//! a hand-fused alignment+tally one for `--sankoff`) that each ran their +//! own `sample_index` call — meaning two of them combined in the same CLI +//! invocation used to draw two different random samples for what was +//! supposed to be one shared site selection. A caller that needs several +//! outputs from the same sample now passes several reducers to one +//! [`sampled_scan`] call instead of calling several higher-level functions +//! and hoping they agree. + +use obikidxcache::index_cache::IndexCache; +use obikindex::OKIResult; + +use super::stats::sibling_family_size_histogram; +use super::subsample::{EntropyBias, SurvivingFamily, sample_index}; + +/// One accumulator fed every layer's [`SurvivingFamily`] batch during a +/// [`sampled_scan`] pass — self-contained (no sampling/masking logic of +/// its own), so any number of these can run side by side over the exact +/// same batches without knowing about each other. Implement this once per +/// kind of output the trunk should be able to produce (see +/// `algorithms::alignment::AlignmentReducer`, `algorithms::pairwise::PairwiseReducer`). +pub trait LayerReducer { + fn reduce(&mut self, partition: usize, layer: usize, survivors: &[SurvivingFamily]); +} + +/// `n`: `Some(target)` samples proportionally per layer, up to `target` +/// sites index-wide (`--subsample`); `None` means exhaustive — every +/// non-monomorphic minorant of the whole index, driven through the exact +/// same [`sample_index`] machinery at a quota equal to the total eligible +/// count (so every family is drawn with certainty) rather than a separate +/// full-scan code path. +fn resolve_target(cache: &IndexCache, n: Option) -> OKIResult { + match n { + Some(target) => Ok(target), + None => { + let counts = sibling_family_size_histogram(cache)?; + Ok((counts[1] + counts[2] + counts[3]) as usize) + } + } +} + +/// Run one shared, possibly-subsampled/entropy-biased selection over +/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see +/// [`sample_index`]'s own docs for what each controls), feeding every +/// surviving family of every visited layer to every reducer in +/// `reducers`, in the same order, before moving to the next layer. A +/// no-op (not an error) if the resolved target is `0`. +pub fn sampled_scan( + cache: &IndexCache, + n: Option, + free_loss: bool, + no_ambiguity: bool, + excluded: &[bool], + entropy_bias: Option, + reducers: &mut [&mut dyn LayerReducer], +) -> OKIResult<()> { + let target = resolve_target(cache, n)?; + if target == 0 { + return Ok(()); + } + sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| { + for reducer in reducers.iter_mut() { + reducer.reduce(partition, layer, &survivors); + } + })?; + Ok(()) +} diff --git a/src/obikphylo/src/siblings/algorithms/session_cache.rs b/src/obikphylo/src/siblings/algorithms/session_cache.rs deleted file mode 100644 index a4ae6709..00000000 --- a/src/obikphylo/src/siblings/algorithms/session_cache.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Shared `--session` checkpoint plumbing for [`super::snp_distance`] and -//! [`super::sankoff`] — both fold [`super::subsample::sample_index`]'s -//! layer-by-layer output into a [`PairwiseTally`]/[`PartitionDispersion`] -//! pair, and both persist it under the *same* artifact names on purpose: -//! a tally checkpointed by one is restorable by the other. Concretely, -//! `--sankoff --session DIR` today followed by `--distance snp-k2p -//! --session DIR` tomorrow restores the first run's tally instead of -//! resampling, even though only `--sankoff` originally built it — -//! `snp_distance`'s own selection-parameter conflict check (via -//! `obiksession::Session::open`, the caller's responsibility) is what -//! guarantees that's actually safe to do. - -use obikindex::{OKIError, OKIResult}; -use obiksession::Session; - -use super::pairwise::{PairwiseTally, PartitionDispersion}; - -pub(crate) const TALLY_ARTIFACT: &str = "pairwise_tally"; -pub(crate) const DISPERSION_ARTIFACT: &str = "partition_dispersion"; -pub(crate) const PROGRESS_ARTIFACT: &str = "progress"; -/// Sentinel `progress` value meaning "every layer was processed — this -/// tally is complete," distinct from any real layer count so a resumed run -/// can tell "finished" apart from "stopped after N layers" without relying -/// on N happening to equal the index's actual layer count. -pub(crate) const PROGRESS_COMPLETE: u64 = u64::MAX; - -/// How much in-progress sampling work a checkpoint can lose: at most -/// `CHECKPOINT_INTERVAL_LAYERS` layers *or* `CHECKPOINT_INTERVAL` of wall -/// time, whichever comes first — not "checkpoint every layer" (rewriting -/// the whole, potentially large, tally/dispersion after every single one of -/// what can be thousands of layers would make the checkpointing overhead -/// itself dominate), and not "only at the end" either (which would defeat -/// the point: on a long run against a real production index, losing 90% of -/// an already-completed scan to a crash is exactly the expensive case -/// resumability exists to avoid — the actual justification, not "the tally -/// is cheap to rebuild," which was an earlier, wrong framing of this -/// tradeoff caught on review). -pub(crate) const CHECKPOINT_INTERVAL_LAYERS: usize = 8; -pub(crate) const CHECKPOINT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30); - -/// Restore a checkpointed `(PairwiseTally, PartitionDispersion, resume-from -/// layer index)` triple from `session` — `None` on any miss (an artifact -/// not cached, an I/O error, or a deserialization failure), all treated -/// identically: something's missing or unusable, fall back to starting -/// from scratch (`skip_layers = 0`) rather than surfacing a hard error over -/// what's meant to be a transparent cache. The resume index is -/// [`PROGRESS_COMPLETE`] when a prior run finished every layer. Layers -/// processed after a resume draw a fresh, independent random sample (see -/// `subsample::sample_index`'s own `skip_layers` docs) — deliberately not -/// bit-identical to what an uninterrupted run would have produced, -/// consistent with wanting run-to-run sampling variance rather than fixed -/// reproducibility. -/// -/// Full owned deserialization (`rkyv::from_bytes`) for the tally/dispersion, -/// not zero-copy (`rkyv::access`) — simpler to integrate with -/// `PairwiseTally`'s existing query methods, which all take `&self` by -/// value semantics, not an `Archived`. Still skips the -/// expensive part (`sample_index` re-scanning the sibling annex for every -/// already-completed layer); the deserialize of an already-in-memory-sized -/// byte buffer is comparatively cheap. Making this genuinely zero-copy -/// later would mean every `PairwiseTally` accessor (`categories`, -/// `base_freq`, ...) working generically over `Archived` -/// too — a bigger, separate change. -pub(crate) fn restore_tally_checkpoint(session: &Session) -> Option<(PairwiseTally, PartitionDispersion, usize)> { - let tally_bytes = session.restore(TALLY_ARTIFACT).ok().flatten()?; - let dispersion_bytes = session.restore(DISPERSION_ARTIFACT).ok().flatten()?; - let progress_bytes = session.restore(PROGRESS_ARTIFACT).ok().flatten()?; - let tally = rkyv::from_bytes::(&tally_bytes[..]).ok()?; - let dispersion = - rkyv::from_bytes::(&dispersion_bytes[..]).ok()?; - let progress_raw: [u8; 8] = progress_bytes.get(..8)?.try_into().ok()?; - let progress = u64::from_le_bytes(progress_raw); - let skip_layers = if progress == PROGRESS_COMPLETE { usize::MAX } else { progress as usize }; - if skip_layers == usize::MAX { - tracing::info!("--session: restored a complete cached SNP tally — skipping resampling"); - } else { - tracing::info!("--session: resuming a cached SNP tally from layer {skip_layers} onward"); - } - Some((tally, dispersion, skip_layers)) -} - -/// Store `tally`/`dispersion` plus how far sampling has gotten -/// (`layers_done`, or [`PROGRESS_COMPLETE`] once every layer has been -/// processed) — a checkpoint [`restore_tally_checkpoint`] can resume from -/// later, whether that "later" is after a crash or a clean, separate -/// invocation. -pub(crate) fn store_tally_checkpoint( - session: &Session, - tally: &PairwiseTally, - dispersion: &PartitionDispersion, - progress: u64, -) -> OKIResult<()> { - let tally_bytes = rkyv::to_bytes::(tally) - .map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?; - session.store(TALLY_ARTIFACT, &tally_bytes).map_err(OKIError::Io)?; - let dispersion_bytes = rkyv::to_bytes::(dispersion) - .map_err(|e| OKIError::Io(std::io::Error::other(e.to_string())))?; - session.store(DISPERSION_ARTIFACT, &dispersion_bytes).map_err(OKIError::Io)?; - session.store(PROGRESS_ARTIFACT, &progress.to_le_bytes()).map_err(OKIError::Io)?; - Ok(()) -} diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 1460fa32..8a6528e4 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -22,21 +22,13 @@ //! change. use ndarray::Array2; -use obikidxcache::index_cache::IndexCache; use obikindex::OKIResult; -use obiksession::Session; use super::pairwise::{PairwiseTally, PartitionDispersion}; -use super::session_cache::{ - CHECKPOINT_INTERVAL, CHECKPOINT_INTERVAL_LAYERS, PROGRESS_COMPLETE, restore_tally_checkpoint, - store_tally_checkpoint, -}; -use super::sibling_family_size_histogram; -use super::subsample::{EntropyBias, sample_index}; -/// One `snp-*` `--distance` value. `pub`: part of -/// [`crate::siblings::extensions::SiblingExt::snp_distance`]'s public -/// signature. +/// One `snp-*` `--distance` value. `pub`: part of the public signature of +/// [`crate::siblings::extensions::SiblingExt`]'s distance-computing +/// callers. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SnpDistanceKind { /// Uncorrected p-distance (`snp / (snp + shared)`). @@ -213,140 +205,18 @@ fn formula(kind: SnpDistanceKind) -> fn(&PairwiseTally, usize, usize, Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - gamma_shape: GammaShape, - session: Option<&Session>, -) -> OKIResult> { - // Fail fast, before paying for `sample_index`, not just inside - // `distance_matrix` (which runs after sampling either way — the right - // place for the check when called from `SankoffBundle::snp_distance`, - // where sampling already happened as part of `--sankoff` itself). - if gamma_shape != GammaShape::Disabled && !kind.supports_gamma() { - return Err(obikindex::OKIError::InvalidInput( - "--gamma-shape has no effect on this --distance value".into(), - )); - } - - let (mut tally, mut dispersion, skip_layers) = match session.and_then(restore_tally_checkpoint) { - Some(triple) => triple, - None => { - let n_genomes = cache.meta().genomes().len(); - (PairwiseTally::new(n_genomes), PartitionDispersion::default(), 0) - } - }; - - if skip_layers != usize::MAX { - let (target, entropy_bias) = match n { - Some(target) => (target, entropy_bias), - None => { - let counts = sibling_family_size_histogram(cache)?; - let total_eligible = (counts[1] + counts[2] + counts[3]) as usize; - (total_eligible, None) - } - }; - - if target > 0 { - // Checkpoint every `CHECKPOINT_INTERVAL_LAYERS` layers or - // `CHECKPOINT_INTERVAL` of wall time, whichever comes first — - // see those constants' own docs for why "just recompute - // everything" isn't the right call to make on a long run - // against a real production index. - let mut last_checkpoint_layer = skip_layers; - let mut last_checkpoint_at = std::time::Instant::now(); - sample_index( - cache, - target, - free_loss, - no_ambiguity, - excluded, - entropy_bias, - skip_layers, - |partition, _layer, raw_index, survivors| { - super::pairwise::reduce_pairwise(&survivors, partition, &mut tally, &mut dispersion); - let Some(session) = session else { return }; - let layers_done = raw_index + 1; - let due_by_count = - layers_done.saturating_sub(last_checkpoint_layer) >= CHECKPOINT_INTERVAL_LAYERS; - let due_by_time = last_checkpoint_at.elapsed() >= CHECKPOINT_INTERVAL; - if !(due_by_count || due_by_time) { - return; - } - match store_tally_checkpoint(session, &tally, &dispersion, layers_done as u64) { - Ok(()) => { - last_checkpoint_layer = layers_done; - last_checkpoint_at = std::time::Instant::now(); - } - Err(e) => { - tracing::warn!("--session: failed to checkpoint at layer {layers_done}: {e}"); - } - } - }, - )?; - } - - if let Some(session) = session { - store_tally_checkpoint(session, &tally, &dispersion, PROGRESS_COMPLETE)?; - } - } - - distance_matrix(&tally, &dispersion, kind, gamma_shape) -} - -/// The post-sampling half of [`snp_distance`] — everything downstream of -/// an already-built [`PairwiseTally`]/[`PartitionDispersion`]: the -/// `--gamma-shape` support check, `alpha` resolution (fixed/auto/disabled), -/// and the final `n×n` matrix build. Factored out so -/// [`super::sankoff::SankoffBundle::snp_distance`] can reuse a tally it -/// already has (from `--sankoff`/`--tnt`/`--phyg`/`--iqtree` sharing the -/// exact same selection parameters as a `snp-*` `--distance` in the same -/// CLI invocation) instead of re-running [`sample_index`] — which, beyond -/// the wasted work, would also draw a **different** random sample the -/// second time (`rand::rng()` is a thread-local generator that advances -/// across calls, never reseeded per call), silently breaking the -/// "same site selection for every algorithm in this run" guarantee that's -/// the actual point of combining these flags in one invocation. -pub(crate) fn distance_matrix( +/// Everything downstream of an already-built +/// [`PairwiseTally`]/[`PartitionDispersion`]: the `--gamma-shape` support +/// check, `alpha` resolution (fixed/auto/disabled), and the final `n×n` +/// matrix build — `O(n²)` post-processing, never a scan of its own. The +/// tally/dispersion themselves come from a +/// [`super::pairwise::PairwiseReducer`] that already ran through +/// [`super::scan::sampled_scan`] — see [`super::sankoff::SankoffBundle::snp_distance`] +/// for why reusing an already-scanned reducer, rather than a second +/// [`sampled_scan`](super::scan::sampled_scan) call, is a correctness +/// requirement (not just a performance nicety) whenever `--sankoff` and a +/// `snp-*` `--distance` are requested together. +pub fn distance_matrix( tally: &PairwiseTally, dispersion: &PartitionDispersion, kind: SnpDistanceKind, diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index e56b6c99..c80bf521 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -9,16 +9,13 @@ use std::io::{BufWriter, Write}; use std::path::Path; -use ndarray::Array2; use obikidxcache::index_cache::IndexCache; use obikindex::{OKIError, OKIResult}; use obisys::progress_bar; use crate::siblings::algorithms::{ - EntropyBias, GammaShape, SankoffBundle, Selection, SiblingAnnexStats, SnpAlignment, - SnpDistanceKind, build_layer_sibling_annex, family_entropy, family_entropy_4, is_fast_mode, - sankoff_bundle, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram, - snp_distance, snp_pseudo_alignment, + Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4, + is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram, }; use crate::siblings::extensions::SiblingBuilder; use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap}; @@ -58,28 +55,6 @@ pub trait SiblingExt { /// `--subsample`/`--entropy-bias` yet (see the project memory on this). fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>; - /// Build a SNP-only pseudo-alignment (`algorithms::subsample::sample_index` - /// — proportional per-layer sampling, `n` sites index-wide, target not - /// guarantee) from an already-built sibling annex (run - /// [`build_sibling_annex`](Self::build_sibling_annex) first). `free_loss` - /// (`--free-loss`), `no_ambiguity` (`--no-ambiguity`) and `excluded` - /// (`--exclude-genome`, `excluded[g]` true to drop genome `g`) all feed - /// straight into the sampling itself, not a later filter — see - /// `algorithms::masking`'s docs: a family whose polymorphism only - /// existed via states/genomes these exclude is caught and discarded - /// during sampling, not left in the output as a false site. Every - /// column of the result has already survived that check, and an - /// excluded genome has no row at all in the result (see - /// `SnpAlignment::genome_indices`). - fn snp_pseudo_alignment( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ) -> OKIResult; - /// The global family-size histogram alone (`SiblingAnnexStats::counts`, /// no `per_genome`) — reads only the already-built annex /// (`FamilyMask::siblings()` + `is_minorant()`, already-stored bits, @@ -97,84 +72,6 @@ pub trait SiblingExt { /// instead when only the global histogram is needed. fn sibling_annex_stats(&self) -> OKIResult; - /// Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` - /// pipeline: one shared, possibly-subsampled/entropy-biased selection - /// (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same - /// meaning as [`snp_pseudo_alignment`](Self::snp_pseudo_alignment)'s - /// own) drives a single scan producing the pseudo-alignment *and* every - /// tally the Sankoff cost-matrix calibration - /// (`crate::siblings::cardinality_transition_probs`/ - /// `composition_transition_probs`/`pairwise_cost_matrix`) needs — - /// never two independent draws of the same index. `ratio_ceiling` - /// (`--sankoff-ratio-ceiling`) excludes genome pairs too close to - /// substitution saturation from `SankoffBundle::base_pair_tally`'s - /// pool (a saturated pair's base composition is noise, not signal) — - /// `SankoffBundle::cardinality_tally` is *not* gated by it (see - /// `crate::siblings::CardinalityTally`'s own docs for why that - /// wouldn't make sense: cardinality reflects each genome's own - /// coverage/duplication structure, not the pair's mutual divergence). - /// `session`: `--session`, `None` disables persistence (always - /// resample). `Some` must already be open under these exact selection - /// parameters (the caller's responsibility) — restores a cached tally/ - /// alignment when present instead of resampling, and checkpoints a - /// fresh one back as sampling progresses otherwise. Shares its - /// artifact namespace with [`Self::snp_distance`]'s own `session` - /// parameter internally: a tally checkpointed by one is restorable by - /// the other, given the same session and selection parameters. - fn sankoff_bundle( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ratio_ceiling: f64, - session: Option<&obiksession::Session>, - ) -> OKIResult; - - /// A `snp-*` `--distance` matrix — one of the closed-form corrections - /// in [`SnpDistanceKind`], over the central-position SNP substitution - /// spectrum (requires an already-built sibling annex, run - /// [`build_sibling_annex`](Self::build_sibling_annex) first). - /// - /// `n`: `Some(target)` samples proportionally per layer exactly like - /// [`sankoff_bundle`](Self::sankoff_bundle)/ - /// [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) (`--subsample`); - /// `None` means exhaustive — every non-monomorphic minorant of the - /// whole index, not an approximation (see - /// `algorithms::snp_distance::snp_distance`'s own docs for how this - /// reuses the same sampling machinery at `p = 1` rather than a second - /// driver). `free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — same - /// meaning as `snp_pseudo_alignment`'s own (the last forced to `None` - /// when `n` is `None`: entropy-biasing only makes sense for a genuine - /// subsample). - /// - /// `gamma_shape`: `--gamma-shape`, the Jin-Nei rate-heterogeneity - /// correction — [`GammaShape::Disabled`] turns it off, - /// [`GammaShape::Fixed`]/[`GammaShape::Auto`] are rejected with - /// [`OKIError::InvalidInput`] for a `kind` that doesn't support it - /// ([`SnpDistanceKind::supports_gamma`]). `Auto` estimates `alpha` from - /// the data itself (method-of-moments over per-partition substitution - /// rate dispersion) rather than requiring a user-supplied value. - /// - /// `session`: `--session`, `None` disables persistence (always - /// resample). `Some` must already be open under these exact selection - /// parameters (the caller's responsibility, via - /// `obiksession::Session::open`'s params-conflict check) — restores a - /// cached tally when present instead of resampling, and stores a - /// freshly computed one back for next time otherwise. - fn snp_distance( - &self, - kind: SnpDistanceKind, - n: Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - gamma_shape: GammaShape, - session: Option<&obiksession::Session>, - ) -> OKIResult>; - /// The Family Overlap annex — number of shared *variable* families per /// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs). /// Built (alongside the entropy annex) if missing, same as every other @@ -183,8 +80,8 @@ pub trait SiblingExt { /// (`FamilyOverlap::mean_row(g) < threshold` — same statistic /// `--family-overlap`'s matrix shows, just averaged per row — /// triggers adding `g` to the excluded-genome set consumed by - /// `snp_pseudo_alignment`/`sankoff_bundle`/`snp_distance`; this needs - /// no computation of its own beyond that check). + /// `algorithms::scan::sampled_scan`'s callers; this needs no + /// computation of its own beyond that check). fn family_overlap(&self) -> OKIResult; } @@ -312,17 +209,6 @@ impl SiblingExt for IndexCache { Ok(()) } - fn snp_pseudo_alignment( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ) -> OKIResult { - snp_pseudo_alignment(self, n, free_loss, no_ambiguity, excluded, entropy_bias) - } - fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> { sibling_family_size_histogram(self) } @@ -331,33 +217,6 @@ impl SiblingExt for IndexCache { sibling_annex_stats(self) } - fn sankoff_bundle( - &self, - n: usize, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - ratio_ceiling: f64, - session: Option<&obiksession::Session>, - ) -> OKIResult { - sankoff_bundle(self, n, free_loss, no_ambiguity, excluded, entropy_bias, ratio_ceiling, session) - } - - fn snp_distance( - &self, - kind: SnpDistanceKind, - n: Option, - free_loss: bool, - no_ambiguity: bool, - excluded: &[bool], - entropy_bias: Option, - gamma_shape: GammaShape, - session: Option<&obiksession::Session>, - ) -> OKIResult> { - snp_distance(self, kind, n, free_loss, no_ambiguity, excluded, entropy_bias, gamma_shape, session) - } - fn family_overlap(&self) -> OKIResult { self.ensure_entropy_and_overlap_annexes()?; let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME); diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index 104a4636..c15d0901 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -37,9 +37,10 @@ pub use iter::{ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use algorithms::{ - BasePairTally, CardinalityTally, EntropyBias, GammaShape, RawSnpDistanceOutput, SankoffBundle, - SiblingAnnexStats, SnpAlignment, SnpDistanceKind, cardinality_transition_probs, - composition_transition_probs, pairwise_cost_matrix, + AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer, + PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment, + SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix, + pairwise_cost_matrix, sampled_scan, }; pub use extensions::SiblingExt; -- 2.54.1 From cd06fa2ce378215c3036fcb26ab0a066534108d4 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 18 Sep 2026 11:55:46 +0200 Subject: [PATCH 05/10] Implement masking logic for pairwise reduction Introduces masking based on `free_loss` and `no_ambiguity` flags within pairwise algorithms. Updates reducer initialization and adds tests to verify cardinality calculation. --- src/obikmer/src/cmd/phylo/mod.rs | 3 +- .../src/siblings/algorithms/pairwise.rs | 99 +++++++++++++++++-- .../src/siblings/algorithms/snp_distance.rs | 2 +- 3 files changed, 93 insertions(+), 11 deletions(-) diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index e77ffa98..94b179e0 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -278,7 +278,8 @@ pub fn run(args: PhyloArgs) { let mut alignment_reducer = need_alignment.then(|| { obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity) }); - let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n)); + let mut pairwise_reducer = + need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n, args.free_loss, args.no_ambiguity)); if need_alignment || need_pairwise { let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new(); diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index e5934b3e..db738bd5 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -407,13 +407,29 @@ impl PartitionDispersion { /// own docs for why the partition axis must be captured here rather than /// after the fact) — the "reduce" half of the map/reduce /// [`super::subsample::sample_index`] drives, self-contained (no -/// sampling/masking logic of its own) so it can run alongside any other -/// reduce over the same batch (see `algorithms::alignment::reduce_alignment`, -/// called from the same `on_layer` closure in -/// `algorithms::sankoff::sankoff_bundle`). +/// sampling logic of its own) so it can run alongside any other reduce +/// over the same batch (see `algorithms::alignment::reduce_alignment`, +/// run alongside this one whenever both an alignment and a tally are +/// wanted from the same [`super::scan::sampled_scan`] call). +/// +/// `free_loss`/`no_ambiguity`: the same masking +/// [`algorithms::alignment::reduce_alignment`] applies per genome via +/// `masking::masked_state`, applied here too so the tally agrees with +/// what the alignment actually shows for the same family/genome — used +/// directly for `cardinality` (a masked-away genome contributes `0`, +/// matching its `?` in the alignment, not its raw, pre-masking bit +/// count). `subst`/`same_all`/`same_variable` don't need to look at +/// `masked_state` separately: `single_form`'s own "exactly one bit" +/// requirement already agrees with "masked, then exactly one bit" for +/// every possible mask, for any combination of these two flags (masking +/// only ever turns an already-`None` case — 0 bits or ≥2 bits — into a +/// differently-shaped `None`, never a `Some` into a `None` or vice +/// versa), so routing them through `masked_state` too would be a no-op. pub(crate) fn reduce_pairwise( survivors: &[SurvivingFamily], partition: usize, + free_loss: bool, + no_ambiguity: bool, tally: &mut PairwiseTally, dispersion: &mut PartitionDispersion, ) { @@ -427,10 +443,14 @@ pub(crate) fn reduce_pairwise( let m = genome_mask[g]; (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) }; + let masked_cardinality = |g: usize| -> usize { + super::masking::masked_state(genome_mask[g], free_loss, no_ambiguity) + .map_or(0, |m| m.count_ones() as usize) + }; for i in 0..n_genomes { let bi = single_form(i); - let card_i = genome_mask[i].count_ones() as usize; + let card_i = masked_cardinality(i); for j in (i + 1)..n_genomes { let stats = tally.pair_mut(i, j); @@ -449,7 +469,7 @@ pub(crate) fn reduce_pairwise( } if variable { - let card_j = genome_mask[j].count_ones() as usize; + let card_j = masked_cardinality(j); stats.cardinality[card_i][card_j] += 1; if card_i != card_j { stats.cardinality[card_j][card_i] += 1; @@ -471,11 +491,23 @@ pub(crate) fn reduce_pairwise( pub struct PairwiseReducer { tally: PairwiseTally, dispersion: PartitionDispersion, + free_loss: bool, + no_ambiguity: bool, } impl PairwiseReducer { - pub fn new(n_genomes: usize) -> Self { - Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() } + /// `free_loss`/`no_ambiguity`: the same masking flags passed to + /// `AlignmentReducer::new` — must match when both run over the same + /// [`super::scan::sampled_scan`] call, so the cardinality tally agrees + /// with what the alignment actually shows (see [`reduce_pairwise`]'s + /// own docs). + pub fn new(n_genomes: usize, free_loss: bool, no_ambiguity: bool) -> Self { + Self { + tally: PairwiseTally::new(n_genomes), + dispersion: PartitionDispersion::default(), + free_loss, + no_ambiguity, + } } pub fn finish(self) -> (PairwiseTally, PartitionDispersion) { @@ -485,7 +517,56 @@ impl PairwiseReducer { impl super::scan::LayerReducer for PairwiseReducer { fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { - reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion); + reduce_pairwise( + survivors, + partition, + self.free_loss, + self.no_ambiguity, + &mut self.tally, + &mut self.dispersion, + ); + } +} + +#[cfg(test)] +mod cardinality_masking_tests { + use super::{PairwiseTally, PartitionDispersion, reduce_pairwise}; + use crate::siblings::FamilyMask; + use crate::siblings::algorithms::SurvivingFamily; + + // Genome 0 is ambiguous (carries both A and C, mask 0b0011 = cardinality + // 2); genome 1 carries only A (mask 0b0001 = cardinality 1). The family + // is variable (both A and C are observed somewhere), so it contributes + // to `cardinality` regardless of masking. + fn ambiguous_vs_single() -> Vec { + vec![SurvivingFamily { + family_idx: 0, + mask: FamilyMask::EMPTY.with(0).with(1), // A, C observed + genome_mask: vec![0b0011, 0b0001], + }] + } + + #[test] + fn cardinality_uses_raw_mask_without_no_ambiguity() { + let mut tally = PairwiseTally::new(2); + let mut dispersion = PartitionDispersion::default(); + reduce_pairwise(&ambiguous_vs_single(), 0, false, false, &mut tally, &mut dispersion); + let counts = tally.cardinality_tally(&[]).counts; + assert_eq!(counts[2][1], 1, "genome 0's raw cardinality (2) should be tallied"); + assert_eq!(counts[0][1], 0, "no masking requested — nothing should collapse to 0"); + } + + #[test] + fn cardinality_masks_ambiguous_genome_to_zero_under_no_ambiguity() { + // Matches what `AlignmentReducer`/`masked_state` would show for + // genome 0 at this site under `--no-ambiguity`: `?`, i.e. no + // information — cardinality 0, not its raw 2. + let mut tally = PairwiseTally::new(2); + let mut dispersion = PartitionDispersion::default(); + reduce_pairwise(&ambiguous_vs_single(), 0, false, true, &mut tally, &mut dispersion); + let counts = tally.cardinality_tally(&[]).counts; + assert_eq!(counts[2][1], 0, "raw cardinality must not leak through under --no-ambiguity"); + assert_eq!(counts[0][1], 1, "the ambiguous genome should be masked to cardinality 0"); } } diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 8a6528e4..099ff3c8 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -298,7 +298,7 @@ mod tests { } let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&families, 0, &mut tally, &mut dispersion); + reduce_pairwise(&families, 0, false, false, &mut tally, &mut dispersion); tally } -- 2.54.1 From 996ebaf977c54355207f85aaf7e204b987a12479 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 18 Sep 2026 12:27:24 +0200 Subject: [PATCH 06/10] Refactor algorithms to use configuration flags Introduce boolean flags for `AlignmentReducer` and `PairwiseReducer` to control behavior. Update masking logic and introduce the `SurvivingFamily` structure for explicit data handling. --- src/obikmer/src/cmd/phylo/mod.rs | 8 +- .../src/siblings/algorithms/alignment.rs | 30 +++--- .../src/siblings/algorithms/masking.rs | 59 +++++++----- .../src/siblings/algorithms/pairwise.rs | 91 +++++++------------ .../src/siblings/algorithms/snp_distance.rs | 10 +- .../src/siblings/algorithms/subsample.rs | 40 ++++---- 6 files changed, 114 insertions(+), 124 deletions(-) diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 94b179e0..45c64119 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -275,11 +275,9 @@ pub fn run(args: PhyloArgs) { std::process::exit(1); } - let mut alignment_reducer = need_alignment.then(|| { - obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask, args.free_loss, args.no_ambiguity) - }); - let mut pairwise_reducer = - need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n, args.free_loss, args.no_ambiguity)); + let mut alignment_reducer = + need_alignment.then(|| obikphylo::siblings::AlignmentReducer::new(&cache, &snp_exclude_mask)); + let mut pairwise_reducer = need_pairwise.then(|| obikphylo::siblings::PairwiseReducer::new(n)); if need_alignment || need_pairwise { let mut reducers: Vec<&mut dyn obikphylo::siblings::LayerReducer> = Vec::new(); diff --git a/src/obikphylo/src/siblings/algorithms/alignment.rs b/src/obikphylo/src/siblings/algorithms/alignment.rs index 1371fc41..fb59a1a4 100644 --- a/src/obikphylo/src/siblings/algorithms/alignment.rs +++ b/src/obikphylo/src/siblings/algorithms/alignment.rs @@ -7,7 +7,7 @@ use obikidxcache::index_cache::IndexCache; use rkyv::{Archive, Deserialize, Serialize}; -use super::masking::{iupac_code, masked_state}; +use super::masking::iupac_code; use super::scan::LayerReducer; use super::subsample::SurvivingFamily; @@ -34,21 +34,22 @@ pub struct SnpAlignment { /// it can run alongside any other reducer (e.g. /// `algorithms::pairwise::PairwiseReducer`) over the exact same batches in /// one shared scan, instead of each needing its own `sample_index` call. +/// No `free_loss`/`no_ambiguity` of its own: `SurvivingFamily::masked` has +/// already applied them, once, upstream in `subsample::sample_layer` — see +/// that field's own docs. pub struct AlignmentReducer { genome_indices: Vec, sequences: Vec>, - free_loss: bool, - no_ambiguity: bool, } impl AlignmentReducer { - pub fn new(cache: &IndexCache, excluded: &[bool], free_loss: bool, no_ambiguity: bool) -> Self { + pub fn new(cache: &IndexCache, excluded: &[bool]) -> Self { let n_genomes = cache.meta().genomes().len(); let genome_indices: Vec = (0..n_genomes) .filter(|&g| !excluded.get(g).copied().unwrap_or(false)) .collect(); let sequences = vec![Vec::new(); genome_indices.len()]; - Self { genome_indices, sequences, free_loss, no_ambiguity } + Self { genome_indices, sequences } } pub fn finish(self) -> SnpAlignment { @@ -58,25 +59,18 @@ impl AlignmentReducer { impl LayerReducer for AlignmentReducer { fn reduce(&mut self, _partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { - reduce_alignment(survivors, &self.genome_indices, self.free_loss, self.no_ambiguity, &mut self.sequences); + reduce_alignment(survivors, &self.genome_indices, &mut self.sequences); } } /// Appends one layer's worth of columns onto `sequences` — self-contained -/// (no sampling/masking logic of its own, just character encoding), the -/// actual reduce [`AlignmentReducer`] wraps. -fn reduce_alignment( - survivors: &[SurvivingFamily], - genome_indices: &[usize], - free_loss: bool, - no_ambiguity: bool, - sequences: &mut [Vec], -) { +/// (no sampling/masking logic of its own, just character encoding over +/// `family.masked`, already computed upstream), the actual reduce +/// [`AlignmentReducer`] wraps. +fn reduce_alignment(survivors: &[SurvivingFamily], genome_indices: &[usize], sequences: &mut [Vec]) { for family in survivors { for (row, &g) in genome_indices.iter().enumerate() { - let ch = masked_state(family.genome_mask[g], free_loss, no_ambiguity) - .map(iupac_code) - .unwrap_or(b'?'); + let ch = family.masked[g].map(iupac_code).unwrap_or(b'?'); sequences[row].push(ch); } } diff --git a/src/obikphylo/src/siblings/algorithms/masking.rs b/src/obikphylo/src/siblings/algorithms/masking.rs index 9ae56b34..afc788d6 100644 --- a/src/obikphylo/src/siblings/algorithms/masking.rs +++ b/src/obikphylo/src/siblings/algorithms/masking.rs @@ -33,28 +33,31 @@ pub(crate) fn masked_state(m: u8, free_loss: bool, no_ambiguity: bool) -> Option Some(m) } -/// Whether a family's resolved `genome_mask` still carries real -/// polymorphism (`>=2` distinct masked states) once [`masked_state`] is -/// applied to every *included* genome — i.e. whether it remains a usable -/// alignment site rather than one that collapsed to monomorphic (or fully -/// missing) once uninformative states, and excluded genomes entirely, are -/// dropped from consideration. `excluded[g]` (`false` if `g` is beyond -/// `excluded`'s own length — a caller that never excludes anyone can pass -/// an empty slice) means genome `g` is skipped outright, not merely masked -/// to "?": it never contributes to `seen`, whether or not `free_loss`/ -/// `no_ambiguity` would also have masked it. -pub(crate) fn survives_masking( - genome_mask: &[u8], - excluded: &[bool], - free_loss: bool, - no_ambiguity: bool, -) -> bool { +/// Whether a family's already-[`masked_state`]d per-genome states still +/// carry real polymorphism (`>=2` distinct states) among *included* +/// genomes — i.e. whether it remains a usable alignment site rather than +/// one that collapsed to monomorphic (or fully missing) once uninformative +/// states, and excluded genomes entirely, are dropped from consideration. +/// Takes `masked` — one [`masked_state`] result per genome — rather than +/// the raw `genome_mask` and the masking flags themselves: the caller +/// (`subsample::sample_layer`) computes `masked_state` exactly once per +/// (family, genome) and reuses that same array for this check, for +/// building the eventual alignment column, and for folding the pairwise +/// tally, instead of every consumer recomputing it independently (which +/// used to let their masking semantics silently drift apart — see +/// `algorithms::pairwise::reduce_pairwise`'s own docs for the bug that +/// caused). `excluded[g]` (`false` if `g` is beyond `excluded`'s own +/// length — a caller that never excludes anyone can pass an empty slice) +/// means genome `g` is skipped outright, not merely masked to "?": it +/// never contributes to `seen`, whether or not `masked[g]` is already +/// `None`. +pub(crate) fn survives_masking(masked: &[Option], excluded: &[bool]) -> bool { let mut seen: u16 = 0; - for (g, &m) in genome_mask.iter().enumerate() { + for (g, &state) in masked.iter().enumerate() { if excluded.get(g).copied().unwrap_or(false) { continue; } - if let Some(state) = masked_state(m, free_loss, no_ambiguity) { + if let Some(state) = state { seen |= 1 << state; } } @@ -96,18 +99,25 @@ pub(crate) fn iupac_code(state: u8) -> u8 { mod tests { use super::*; + /// One [`masked_state`] call per raw mask — mirrors what + /// `subsample::sample_layer` does once, up front, before calling + /// [`survives_masking`]. + fn masked(raw: &[u8], free_loss: bool, no_ambiguity: bool) -> Vec> { + raw.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect() + } + #[test] fn plain_no_masking_counts_absence_as_its_own_state() { // Two genomes: one absent (0), one carrying base A (1) — 2 distinct // states without free-loss, so this survives. - assert!(survives_masking(&[0, 1], &[], false, false)); + assert!(survives_masking(&masked(&[0, 1], false, false), &[])); } #[test] fn free_loss_drops_absence_and_can_collapse_to_monomorphic() { // Same data as above, but with free-loss the absent genome becomes // "?" (excluded) — only one real state (A) remains. - assert!(!survives_masking(&[0, 1], &[], true, false)); + assert!(!survives_masking(&masked(&[0, 1], true, false), &[])); } #[test] @@ -116,8 +126,8 @@ mod tests { // carries only A. Without no-ambiguity: 2 distinct states (0b0011, // 0b0001) -> survives. With no-ambiguity: genome 0 becomes "?", // only one real state left -> collapses. - assert!(survives_masking(&[0b0011, 0b0001], &[], false, false)); - assert!(!survives_masking(&[0b0011, 0b0001], &[], false, true)); + assert!(survives_masking(&masked(&[0b0011, 0b0001], false, false), &[])); + assert!(!survives_masking(&masked(&[0b0011, 0b0001], false, true), &[])); } #[test] @@ -126,7 +136,8 @@ mod tests { // this survives (2 distinct states, matching the plain test above). // Excluding genome 1 (the only real signal) leaves just genome 0's // absence — a single state, collapses. - assert!(survives_masking(&[0, 1], &[false, false], false, false)); - assert!(!survives_masking(&[0, 1], &[false, true], false, false)); + let m = masked(&[0, 1], false, false); + assert!(survives_masking(&m, &[false, false])); + assert!(!survives_masking(&m, &[false, true])); } } diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index db738bd5..86f31921 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -407,29 +407,23 @@ impl PartitionDispersion { /// own docs for why the partition axis must be captured here rather than /// after the fact) — the "reduce" half of the map/reduce /// [`super::subsample::sample_index`] drives, self-contained (no -/// sampling logic of its own) so it can run alongside any other reduce -/// over the same batch (see `algorithms::alignment::reduce_alignment`, +/// sampling/masking logic of its own) so it can run alongside any other +/// reduce over the same batch (see `algorithms::alignment::reduce_alignment`, /// run alongside this one whenever both an alignment and a tally are /// wanted from the same [`super::scan::sampled_scan`] call). /// -/// `free_loss`/`no_ambiguity`: the same masking -/// [`algorithms::alignment::reduce_alignment`] applies per genome via -/// `masking::masked_state`, applied here too so the tally agrees with -/// what the alignment actually shows for the same family/genome — used -/// directly for `cardinality` (a masked-away genome contributes `0`, -/// matching its `?` in the alignment, not its raw, pre-masking bit -/// count). `subst`/`same_all`/`same_variable` don't need to look at -/// `masked_state` separately: `single_form`'s own "exactly one bit" -/// requirement already agrees with "masked, then exactly one bit" for -/// every possible mask, for any combination of these two flags (masking -/// only ever turns an already-`None` case — 0 bits or ≥2 bits — into a -/// differently-shaped `None`, never a `Some` into a `None` or vice -/// versa), so routing them through `masked_state` too would be a no-op. +/// Reads `family.masked` directly — [`super::subsample::SurvivingFamily`]'s +/// own docs explain why that's already the masked (`--free-loss`/ +/// `--no-ambiguity`-applied) per-genome state, computed once upstream, the +/// same array [`algorithms::alignment::reduce_alignment`] reads too: this +/// is what guarantees `cardinality` agrees with what the alignment +/// actually shows for the same family/genome (a masked-away genome +/// contributes `0`, matching its `?`). `subst`/`same_all`/`same_variable` +/// don't need anything beyond `single_form`'s "exactly one bit" check on +/// the already-masked state. pub(crate) fn reduce_pairwise( survivors: &[SurvivingFamily], partition: usize, - free_loss: bool, - no_ambiguity: bool, tally: &mut PairwiseTally, dispersion: &mut PartitionDispersion, ) { @@ -438,15 +432,10 @@ pub(crate) fn reduce_pairwise( let mut partition_eligible: u64 = 0; for family in survivors { let variable = family.mask.family_size() >= 2; - let genome_mask = &family.genome_mask; let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - let masked_cardinality = |g: usize| -> usize { - super::masking::masked_state(genome_mask[g], free_loss, no_ambiguity) - .map_or(0, |m| m.count_ones() as usize) + family.masked[g].filter(|m| m.count_ones() == 1).map(|m| m.trailing_zeros() as u8) }; + let masked_cardinality = |g: usize| -> usize { family.masked[g].map_or(0, |m| m.count_ones() as usize) }; for i in 0..n_genomes { let bi = single_form(i); @@ -491,23 +480,14 @@ pub(crate) fn reduce_pairwise( pub struct PairwiseReducer { tally: PairwiseTally, dispersion: PartitionDispersion, - free_loss: bool, - no_ambiguity: bool, } impl PairwiseReducer { - /// `free_loss`/`no_ambiguity`: the same masking flags passed to - /// `AlignmentReducer::new` — must match when both run over the same - /// [`super::scan::sampled_scan`] call, so the cardinality tally agrees - /// with what the alignment actually shows (see [`reduce_pairwise`]'s - /// own docs). - pub fn new(n_genomes: usize, free_loss: bool, no_ambiguity: bool) -> Self { - Self { - tally: PairwiseTally::new(n_genomes), - dispersion: PartitionDispersion::default(), - free_loss, - no_ambiguity, - } + /// No `free_loss`/`no_ambiguity` of its own: `SurvivingFamily::masked` + /// has already applied them, once, upstream in + /// `subsample::sample_layer` — see that field's own docs. + pub fn new(n_genomes: usize) -> Self { + Self { tally: PairwiseTally::new(n_genomes), dispersion: PartitionDispersion::default() } } pub fn finish(self) -> (PairwiseTally, PartitionDispersion) { @@ -517,14 +497,7 @@ impl PairwiseReducer { impl super::scan::LayerReducer for PairwiseReducer { fn reduce(&mut self, partition: usize, _layer: usize, survivors: &[SurvivingFamily]) { - reduce_pairwise( - survivors, - partition, - self.free_loss, - self.no_ambiguity, - &mut self.tally, - &mut self.dispersion, - ); + reduce_pairwise(survivors, partition, &mut self.tally, &mut self.dispersion); } } @@ -537,20 +510,20 @@ mod cardinality_masking_tests { // Genome 0 is ambiguous (carries both A and C, mask 0b0011 = cardinality // 2); genome 1 carries only A (mask 0b0001 = cardinality 1). The family // is variable (both A and C are observed somewhere), so it contributes - // to `cardinality` regardless of masking. - fn ambiguous_vs_single() -> Vec { - vec![SurvivingFamily { - family_idx: 0, - mask: FamilyMask::EMPTY.with(0).with(1), // A, C observed - genome_mask: vec![0b0011, 0b0001], - }] + // to `cardinality` regardless of masking. `masked` is what + // `subsample::sample_layer` would have computed upstream (see + // `SurvivingFamily::masked`'s own docs) — the masking decision is baked + // in here, not passed to `reduce_pairwise` itself anymore. + fn ambiguous_vs_single(masked: Vec>) -> Vec { + vec![SurvivingFamily { family_idx: 0, mask: FamilyMask::EMPTY.with(0).with(1), masked }] } #[test] fn cardinality_uses_raw_mask_without_no_ambiguity() { let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&ambiguous_vs_single(), 0, false, false, &mut tally, &mut dispersion); + let families = ambiguous_vs_single(vec![Some(0b0011), Some(0b0001)]); + reduce_pairwise(&families, 0, &mut tally, &mut dispersion); let counts = tally.cardinality_tally(&[]).counts; assert_eq!(counts[2][1], 1, "genome 0's raw cardinality (2) should be tallied"); assert_eq!(counts[0][1], 0, "no masking requested — nothing should collapse to 0"); @@ -558,12 +531,14 @@ mod cardinality_masking_tests { #[test] fn cardinality_masks_ambiguous_genome_to_zero_under_no_ambiguity() { - // Matches what `AlignmentReducer`/`masked_state` would show for - // genome 0 at this site under `--no-ambiguity`: `?`, i.e. no - // information — cardinality 0, not its raw 2. + // `masked[0] == None` matches what `sample_layer` would have + // computed for genome 0 at this site under `--no-ambiguity`: `?` + // in the alignment, i.e. no information — cardinality 0, not its + // raw 2. let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&ambiguous_vs_single(), 0, false, true, &mut tally, &mut dispersion); + let families = ambiguous_vs_single(vec![None, Some(0b0001)]); + reduce_pairwise(&families, 0, &mut tally, &mut dispersion); let counts = tally.cardinality_tally(&[]).counts; assert_eq!(counts[2][1], 0, "raw cardinality must not leak through under --no-ambiguity"); assert_eq!(counts[0][1], 1, "the ambiguous genome should be masked to cardinality 0"); diff --git a/src/obikphylo/src/siblings/algorithms/snp_distance.rs b/src/obikphylo/src/siblings/algorithms/snp_distance.rs index 099ff3c8..01d62177 100644 --- a/src/obikphylo/src/siblings/algorithms/snp_distance.rs +++ b/src/obikphylo/src/siblings/algorithms/snp_distance.rs @@ -278,12 +278,16 @@ mod tests { /// single-base composition, an edge case irrelevant to real data, not /// something these tests need to exercise). fn tally_2genomes(pairs: &[(u8, u8)], n_shared: usize) -> PairwiseTally { + // No masking requested (`free_loss = no_ambiguity = false`) — + // `masked_state(m, false, false)` is always `Some(m)`, so the raw + // mask is exactly what `subsample::sample_layer` would have + // computed here. let mut families = Vec::new(); for &(a, b) in pairs { families.push(SurvivingFamily { family_idx: families.len(), mask: FamilyMask::EMPTY.with(a.trailing_zeros() as u8).with(b.trailing_zeros() as u8), - genome_mask: vec![a, b], + masked: vec![Some(a), Some(b)], }); } const BASES: [u8; 4] = [A, C, G, T]; @@ -293,12 +297,12 @@ mod tests { families.push(SurvivingFamily { family_idx: families.len(), mask: FamilyMask::EMPTY.with(b.trailing_zeros() as u8).with(other.trailing_zeros() as u8), - genome_mask: vec![b, b], + masked: vec![Some(b), Some(b)], }); } let mut tally = PairwiseTally::new(2); let mut dispersion = PartitionDispersion::default(); - reduce_pairwise(&families, 0, false, false, &mut tally, &mut dispersion); + reduce_pairwise(&families, 0, &mut tally, &mut dispersion); tally } diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index 31ee836e..9ad7a2a9 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -54,7 +54,7 @@ use rand::RngExt; use super::entropy::iter_full_entropy; use super::family_scan::{Selection, scan_layer_families}; use super::is_fast_mode; -use super::masking::survives_masking; +use super::masking::{masked_state, survives_masking}; use super::minorant_selection::eligible_bitset_layer; use crate::siblings::FamilyMask; use crate::siblings::extensions::SiblingBuilder; @@ -66,15 +66,24 @@ const MAX_TOPUP_ROUNDS: usize = 5; /// One surviving family — already drawn (Bernoulli/entropy-bias), already /// resolved (`scan_layer_families`), already passed post-hoc masking -/// (`super::masking::survives_masking`). `genome_mask` is an owned copy: the -/// borrowed slice `scan_layer_families` hands its own callback doesn't -/// outlive that callback, but a `SurvivingFamily` is meant to be handed to -/// [`sample_index`]'s caller as part of a whole layer's batch, well after -/// that callback returns. +/// (`super::masking::survives_masking`). `masked[g]` is genome `g`'s +/// [`super::masking::masked_state`] result — computed exactly once, here, +/// from the raw (per-family, per-genome) resolved state, *before* +/// `--free-loss`/`--no-ambiguity` are applied — never the raw state +/// itself: every consumer downstream (this family's own survival check, +/// its eventual alignment column, its contribution to the pairwise tally) +/// needs the masked view, and computing it once and sharing it is what +/// guarantees they can never disagree about what a genome's state was at +/// this site (see `algorithms::pairwise::reduce_pairwise`'s own docs for +/// the inconsistency that arose from each consumer masking independently). +/// An owned `Vec`: the borrowed slice `scan_layer_families` hands its own +/// callback doesn't outlive that callback, but a `SurvivingFamily` is meant +/// to be handed to [`sample_index`]'s caller as part of a whole layer's +/// batch, well after that callback returns. pub struct SurvivingFamily { pub family_idx: usize, pub mask: FamilyMask, - pub genome_mask: Vec, + pub masked: Vec>, } /// Gaussian-kernel entropy bias parameters — `mu`/`sigma` are the @@ -111,9 +120,10 @@ fn circular_entropy_values( /// [`SurvivingFamily`] this produces is already a usable alignment site, no /// second resolution pass needed by the caller. `excluded[g]` (see /// `survives_masking`'s own docs) never counts genome `g` toward that check -/// — `genome_mask` itself still carries every genome's own resolved state, -/// excluded or not, so a caller building per-genome output (e.g. an -/// alignment row) must apply the same exclusion itself when consuming it. +/// — `SurvivingFamily::masked` itself still carries every genome's own +/// masked state, excluded or not, so a caller building per-genome output +/// (e.g. an alignment row) must apply the same exclusion itself when +/// consuming it. /// /// `survivors` is a single `Arc>` — one allocation per /// layer (bounded by that layer's own quota, never the whole index), handed @@ -348,12 +358,10 @@ fn sample_layer( mask.family_size() >= 2, "eligible bitset must only accept non-monomorphic minorants" ); - if survives_masking(genome_mask, excluded, free_loss, no_ambiguity) { - survivors.push(SurvivingFamily { - family_idx, - mask, - genome_mask: genome_mask.to_vec(), - }); + let masked: Vec> = + genome_mask.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect(); + if survives_masking(&masked, excluded) { + survivors.push(SurvivingFamily { family_idx, mask, masked }); kept += 1; } }, -- 2.54.1 From fdfc1eb3ddf8e44732ef0dcb26f64a11904168af Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 18 Sep 2026 18:57:51 +0200 Subject: [PATCH 07/10] Refactor siblings module and enhance sampling logic Introduces filtering for scan families, handles genome exclusion arguments, refactors the `siblings` module architecture, and clarifies invariants for statistical calculations. --- src/obikmer/src/cmd/phylo/args.rs | 23 +- src/obikmer/src/cmd/phylo/mod.rs | 8 +- src/obikmer/src/cmd/phylo/sankoff.rs | 6 +- src/obikphylo/src/lib.rs | 15 +- .../src/siblings/algorithms/family_scan.rs | 10 +- src/obikphylo/src/siblings/algorithms/mod.rs | 1 + .../src/siblings/algorithms/pairwise.rs | 57 +- .../src/siblings/algorithms/stats.rs | 2 +- .../src/siblings/algorithms/subsample.rs | 33 +- .../src/siblings/extensions/sibling_ext.rs | 41 +- src/obikphylo/src/siblings_old/alignment.rs | 134 ----- src/obikphylo/src/siblings_old/build.rs | 368 ------------ src/obikphylo/src/siblings_old/cache.rs | 235 -------- src/obikphylo/src/siblings_old/cardcomp.rs | 336 ----------- src/obikphylo/src/siblings_old/cardinality.rs | 132 ----- src/obikphylo/src/siblings_old/distance.rs | 244 -------- src/obikphylo/src/siblings_old/entropy.rs | 269 --------- .../src/siblings_old/entropy_annex.rs | 162 ----- src/obikphylo/src/siblings_old/family_scan.rs | 373 ------------ src/obikphylo/src/siblings_old/helpers.rs | 23 - src/obikphylo/src/siblings_old/iter.rs | 245 -------- src/obikphylo/src/siblings_old/mod.rs | 76 --- .../src/siblings_old/sankoff_bundle.rs | 250 -------- .../src/siblings_old/siblingannex.rs | 433 -------------- src/obikphylo/src/siblings_old/stats.rs | 153 ----- src/obikphylo/src/siblings_old/subsample.rs | 303 ---------- src/obikphylo/src/siblings_old/tests.rs | 557 ------------------ 27 files changed, 106 insertions(+), 4383 deletions(-) delete mode 100644 src/obikphylo/src/siblings_old/alignment.rs delete mode 100644 src/obikphylo/src/siblings_old/build.rs delete mode 100644 src/obikphylo/src/siblings_old/cache.rs delete mode 100644 src/obikphylo/src/siblings_old/cardcomp.rs delete mode 100644 src/obikphylo/src/siblings_old/cardinality.rs delete mode 100644 src/obikphylo/src/siblings_old/distance.rs delete mode 100644 src/obikphylo/src/siblings_old/entropy.rs delete mode 100644 src/obikphylo/src/siblings_old/entropy_annex.rs delete mode 100644 src/obikphylo/src/siblings_old/family_scan.rs delete mode 100644 src/obikphylo/src/siblings_old/helpers.rs delete mode 100644 src/obikphylo/src/siblings_old/iter.rs delete mode 100644 src/obikphylo/src/siblings_old/mod.rs delete mode 100644 src/obikphylo/src/siblings_old/sankoff_bundle.rs delete mode 100644 src/obikphylo/src/siblings_old/siblingannex.rs delete mode 100644 src/obikphylo/src/siblings_old/stats.rs delete mode 100644 src/obikphylo/src/siblings_old/subsample.rs delete mode 100644 src/obikphylo/src/siblings_old/tests.rs diff --git a/src/obikmer/src/cmd/phylo/args.rs b/src/obikmer/src/cmd/phylo/args.rs index 305607bb..8226533f 100644 --- a/src/obikmer/src/cmd/phylo/args.rs +++ b/src/obikmer/src/cmd/phylo/args.rs @@ -35,8 +35,9 @@ fn parse_gamma_shape(s: &str) -> Result { /// `--distance` value — either one of `obikphylo::DistanceMetric`'s /// whole-index metrics (routed to `IndexCache::distance`) or one of /// `obikphylo::SnpDistanceKind`'s `snp-*` corrections (routed to -/// `SiblingExt::snp_distance`, the sibling-annex pipeline) — two genuinely -/// different code paths behind one CLI vocabulary, see +/// `obikphylo::siblings::distance_matrix`, over a tally built by +/// `sampled_scan`/`PairwiseReducer`, the sibling-annex pipeline) — two +/// genuinely different code paths behind one CLI vocabulary, see /// `DevDocMD/theory/evolutionary_distances.md`, "`--distance` unification". #[derive(clap::ValueEnum, Clone, Copy, Debug)] pub enum DistanceArg { @@ -121,10 +122,10 @@ pub struct PhyloArgs { /// Exclude a genome (by its exact label) — from `--pseudo-alignment`'s /// sampling (a family whose only polymorphism lived in an excluded /// genome is discarded during sampling, not filtered afterward — see - /// `obikphylo::siblings::extensions::SiblingExt::snp_pseudo_alignment`'s - /// own docs) and from the distance matrix / shared-kmer matrix CSV - /// output (row and column both dropped; the underlying computation - /// itself is unaffected). Repeatable. + /// `obikphylo::siblings::sampled_scan`'s own docs) and from the + /// distance matrix / shared-kmer matrix CSV output (row and column + /// both dropped; the underlying computation itself is unaffected). + /// Repeatable. #[arg(long = "exclude-genome", value_name = "LABEL")] pub exclude_genome: Vec, @@ -194,10 +195,9 @@ pub struct PhyloArgs { /// `--pseudo-alignment`/`--sankoff` (mandatory for both) and for a /// `snp-*` `--distance` value (optional there: omitted means exhaustive /// — every non-monomorphic minorant of the whole index, not an - /// approximation, see `obikphylo::siblings::SiblingExt::snp_distance`'s - /// own docs) — a target, not a guarantee when given (proportional - /// per-layer sampling; see - /// `obikphylo::siblings::SiblingExt::snp_pseudo_alignment`'s own docs). + /// approximation) — a target, not a guarantee when given (proportional + /// per-layer sampling; see `obikphylo::siblings::sampled_scan`'s own + /// docs). #[arg(long)] pub subsample: Option, @@ -329,7 +329,8 @@ pub struct PhyloArgs { /// the central-position SNP substitution spectrum (`snp-raw`, `snp-jc`, /// `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv`) — /// the latter route to a different computation entirely - /// (`SiblingExt::snp_distance`, requires `--sibling-annex` first; see + /// (`obikphylo::siblings::sampled_scan` + `distance_matrix`, requires + /// `--sibling-annex` first; see /// `DevDocMD/theory/evolutionary_distances.md`, "`--distance` /// unification" for the full catalog and why LogDet/Tajima-Nei/F84/ /// HKY85 aren't offered yet). diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 45c64119..43e451bf 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -45,10 +45,10 @@ pub fn run(args: PhyloArgs) { let n = labels.len(); // ── Genome exclusion (`--exclude-genome`) ─────────────────────────────────── - // Resolved once, up front: `snp_pseudo_alignment` needs it baked into - // sampling itself (see its own docs), and the distance/shared-kmer CSV - // writers below just skip these rows/columns at write time — the - // underlying `cache.distance(...)` computation is unaffected either way. + // Resolved once, up front: `sampled_scan` needs it baked into sampling + // itself (see its own docs), and the distance/shared-kmer CSV writers + // below just skip these rows/columns at write time — the underlying + // `cache.distance(...)` computation is unaffected either way. let exclude_mask: Vec = { let mut mask = vec![false; n]; for label in &args.exclude_genome { diff --git a/src/obikmer/src/cmd/phylo/sankoff.rs b/src/obikmer/src/cmd/phylo/sankoff.rs index 0a412a86..9168c80f 100644 --- a/src/obikmer/src/cmd/phylo/sankoff.rs +++ b/src/obikmer/src/cmd/phylo/sankoff.rs @@ -1,5 +1,5 @@ //! Output writers for `--sankoff` — no calibration logic here, just -//! formatting: `obikphylo::siblings::SiblingExt::sankoff_bundle` and the +//! formatting: `obikphylo::siblings::SankoffBundle::finish` and the //! `cardinality_transition_probs`/`composition_transition_probs`/ //! `pairwise_cost_matrix` calibration functions do all the actual work in //! `mod.rs`, this module only serialises their results. @@ -13,8 +13,8 @@ use tracing::info; // ── Sankoff pseudo-alignment → FASTA ──────────────────────────────────────── // -// Same data as `--pseudo-alignment`'s output (`SnpAlignment`/ -// `snp_pseudo_alignment`), re-coded so its symbols match the accompanying +// Same data as `--pseudo-alignment`'s own output (`SnpAlignment`, built by +// `AlignmentReducer`), re-coded so its symbols match the accompanying // `--sankoff` matrix output exactly: `0` for the empty/absent state instead // of `-`, which TNT/PhyG would otherwise read as their own gap character // rather than our "family absent" state. Unless `free_loss` (`--free-loss`) diff --git a/src/obikphylo/src/lib.rs b/src/obikphylo/src/lib.rs index 7e80a1bc..8610d84b 100644 --- a/src/obikphylo/src/lib.rs +++ b/src/obikphylo/src/lib.rs @@ -14,12 +14,15 @@ //! to a lower-level crate if an indexing-side consumer (e.g. default-`k` //! guidance) appears. //! -//! `siblings` is being reconnected piece by piece while `obikindex`'s -//! stateless audit is caught up with — see the project memory on this. So -//! far it only carries the annex-construction path -//! (`SiblingExt::build_sibling_annex`); the rest of the old -//! `siblings_old` (SNP distance, cardinality, pseudo-alignment, stats) is -//! still disconnected. +//! `siblings` carries the full pipeline today: annex construction +//! (`SiblingExt::build_sibling_annex`), Shannon entropy/family-overlap +//! annexes, and the sampled SNP path (pseudo-alignment, `snp-*` +//! `--distance`, Sankoff cost-matrix calibration) driven through a single +//! shared scan (`siblings::sampled_scan` + `LayerReducer` impls) — see +//! `siblings::algorithms::scan`'s own module docs. `siblings_old` is its +//! predecessor, fully superseded and not compiled into this crate (no +//! `mod siblings_old;` anywhere) — dead source left on disk, not an +//! in-progress migration target. mod distance; pub mod karlin; diff --git a/src/obikphylo/src/siblings/algorithms/family_scan.rs b/src/obikphylo/src/siblings/algorithms/family_scan.rs index 692b7272..4afa908b 100644 --- a/src/obikphylo/src/siblings/algorithms/family_scan.rs +++ b/src/obikphylo/src/siblings/algorithms/family_scan.rs @@ -78,9 +78,13 @@ fn family_batch_size(n_genomes: usize) -> usize { } /// Restricts [`scan_layer_families`] to a subset of a layer's families, -/// identified by their iteration-order index. `All` (the only variant any -/// current caller uses) keeps a full, unsampled scan; `Some` is exercised -/// once sampling (`--subsample`/`--entropy-bias`) is reconnected. +/// identified by their iteration-order index. `All` keeps a full, +/// unsampled scan (`SiblingExt::shannon_entropy_csv`, +/// `sibling_annex_stats`); `Some` bounds it to a caller-picked set — +/// `subsample::sample_layer`'s own accepted-candidates batch, or +/// `entropy::ensure_layer_entropy_annex`'s non-monomorphic-only +/// restriction — skipping the expensive cross-partition resolution +/// entirely for every family outside the set. pub enum Selection<'a> { All, Some(&'a std::collections::HashSet), diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 7eff4aff..24b62f4c 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -43,6 +43,7 @@ pub use subsample::{EntropyBias, SurvivingFamily}; pub(crate) use annex::build_layer_sibling_annex; pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4}; pub(crate) use family_scan::{Selection, scan_layer_families}; +pub(crate) use minorant_selection::non_monomorphic_reindex_layer; pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram}; /// Whether every layer number in `cache` fits in a `FamilyMask` field diff --git a/src/obikphylo/src/siblings/algorithms/pairwise.rs b/src/obikphylo/src/siblings/algorithms/pairwise.rs index 86f31921..c358e2e8 100644 --- a/src/obikphylo/src/siblings/algorithms/pairwise.rs +++ b/src/obikphylo/src/siblings/algorithms/pairwise.rs @@ -22,15 +22,19 @@ use rkyv::{Archive, Deserialize, Serialize}; use super::subsample::SurvivingFamily; -/// One pair's own tallies, unfiltered — `subst` is populated only by -/// variable families (a substitution, `bi != bj`, is only possible when the -/// family varies *somewhere* in the index; see the module docs), so it -/// needs no separate `variable` gate of its own. `same_all` counts every -/// agreement (`bi == bj`, any family); `same_variable` is the subset from -/// variable families only — [`RawSnpDistanceOutput`] wants the former, -/// [`BasePairTally`] the latter (they are *not* the same population — see -/// `BasePairTally::same`'s own docs). `cardinality` is restricted to -/// variable families only, matching [`CardinalityTally`]'s own scope. +/// One pair's own tallies, unfiltered. Both `subst` and `same_all` are +/// populated by every family reaching [`reduce_pairwise`] — no separate +/// "variable families only" gate needed for either: [`super::subsample::sample_layer`] +/// only ever draws from `minorant_selection::eligible_bitset_layer`, which +/// contains exclusively non-monomorphic minorants (`family_size() >= 2`, +/// enforced by a `debug_assert!` right where a [`SurvivingFamily`] is +/// built), so *every* family folded here already varies somewhere in the +/// index. `same_all` is what both [`RawSnpDistanceOutput`] and +/// [`BasePairTally`] read (there used to be a second, `same_variable` +/// field restricted to variable families for the latter — removed once it +/// was confirmed, via the invariant above, to always hold the exact same +/// values as `same_all`). `cardinality` is populated for every family too, +/// same reasoning. /// /// `Archive`/`Serialize`/`Deserialize`: part of [`PairwiseTally`]'s own /// `--session` persistence (see that struct's docs) — `rkyv` needs every @@ -40,7 +44,6 @@ use super::subsample::SurvivingFamily; struct PairStats { subst: [[u64; 4]; 4], same_all: [u64; 4], - same_variable: [u64; 4], cardinality: [[u64; 5]; 5], } @@ -145,7 +148,7 @@ impl PairwiseTally { } let stats = self.pair(i, j); for a in 0..4 { - same[a] += stats.same_variable[a]; + same[a] += stats.same_all[a]; for b in 0..4 { counts[a][b] += stats.subst[a][b]; } @@ -267,19 +270,17 @@ pub struct BasePairTally { /// `same`, not here. pub counts: [[u64; 4]; 4], /// `same[a]` = number of eligible loci, pooled over included genome - /// pairs, where both genomes' single forms are `a` **and** the family - /// varies somewhere in the index — a fully invariant family's - /// agreement is genome-wide background, not SNP-adjacent signal (see - /// `PairStats::same_all` vs `same_variable`). + /// pairs, where both genomes' single forms are `a` — every family + /// folded into [`PairwiseTally`] already varies somewhere in the index + /// by construction (see `PairStats`'s own docs), so there is no + /// further "variable families only" restriction to apply here. pub same: [u64; 4], } /// Cardinality (`0..=4` members of a family) co-occurrence, pooled over /// every genome pair except `--exclude-genome` ones (see /// [`PairwiseTally::cardinality_tally`]'s own docs for why this is *not* -/// gated by `ratio_ceiling` the way [`BasePairTally`] is), restricted to -/// variable families (`family_size() >= 2`) — matching -/// `snp_pseudo_alignment`'s own scope. `pub`: part of +/// gated by `ratio_ceiling` the way [`BasePairTally`] is). `pub`: part of /// [`super::SankoffBundle`]'s public signature. pub struct CardinalityTally { /// `counts[a][b] == counts[b][a]` = number of family sites, pooled over @@ -418,9 +419,9 @@ impl PartitionDispersion { /// same array [`algorithms::alignment::reduce_alignment`] reads too: this /// is what guarantees `cardinality` agrees with what the alignment /// actually shows for the same family/genome (a masked-away genome -/// contributes `0`, matching its `?`). `subst`/`same_all`/`same_variable` -/// don't need anything beyond `single_form`'s "exactly one bit" check on -/// the already-masked state. +/// contributes `0`, matching its `?`). No "variable families only" gate +/// on any field: see [`PairStats`]'s own docs for why every family reaching +/// this function already varies somewhere in the index, by construction. pub(crate) fn reduce_pairwise( survivors: &[SurvivingFamily], partition: usize, @@ -431,7 +432,6 @@ pub(crate) fn reduce_pairwise( let mut partition_subst: u64 = 0; let mut partition_eligible: u64 = 0; for family in survivors { - let variable = family.mask.family_size() >= 2; let single_form = |g: usize| -> Option { family.masked[g].filter(|m| m.count_ones() == 1).map(|m| m.trailing_zeros() as u8) }; @@ -451,18 +451,13 @@ pub(crate) fn reduce_pairwise( stats.subst[bj as usize][bi as usize] += 1; } else { stats.same_all[bi as usize] += 1; - if variable { - stats.same_variable[bi as usize] += 1; - } } } - if variable { - let card_j = masked_cardinality(j); - stats.cardinality[card_i][card_j] += 1; - if card_i != card_j { - stats.cardinality[card_j][card_i] += 1; - } + let card_j = masked_cardinality(j); + stats.cardinality[card_i][card_j] += 1; + if card_i != card_j { + stats.cardinality[card_j][card_i] += 1; } } } diff --git a/src/obikphylo/src/siblings/algorithms/stats.rs b/src/obikphylo/src/siblings/algorithms/stats.rs index e8417ef8..b0536788 100644 --- a/src/obikphylo/src/siblings/algorithms/stats.rs +++ b/src/obikphylo/src/siblings/algorithms/stats.rs @@ -93,7 +93,7 @@ pub(crate) fn sibling_family_size_histogram(cache: &IndexCache) -> OKIResult<[u6 /// at a finer grain, across partitions — stacking another parallel layer /// loop on top would fight it for locality instead of helping, exactly the /// same reasoning `SiblingExt::build_sibling_annex`/`shannon_entropy_csv`/ -/// `snp_pseudo_alignment` already follow). +/// `algorithms::scan::sampled_scan` already follow). pub(crate) fn sibling_annex_stats(cache: &IndexCache) -> OKIResult { let n_genomes = cache.meta().genomes().len(); let fast_mode = is_fast_mode(cache); diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index 9ad7a2a9..c33cde1e 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -89,7 +89,7 @@ pub struct SurvivingFamily { /// Gaussian-kernel entropy bias parameters — `mu`/`sigma` are the /// `--entropy`/`--entropy-sd` CLI values (defaulted by the caller, not /// here). `pub`, not `pub(crate)`: part of the public signature of -/// [`crate::siblings::extensions::SiblingExt::snp_pseudo_alignment`]. +/// [`super::scan::sampled_scan`]. #[derive(Clone, Copy)] pub struct EntropyBias { pub mu: f64, @@ -138,24 +138,19 @@ fn circular_entropy_values( /// function's own iteration order (`cache.partitions()` then `0..n_layer`, /// both deterministic for an unchanged index), to skip *entirely* — /// neither their (cheap) eligibility-bitset lookup's `sample_layer` call -/// nor `on_layer` runs for them. Exists for `--session` resume: quotas are -/// computed from `total_eligible` (summed over *every* layer regardless of -/// `skip_layers`), so a resumed run reproduces the exact same *per-layer -/// quotas* an uninterrupted one would have — i.e. how many sites each -/// remaining layer contributes stays correct. It does **not** reproduce -/// the exact same *sample*: `sample_layer`'s draws come from `rand::rng()` -/// (`Self::sample_layer`'s own docs), a per-process, OS-seeded generator -/// with no continuity across a process boundary, so the layers processed -/// after a resume draw different (but equally valid) random outcomes than -/// an uninterrupted run would have. Confirmed by direct test (kill mid-run, -/// resume, compare to an uninterrupted run against the same index/params: -/// different distance matrices, both legitimate samples). This is -/// deliberate, not a defect: run-to-run sampling variance is wanted (see -/// `DevDocMD/theory/evolutionary_distances.md`, "`--session`" section) — -/// making resume bit-reproducible would require seeding deterministically, -/// which would also remove that variance for ordinary repeated runs -/// unless scoped carefully, and was explicitly declined for now. `0` for -/// every caller not doing checkpointed resume. +/// nor `on_layer` runs for them. Built for checkpointed-resume support +/// (quotas are computed from `total_eligible`, summed over *every* layer +/// regardless of `skip_layers`, so a resumed run reproduces the exact same +/// *per-layer quotas* an uninterrupted one would have — i.e. how many +/// sites each remaining layer contributes stays correct — without +/// reproducing the exact same *sample*: `sample_layer`'s draws come from +/// `rand::rng()`, a per-process, OS-seeded generator with no continuity +/// across a process boundary). No current caller passes anything but `0`: +/// the `--session`-based resume mechanism this was built for was removed +/// (see `algorithms::scan`'s own module docs) pending a rebuild on top of +/// [`sampled_scan`](super::scan::sampled_scan) — the parameter stays, as +/// the seam a future resume mechanism would hook into, rather than being +/// ripped out and re-added later. /// /// Returns the actual number of *newly processed* sites kept (not /// counting whatever `skip_layers` layers already contributed on an diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index c80bf521..7cc26ba3 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -15,7 +15,8 @@ use obisys::progress_bar; use crate::siblings::algorithms::{ Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4, - is_fast_mode, scan_layer_families, sibling_annex_stats, sibling_family_size_histogram, + is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, sibling_annex_stats, + sibling_family_size_histogram, }; use crate::siblings::extensions::SiblingBuilder; use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap}; @@ -51,8 +52,12 @@ pub trait SiblingExt { /// every cached layer, from an already-built sibling annex (run /// [`build_sibling_annex`](Self::build_sibling_annex) first). /// - /// Full, unsampled scan of every cached layer (`Selection::All`) — no - /// `--subsample`/`--entropy-bias` yet (see the project memory on this). + /// Full, unsampled scan of every non-monomorphic minorant of every + /// cached layer (`Selection::Some`, restricted the same way + /// `ensure_layer_entropy_annex` is — monomorphic minorants have + /// nothing to report here, so never pay `scan_layer_families`'s + /// cross-partition resolution for them) — no `--subsample`/ + /// `--entropy-bias` yet (see the project memory on this). fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()>; /// The global family-size histogram alone (`SiblingAnnexStats::counts`, @@ -152,11 +157,9 @@ impl SiblingExt for IndexCache { fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> { // Entropy/family-overlap annexes aren't this method's own output, - // but building them is exactly the same full scan this method is - // about to run anyway — see - // `SiblingBuilder::ensure_entropy_and_overlap_annexes`'s own docs - // for why that's a side effect worth taking here rather than a - // separate, explicit step. + // but ensuring they exist is a cheap prerequisite check, not a + // reason to reuse their scan here — see below, this method + // deliberately runs its own, differently-scoped one. self.ensure_entropy_and_overlap_annexes()?; let n_genomes = self.meta().genomes().len(); @@ -170,6 +173,19 @@ impl SiblingExt for IndexCache { for part in self.partitions() { let n_layer = self.n_layer(part).unwrap_or(0); for l in 0..n_layer { + let layer = self + .get_layer(part, l) + .expect("cache-consistent: n_layer(part) already bounds l"); + // Bounded to non-monomorphic minorants only, same reasoning + // as `ensure_layer_entropy_annex`'s own docs: monomorphism + // is already known from the annex bits alone (~98% of a + // layer's minorants, measured on real data), so there is no + // reason to pay `scan_layer_families`'s expensive + // cross-partition resolution for families this method + // skips right below anyway. + let reindex = non_monomorphic_reindex_layer(layer.dir())?; + let eligible: std::collections::HashSet = reindex.keys().copied().collect(); + let mut write_err = None; scan_layer_families( self, @@ -177,14 +193,15 @@ impl SiblingExt for IndexCache { l, n_genomes, fast_mode, - &Selection::All, + &Selection::Some(&eligible), |family_idx, mask, genome_mask| { if write_err.is_some() { return; } - if mask.family_size() < 2 { - return; // monomorphic family — entropy trivially 0, no signal, skip - } + debug_assert!( + mask.family_size() >= 2, + "Selection::Some(eligible) must only visit non-monomorphic minorants" + ); let Some((h15, n_present)) = family_entropy(genome_mask) else { return; }; diff --git a/src/obikphylo/src/siblings_old/alignment.rs b/src/obikphylo/src/siblings_old/alignment.rs deleted file mode 100644 index 7efbfff5..00000000 --- a/src/obikphylo/src/siblings_old/alignment.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::sync::Arc; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::family_scan::{Selection, scan_layer_families}; -use super::subsample::EntropyBias; - -/// IUPAC ambiguity code for a per-genome family presence mask (bit `b` set -/// iff the genome carries the member whose own central base is `b`): -/// single bit -> the plain base; 2 or 3 bits -> the matching IUPAC -/// ambiguity code (preserves partial information instead of collapsing to -/// `N`, the same convention used for diploid heterozygous VCF/FASTA sites); -/// all 4 bits -> `N`; no bits (genome carries none of the family's observed -/// members) -> `-` (no data at this locus for this genome). -pub(super) fn iupac_code(mask: u8) -> u8 { - match mask & 0b1111 { - 0b0000 => b'-', - 0b0001 => b'A', - 0b0010 => b'C', - 0b0100 => b'G', - 0b1000 => b'T', - 0b0101 => b'R', // A/G - 0b1010 => b'Y', // C/T - 0b0110 => b'S', // C/G - 0b1001 => b'W', // A/T - 0b1100 => b'K', // G/T - 0b0011 => b'M', // A/C - 0b1110 => b'B', // C/G/T - 0b1101 => b'D', // A/G/T - 0b1011 => b'H', // A/C/T - 0b0111 => b'V', // A/C/G - 0b1111 => b'N', - _ => unreachable!("masked to 4 bits"), - } -} - -/// A SNP-only pseudo-alignment: one row (byte sequence, IUPAC-coded) per -/// genome, one column per variable family (`family_size() >= 2` — monomorphic -/// families carry no signal and are skipped, unlike `raw_snp_distance`'s -/// tally which does count them as `shared`). Column order is the same, -/// deterministic sweep order as the annex build (partition, then layer, then -/// slot) — arbitrary but stable and identical across genomes, which is all a -/// pseudo-alignment needs (there is no natural genomic coordinate to sort by -/// once flanks are dropped). See `DevDocMD/theory/evolutionary_distances.md`, -/// "Multi-genome framing: family as pseudo-alignment column". -pub struct SnpAlignment { - /// `sequences[g]` = genome `g`'s IUPAC-coded row, same length for every - /// genome (`sequences.len()` columns). - pub sequences: Vec>, -} - -/// Adds [`snp_pseudo_alignment`](Self::snp_pseudo_alignment) to `KmerIndex` — -/// phylo-domain functionality, kept out of `obikindex` itself (see -/// `DevDocMD/architecture/siblings.md`). -pub trait SnpAlignmentExt { - /// Build the SNP-only pseudo-alignment from an already-built sibling - /// annex (run [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) first). - /// - /// `subsample`: `--subsample N` — cap the number of variable families - /// retained to (approximately) `N`, sampled proportionally per layer - /// among non-monomorphic minorants (see - /// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`"). - /// `None` keeps today's behaviour: every non-monomorphic minorant of - /// every layer. `entropy_bias`: `--entropy`/`--entropy-sd` — weight - /// that sample (or, with `subsample = None`, filter the whole index) - /// by a Gaussian kernel on each family's entropy — see - /// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". - fn snp_pseudo_alignment( - &self, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult; -} - -impl SnpAlignmentExt for KmerIndex { - fn snp_pseudo_alignment( - &self, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - let selections = - super::subsample::compute_selections(self, &layer_dirs, subsample, entropy_bias)?; - - let pb = progress_bar("snp_pseudo_alignment", layer_dirs.len() as u64, "layers"); - // One layer at a time, not `par_iter()` over layers — same - // rationale as `build_sibling_annex`: running many layers' - // `scan_layer_families` concurrently would each group their own - // lookups by partition internally, but interleave those sweeps - // across layers at the OS level, scattering page-cache access over - // every partition at once again and defeating the whole point of - // the grouping. Columns appended straight into `sequences` as each - // family comes back from `scan_layer_families` (bounded-batch, not - // whole-layer) — no intermediate per-layer buffer, layer order - // giving a single deterministic column order across the whole index. - let mut sequences: Vec> = vec![Vec::new(); n_genomes]; - for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |_family_idx, mask, genome_mask| { - if mask.family_size() < 2 { - return; // monomorphic family — no signal, skip - } - for (g, &m) in genome_mask.iter().enumerate() { - sequences[g].push(iupac_code(m)); - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(SnpAlignment { sequences }) - } -} diff --git a/src/obikphylo/src/siblings_old/build.rs b/src/obikphylo/src/siblings_old/build.rs deleted file mode 100644 index 4ed4a703..00000000 --- a/src/obikphylo/src/siblings_old/build.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::Ordering; - -use rayon::prelude::*; - -use obikseq::CanonicalKmer; -use obikindex::layer::MphfLayer; -use obipipeline::ThrottleGuard; -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::OKIResult; - -use super::cache::PartitionCache; -use super::helpers::{central_base, is_minorant}; -use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnexBuilder}; - -// ── obipipeline data types ───────────────────────────────────────────────── - -/// A batch of this layer's distinct k-mers (iteration-order index + k-mer), -/// the pipeline's source item — batched, not one k-mer per item, so that -/// pipeline messages and their synchronisation cost stay amortised over -/// thousands of lookups (see `build_layer_sibling_annex`'s comment on -/// `BATCH_SIZE`). Carries the throttle permit for the whole batch, moved -/// (not cloned) into the corresponding `VariantBatch` — a 1-to-1 transform, -/// unlike the fan-out `Flat` stage this replaced, needs no `Arc` sharing. -/// -/// The `usize` is this k-mer's position in `iter_kmers()`'s enumeration -/// order, **not** an MPHF slot — see `DevDocMD/architecture/siblings.md`. It -/// is the same index the annex file is written under. -struct SourceBatch { - items: Vec<(usize, CanonicalKmer)>, - _permit: ThrottleGuard, -} - -/// One batch's worth of central-substitution variants (up to 3 per source -/// k-mer), each already routed to its destination partition and carrying -/// its own central base (0=A/1=C/2=G/3=T) — the mask bit it will set on a -/// hit. `(dest_partition, variant, source_order, base)` per entry, where -/// `source_order` is the source k-mer's iteration-order index (see -/// `SourceBatch`), not an MPHF slot. -struct VariantBatch { - items: Vec<(usize, CanonicalKmer, usize, u8)>, - _permit: ThrottleGuard, -} - -enum SibData { - Batch(SourceBatch), - Variants(VariantBatch), -} - -/// Adds [`build_sibling_annex`](Self::build_sibling_annex) to `KmerIndex`. -pub trait SiblingAnnexBuildExt { - /// Build the sibling-count/minorant annex for every layer of every - /// partition of this (already built) index, writing one annex file per - /// layer alongside its existing index files. Safe to call again later - /// (e.g. after a fresh `merge`) — each run simply overwrites the annex - /// files of the index it is called on. - /// - /// Construction only — no statistics gathered here on purpose: this is - /// meant to run routinely (it is the artefact the SNP-family distances - /// will consume), while the sibling-count distribution - /// ([`sibling_annex_stats`](super::stats::SiblingStatsExt::sibling_annex_stats)) - /// is a separate, occasional diagnostic pass over the result, not run - /// every time. - /// - /// Cross-partition/cross-layer lookups are required (a k-mer's siblings - /// can live in any partition), but the layer loop itself — and thus the - /// annex file this produces — stays local to one layer at a time. - fn build_sibling_annex(&self) -> OKIResult<()>; -} - -impl SiblingAnnexBuildExt for KmerIndex { - fn build_sibling_annex(&self) -> OKIResult<()> { - let n_parts = self.n_partitions(); - - tracing::info!("opening {n_parts} partition(s) for the sibling-annex sweep"); - let cache = Arc::new(PartitionCache::build( - self, - n_parts, - self.meta().config.with_counts, - )?); - - let pb = progress_bar("sibling_annex", n_parts as u64, "partitions"); - let mut total_slots: u64 = 0; - for part in 0..n_parts { - let index_dir = self.index_dir(part); - if !index_dir.exists() { - pb.inc(1); - continue; - } - let meta = self.partition_meta(part)?; - - let mut part_slots: u64 = 0; - for l in 0..meta.n_layers { - part_slots += build_layer_sibling_annex( - self, - &self.layer_dir(part, l), - n_parts, - l, - &cache, - )?; - } - total_slots += part_slots; - pb.inc(1); - pb.set_message(format!( - "partition {part}: {part_slots} kmers ({total_slots} total)" - )); - } - pb.finish_and_clear(); - tracing::info!("sibling annex built — {total_slots} kmers across {n_parts} partitions"); - - Ok(()) - } -} - -/// Returns the number of distinct k-mers (annex entries) processed, for -/// progress reporting. A free function, not a `KmerIndex` method — called -/// only from `build_sibling_annex` above, in the same file. -fn build_layer_sibling_annex( - index: &KmerIndex, - layer_dir: &Path, - n_parts: usize, - l: usize, - cache: &Arc, -) -> OKIResult { - let mphf = MphfLayer::open(layer_dir)?; - let k = index.kmer_size(); - let n = mphf.n(); - - // Whether this layer's family-member fields can carry a real layer - // number (exploited by `family_scan::scan_layer_families`'s fast path) - // or must stay presence-only — a single fact about the whole index, - // decided once by `cache` (built from the same `PartitionMeta` this - // function would otherwise re-derive) so the writer and every reader - // agree; see `PartitionCache::fast_mode`'s docs. - let fast_mode = cache.fast_mode(); - - // ── The annex file itself is the reconciliation state, indexed by - // this layer's k-mer iteration order (the physical layout of - // `unitigs.bin`), never by MPHF slot — this layer's own k-mers are - // known members by construction, so no evidence check, no MPHF - // slot, and no slot -> k-mer reconstruction is needed or legitimate - // here (see `DevDocMD/architecture/siblings.md`: the MPHF is not - // invertible, and evidence answers membership, not identity). No - // separate in-memory accumulator: every concurrent write goes - // straight through `SiblingAnnexBuilder::atomic_slot` into the - // mmap — a layer can hold billions of k-mers, so a `Vec` shadowing - // the whole file in RAM just to copy it out again at the end (the - // previous design) doubles memory for no benefit once the builder - // itself can be written concurrently. - // - // `Arc`, not a bare `SiblingAnnexBuilder` — - // `Pipe::apply` requires its source iterator to be `Send + 'static` - // (its items are dispatched to worker threads that outlive this - // call), so the batch-generating closure below needs an owned - // handle it can move in, not a borrow of a local. `atomic_slot`, - // not `set`, for every concurrent write below: the gather phase - // parallelises across destination partitions (independent - // `cache.find` calls, safe to run concurrently) and their hits can - // land on arbitrary, possibly-shared source entries — a lock-free - // `fetch_or` avoids needing any synchronisation beyond that. - // Reclaimed as a plain owned value (`Arc::try_unwrap`) once every - // concurrent phase below is done, for the sequential finalisation - // pass. ───────────────────────────────────────────────────────── - let annex_path = layer_dir.join(ANNEX_FILE_NAME); - let builder = Arc::new(SiblingAnnexBuilder::new(n, &annex_path)?); - - // ── obipipeline: a *batch* transform, not a per-k-mer `Flat` one — - // the actual cross-partition lookup reuses - // `KmerPartition::query_partition_with` (the same batching mechanism - // `obikmer query` already uses: open a partition's files once, - // answer a whole batch of queries against it) instead of one lookup - // per pipeline item. A per-item lookup (tried first) reopened/ - // re-mmap'd every target partition's files on every single variant - // — fine at toy scale, but ~90% system time against a real index, - // observed in practice. A *later* attempt still pushed one pipeline - // message per generated variant (a `Flat` stage, `SourceItem` => - // `VariantQuery`, one k-mer in => up to 3 variants out as separate - // messages) — cheaper than reopening files, but sampling a real run - // showed most wall-clock time going into per-message channel - // send/notify syscalls instead of the lookup itself: the pipeline's - // whole point is amortising synchronisation over a batch, and a - // single k-mer's ≤3 variants is far too fine a granularity for - // that. Batching `BATCH_SIZE` source k-mers into one pipeline item - // — a plain 1-to-1 (`|`, not `||`) transform, batch in, batch of - // variants out, one message either way — keeps the per-message - // synchronisation cost amortised over thousands of lookups instead - // of one to three. `BATCH_SIZE` was originally tuned back when the - // per-k-mer cost inside this stage (minimiser via `RollingStat`) - // was ~3x higher than it is now (see `CanonicalKmerOf::minimizer`, - // a direct O(k) bit-arithmetic replacement) — at the old per-k-mer - // cost, dispatch overhead (the scheduler's single-threaded `Select` - // loop: one channel round-trip per batch) was negligible next to - // the work each batch represented; now that the work itself is - // cheaper, that fixed per-batch overhead is proportionally larger, - // so a bigger batch amortises it over more k-mers again. ───────── - const BATCH_SIZE: usize = 32768; - let n_workers = obisys::effective_parallelism(); - let capacity = 256; - - // Throttling limits how many *batches* are in flight at once — the - // permit is acquired per batch (not per k-mer) in the source - // thread, and released once its `VariantBatch` has been read out of - // the pipeline by the accumulation loop below. See - // `obipipeline::throttle`'s docs for why this is required, not - // optional, once a `Flat`-style stage sits in the pipeline. - // - // Batches stream straight from `unitigs.bin` via - // `enumerate_kmers_batch` — never a full-layer `Vec` collect (a - // layer can hold billions of k-mers; see the "no full collect" - // rule). Each k-mer's own base is seeded straight into the annex in - // this same pass, since this is exactly the iteration order the - // annex is keyed on — no separate seeding pass needed. - let seed_builder = Arc::clone(&builder); - let batches = mphf - .enumerate_kmers_batch(BATCH_SIZE) - .map(move |(start, kmers)| { - kmers - .into_iter() - .enumerate() - .map(|(i, kmer)| { - let base = central_base(kmer, k); - let bits = if fast_mode { - FamilyMask::layer_bits(base, l) - } else { - FamilyMask::presence_bits(base) - }; - seed_builder - .atomic_slot(start + i) - .fetch_or(bits, Ordering::Relaxed); - (start + i, kmer) - }) - .collect::>() - }); - - // Diagnostic: count still-empty annex slots after seeding + cross-partition - // resolution. A non-zero count here means some k-mer's own base was never - // written (should be impossible after the batch_offset fix above). - let check_empty = |label: &str| { - let empty = (0..n).filter(|&slot| builder.get(slot).is_none()).count(); - if empty > 0 { - tracing::warn!( - "{label}: {empty} mask slots still empty after resolution (layer {layer_dir:?})" - ); - } - }; - let throttled = obipipeline::throttle(batches, n_workers).map(|t| SourceBatch { - items: t.item, - _permit: t.guard, - }); - - let pipe = obipipeline::make_pipe! { - SibData : SourceBatch => VariantBatch, - | { - move |batch: SourceBatch| -> VariantBatch { - let mut items = Vec::with_capacity(batch.items.len() * 3); - for (order, kmer) in batch.items { - for variant in kmer.central_canonical_neighbors() { - if variant == kmer { - continue; - } - items.push(( - variant.partition(n_parts), - variant, - order, - central_base(variant, k), - )); - } - } - VariantBatch { items, _permit: batch._permit } - } - } : Batch => Variants, - }; - - // ── Group generated variants by destination partition. `cache` - // holds every partition already mmap'd (no more `open()` cost), but - // `mmap` pages are still loaded on demand and can be evicted — a - // lookup is not free just because the file isn't reopened. Grouping - // keeps one partition's pages hot while its whole batch is resolved, - // instead of faulting pages in and out as lookups jump between - // partitions in whatever order the pipeline happens to produce - // them. Each batch's throttle permit drops here, once accumulated. - let mut outgoing: Vec> = - (0..n_parts).map(|_| Vec::new()).collect(); - for vb in pipe.apply(throttled, n_workers, capacity) { - for (dest_partition, variant, source_order, base) in vb.items { - outgoing[dest_partition].push((variant, source_order, base)); - } - } - - check_empty("after_seeding"); - - // ── Resolve each partition's batch against the cache, parallelised - // over roughly equal-sized *chunks*, not over partitions — a plain - // `outgoing.par_iter()` over `n_parts` buckets gives one thread the - // whole of one partition's bucket, however large, and a lot of - // buckets are far from equal: a central-base substitution changes - // the minimiser (and thus routes to a different partition) only - // when the winning minimiser window overlaps the central position. - // For k=31/m=11 that is ~11 of the 21 possible windows — so *most* - // of the remaining ~10/21 send the variant right back to the - // partition already being built. That self-partition bucket ends up - // far larger than any other, so the naive per-partition split - // leaves one thread grinding through it alone long after every - // other partition's (tiny) bucket is done — confirmed by sampling a - // real run: one thread solely in `MphfLayer::find` while the rest - // of the pool sits idle. Splitting each non-empty bucket into - // `chunk_size`-sized pieces first keeps this pass's per-partition - // locality (each chunk is still contiguous within one partition, - // still resolved via `cache.find` grouped by that partition) while - // letting Rayon spread a single oversized bucket across several - // threads instead of pinning it to one. ────────────────────────── - let chunk_size = ((outgoing.iter().map(Vec::len).sum::() / n_workers).max(1)).min(4096); - let work: Vec<(usize, &[(CanonicalKmer, usize, u8)])> = outgoing - .iter() - .enumerate() - .filter(|(_, q)| !q.is_empty()) - .flat_map(|(dest, q)| q.chunks(chunk_size).map(move |c| (dest, c))) - .collect(); - work.par_iter().for_each(|&(dest, chunk)| { - for &(variant, source_order, base) in chunk { - if let Some(layer) = cache.find(dest, variant) { - let bits = if fast_mode { - FamilyMask::layer_bits(base, layer) - } else { - FamilyMask::presence_bits(base) - }; - builder - .atomic_slot(source_order) - .fetch_or(bits, Ordering::Relaxed); - } - } - }); - - check_empty("after_resolution"); - - // ── Write the layer's annex file, indexed by iteration order — a - // second streamed pass over `unitigs.bin` (via `enumerate_kmers`), - // now that every entry's mask is final; never a `Vec` hold of the - // whole layer. The minorant flag is computed here, not by every - // later reader: this is the one place the whole family's *final* - // mask and this entry's own k-mer (already in hand, no extra - // lookup) are both available together. Every consumer that only - // needs to know "is this k-mer the family's minorant" — the common - // case, since a family is tallied once, at its minorant — reads - // the bit straight back instead of re-deriving it (re-scanning - // `unitigs.bin` and re-hashing through the MPHF, the cost - // `is_minorant` was cheap to *compute* but expensive to *get the - // inputs for* every time). - // Every concurrent phase above is done and its `Arc` clone (the - // seeding pass's `seed_builder`) has already been dropped along - // with the now-fully-drained `batches` iterator — this is the only - // remaining strong reference, so reclaiming plain ownership for the - // sequential `set` calls below is guaranteed to succeed. - let mut builder = Arc::try_unwrap(builder).unwrap_or_else(|_| { - panic!("sibling-annex builder still shared after every concurrent phase completed") - }); - for (order, kmer) in mphf.enumerate_kmers() { - let final_mask = builder.get(order).expect("seeded by construction"); - let minorant = is_minorant(kmer, final_mask, k); - builder.set(order, final_mask.with_minorant(minorant)); - } - builder.close()?; - - Ok(n as u64) -} diff --git a/src/obikphylo/src/siblings_old/cache.rs b/src/obikphylo/src/siblings_old/cache.rs deleted file mode 100644 index b0422cbe..00000000 --- a/src/obikphylo/src/siblings_old/cache.rs +++ /dev/null @@ -1,235 +0,0 @@ -use rayon::prelude::*; - -use obikindex::layer::KmerLayer; -use obikseq::CanonicalKmer; -use obisys::progress_bar; - -use obikindex::{KmerIndex, OKIResult}; - -/// Every partition's already-open layers, built **once** for the whole -/// `build_sibling_annex` run and shared (read-only) across every lookup, in -/// every source layer, for the rest of the run — not reopened/re-mmap'd per -/// query, nor per source layer. -/// -/// Confirmed necessary by sampling a real run: routing lookups through -/// `KmerPartition::query_partition_with` (the same batching `obikmer query` -/// uses) still reopens+re-mmaps every target partition's files on every -/// call, and it is called once per destination partition **per source -/// layer** — for an index with many layers this repeats the same -/// `MphfLayer::open`/`Evidence::open`/`PersistentBitMatrix::open` work over -/// and over. Parallelising those calls (see the gather step below) spread -/// the redundant work across more cores but did not reduce it: sampling -/// showed Rayon workers spending their time inside repeated `open()` -/// syscalls, not computation. This cache amortises that cost to once per -/// partition for the entire run, regardless of how many source layers or -/// lookups follow. -/// -/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the -/// format-erased `Count`/`Presence` handle, not a sibling-specific type: -/// this module used to bundle its own `Mat` enum here, duplicating exactly -/// what `Layer` now does one crate down (see -/// `DevDocMD/implementation/partition_layer_cache.md`). Its -/// sibling-specific extension (`iter_minorants_batch`) lives in `iter.rs`. -pub(super) struct PartitionCache { - mats: Vec>, - /// Whether every `FamilyMask` field in this index's sibling annexes can - /// be trusted as a real layer index (`n_layers <= 7`, the same decision - /// `build_layer_sibling_annex` makes when writing them) — computed once - /// here, from `PartitionMeta::n_layers` (not `mats[p].len()`, which can - /// undercount if a layer failed to open), reused by both the annex - /// writer (`build.rs`) and reader (`family_scan.rs`) so the two never - /// disagree. `n_layers` is guaranteed identical across every partition - /// (a merge adds one layer to all of them at once), so the first - /// non-empty partition's count speaks for the whole index. - fast_mode: bool, -} - -impl PartitionCache { - pub(super) fn build(index: &KmerIndex, n_parts: usize, with_counts: bool) -> OKIResult { - let pb = progress_bar("open_partitions", n_parts as u64, "partitions"); - let built: Vec<(Vec, usize)> = (0..n_parts) - .into_par_iter() - .map(|part| -> OKIResult<(Vec, usize)> { - let index_dir = index.index_dir(part); - if !index_dir.exists() { - pb.inc(1); - return Ok((Vec::new(), 0)); - } - let meta = index.partition_meta(part)?; - let mut mats = Vec::with_capacity(meta.n_layers); - for l in 0..meta.n_layers { - let Ok(mat) = - KmerLayer::open(&index.layer_dir(part, l), &meta.mode, with_counts) - else { - continue; - }; - mats.push(mat); - } - pb.inc(1); - Ok((mats, meta.n_layers)) - }) - .collect::>>()?; - pb.finish_and_clear(); - - let n_layers = built.iter().map(|&(_, n)| n).find(|&n| n > 0).unwrap_or(0); - let fast_mode = n_layers <= 7; - if !fast_mode { - tracing::warn!( - "index has {n_layers} layers (>7) — sibling-annex fast layer lookup disabled; \ - consider compacting this index" - ); - } - - let mats = built.into_iter().map(|(m, _)| m).collect(); - Ok(Self { mats, fast_mode }) - } - - /// See the [`fast_mode`](Self::fast_mode) field docs. - pub(super) fn fast_mode(&self) -> bool { - self.fast_mode - } - - /// Existence lookup of `variant` in partition `dest_partition`: tries - /// each of the partition's already-open layers in turn, stopping at the - /// first hit and reporting which layer it was — `find_slot`, not - /// `sub_matrix`/`carries`, no data read needed for a plain existence - /// check. - pub(super) fn find(&self, dest_partition: usize, variant: CanonicalKmer) -> Option { - self.mats - .get(dest_partition)? - .iter() - .enumerate() - .find_map(|(li, mat)| mat.find_slot(variant).map(|_| li)) - } - - /// Resolve many `(variant, family_idx, base)` queries against one - /// partition's matrices at once, calling `on_hit(family_idx, base, g)` - /// for every genome `g` that carries the resolved variant. - /// - /// Genome-major, not query-major: `PersistentBitMatrix`/ - /// `PersistentCompactIntMatrix` are stored one contiguous block per - /// genome (column), slot as the offset within it (see - /// `obicompactvec::bitmatrix::packed::PackedBitMatrix` — each column's - /// own `mmap` region). Resolving query-by-query (`for query { for genome - /// { mat.carries(genome, slot) } }`, this function's predecessor) visits - /// every genome's block once *per query* — for a batch of thousands of - /// queries against ~90 genomes, that is thousands of jumps into each of - /// ~90 widely separated multi-MB regions, in query order, not genome - /// order: the access pattern a column-major layout is least suited to. - /// Grouping first (by layer, since each layer's matrix is a separate - /// column set) and letting `fill_sub_matrix_carries` sort each group by - /// slot internally, then visiting genome by genome, turns that into ~90 - /// mostly-sequential sweeps through one column's own bytes — the - /// layout's fast axis — confirmed by sampling a real run: - /// `PersistentBitMatrix::get` dominated wall-clock time, mostly blocked - /// on page faults, even after every partition/batch locality fix above - /// this in the traversal. - /// `queries`' 4th element (the annex-recorded layer) is ignored here — - /// every layer is probed with evidence regardless, since this is the - /// `!fast_mode` fallback where that field isn't trustworthy (see - /// [`find_presence_batch_fast`](Self::find_presence_batch_fast)). Same - /// tuple shape as the fast path purely so callers don't need two - /// query representations. - pub(super) fn find_presence_batch( - &self, - dest_partition: usize, - queries: &[(CanonicalKmer, usize, u8, u8)], - n_genomes: usize, - mut on_hit: impl FnMut(usize, u8, usize), - ) { - let Some(mats) = self.mats.get(dest_partition) else { - return; - }; - - // First hit wins, same semantics as the old per-query loop (a - // variant present in an earlier layer shadows later ones). - let mut by_layer: Vec> = vec![Vec::new(); mats.len()]; - for &(variant, family_idx, base, _layer) in queries { - for (li, mat) in mats.iter().enumerate() { - if let Some(slot) = mat.find_slot(variant) { - by_layer[li].push((slot, family_idx, base)); - break; - } - } - } - - for (li, hits) in by_layer.into_iter().enumerate() { - if hits.is_empty() { - continue; - } - resolve_layer_hits(&mats[li], &hits, n_genomes, &mut on_hit); - } - } - - /// Same contract as [`find_presence_batch`](Self::find_presence_batch), - /// but only valid when [`fast_mode`](Self::fast_mode) is true for this - /// whole index: every query's 4th element is trusted outright as its - /// real destination layer (recorded once, evidence-checked, at - /// annex-build time by [`find`](Self::find)) — no per-layer probing, a - /// direct `index_batch` (no evidence) straight to that layer instead. - /// Safe for the same reason `build_layer_sibling_annex`'s own - /// iteration-pipeline lookups are: this exact variant's membership in - /// this exact layer was already positively established once: replaying - /// it via `index`/`index_batch` here isn't a fresh, unverified - /// assumption, it's reusing a fact already paid for. - pub(super) fn find_presence_batch_fast( - &self, - dest_partition: usize, - queries: &[(CanonicalKmer, usize, u8, u8)], - n_genomes: usize, - mut on_hit: impl FnMut(usize, u8, usize), - ) { - let Some(mats) = self.mats.get(dest_partition) else { - return; - }; - - let mut by_layer: Vec> = vec![Vec::new(); mats.len()]; - for &(variant, family_idx, base, layer) in queries { - if let Some(bucket) = by_layer.get_mut(layer as usize) { - bucket.push((variant, family_idx, base)); - } - } - - for (li, entries) in by_layer.into_iter().enumerate() { - if entries.is_empty() { - continue; - } - let mat = &mats[li]; - let variants: Vec = - entries.iter().map(|&(variant, _, _)| variant).collect(); - let slots = mat.hash_batch(&variants); - let hits: Vec<(usize, usize, u8)> = slots - .into_iter() - .zip(entries.iter()) - .map(|(slot, &(_, family_idx, base))| (slot, family_idx, base)) - .collect(); - resolve_layer_hits(mat, &hits, n_genomes, &mut on_hit); - } - } -} - -/// Shared tail of both [`PartitionCache::find_presence_batch`] and -/// [`PartitionCache::find_presence_batch_fast`]: given one layer's already- -/// resolved `(slot, family_idx, base)` hits, sweep genome-major (via -/// `fill_sub_matrix_carries`, buffer-reusing, no per-call allocation) and -/// call `on_hit` for every carry. The two callers differ only in *how* -/// `hits` gets its slots (evidence-probed vs. trusted `index_batch`) — this -/// is everything after that. -fn resolve_layer_hits( - mat: &KmerLayer, - hits: &[(usize, usize, u8)], - n_genomes: usize, - on_hit: &mut impl FnMut(usize, u8, usize), -) { - let n_cols = mat.n_cols().min(n_genomes); - let slots: Vec = hits.iter().map(|&(slot, _, _)| slot).collect(); - let mut carries: Vec> = (0..n_cols).map(|_| Vec::new()).collect(); - mat.fill_sub_matrix_carries(&slots, &mut carries); - for (g, col) in carries.iter().enumerate() { - for (&(_, family_idx, base), &carries_it) in hits.iter().zip(col.iter()) { - if carries_it { - on_hit(family_idx, base, g); - } - } - } -} diff --git a/src/obikphylo/src/siblings_old/cardcomp.rs b/src/obikphylo/src/siblings_old/cardcomp.rs deleted file mode 100644 index dbc87e3d..00000000 --- a/src/obikphylo/src/siblings_old/cardcomp.rs +++ /dev/null @@ -1,336 +0,0 @@ -//! Cardinality × composition decomposition of the 16-state transition cost -//! matrix — replaces `sankoff::build_cost_matrix`'s elementary-edge graph -//! and its Floyd-Warshall shortest-path closure, which double-counts -//! multi-hop transitions once IQ-TREE's own matrix exponential composes -//! them again. See `DevDocMD/theory/evolutionary_distances.md`, "`R` via -//! `exp(-cost)` is wrong for a CTMC; cardinality/composition decomposition". -//! -//! Two small, directly-estimated first-order Markov models (diagonals -//! included — "stay the same" is a real, calibrated outcome, not implicit): -//! a 5-state cardinality model (`0..=4` members of a family) and a 4-state -//! base-composition model (`A,C,G,T`, unambiguous/cardinality-1 loci only — -//! composition bias is a substitution phenomenon, only meaningful when -//! cardinality is conserved). Composed per pair of 16 states via a -//! parsimony-style pairing of the non-shared elements (see -//! `pairwise_transition_matrix`'s doc), then symmetrised — Sankoff -//! parsimony's score is only independent of root placement (as required, -//! since TNT/PhyG search *unrooted* trees) if the cost matrix is -//! symmetric, the discrete-parsimony analogue of CTMC reversibility. - -use super::{BasePairTally, CardinalityTally}; - -/// Row-stochastic 5×5 cardinality transition probabilities (`0..=4`), -/// diagonal included ("stay at the same cardinality"), from -/// [`CardinalityTally`]'s pooled, symmetric co-occurrence counts. -pub fn cardinality_transition_probs(tally: &CardinalityTally) -> [[f64; 5]; 5] { - let mut p = [[0.0f64; 5]; 5]; - for a in 0..5 { - let row_sum: u64 = tally.counts[a].iter().sum(); - if row_sum == 0 { - continue; - } - for b in 0..5 { - p[a][b] = tally.counts[a][b] as f64 / row_sum as f64; - } - } - p -} - -/// Row-stochastic 4×4 base-composition transition probabilities -/// (`A,C,G,T`), diagonal included ("stay the same base"), from -/// [`BasePairTally`]'s pooled substitution (off-diagonal) and agreement -/// (`same`, diagonal) counts — unambiguous, cardinality-1 loci only. -pub fn composition_transition_probs(tally: &BasePairTally) -> [[f64; 4]; 4] { - let mut p = [[0.0f64; 4]; 4]; - for a in 0..4 { - let row_sum = tally.same[a] + (0..4).map(|b| tally.counts[a][b]).sum::(); - if row_sum == 0 { - continue; - } - p[a][a] = tally.same[a] as f64 / row_sum as f64; - for b in 0..4 { - if a != b { - p[a][b] = tally.counts[a][b] as f64 / row_sum as f64; - } - } - } - p -} - -/// Enumerate every injective mapping from `small` (indices `0..small.len()`) -/// into `large` (indices `0..large.len()`, `large.len() >= small.len()`), -/// calling `visit` with each full permutation of chosen `large` indices. -/// `small.len() <= 4` always here (at most 4 bases can be lost or gained), -/// so a plain recursive enumeration (at most `4! = 24` calls) is simplest — -/// not worth a dependency for. -fn for_each_injection(small_len: usize, large_len: usize, visit: &mut dyn FnMut(&[usize])) { - let mut chosen = vec![0usize; small_len]; - let mut used = vec![false; large_len]; - fn go(pos: usize, chosen: &mut [usize], used: &mut [bool], visit: &mut dyn FnMut(&[usize])) { - if pos == chosen.len() { - visit(chosen); - return; - } - for l in 0..used.len() { - if used[l] { - continue; - } - used[l] = true; - chosen[pos] = l; - go(pos + 1, chosen, used, visit); - used[l] = false; - } - } - if small_len == 0 { - visit(&[]); - return; - } - go(0, &mut chosen, &mut used, visit); -} - -/// Parsimony-style pairing cost for the non-shared elements of a -/// transition: pair every element of the smaller of `lost`/`gained` with -/// some element of the larger one, choosing the pairing that minimises -/// total `-ln(P_composition)` (maximises likelihood) — the discrete -/// analogue of "prefer a substitution over an independent loss+gain". -/// Leftover elements of the larger side (`|large|-|small|` of them) are -/// *not* charged here — they're pure cardinality change, already priced by -/// `P_cardinality(|A|→|B|)` in the caller. -fn best_pairing_cost(lost: &[u8], gained: &[u8], p_comp: &[[f64; 4]; 4]) -> f64 { - let (small, large) = if lost.len() <= gained.len() { (lost, gained) } else { (gained, lost) }; - if small.is_empty() { - return 0.0; - } - let mut best = f64::INFINITY; - for_each_injection(small.len(), large.len(), &mut |perm| { - let mut cost = 0.0; - for (i, &li) in perm.iter().enumerate() { - let (x, y) = (small[i], large[li]); - let p = p_comp[x as usize][y as usize]; - cost += if p > 0.0 { -p.ln() } else { f64::INFINITY }; - } - if cost < best { - best = cost; - } - }); - best -} - -/// The full 16-state transition cost matrix, replacing -/// `sankoff::build_cost_matrix`. For every pair of states `A`, `B` -/// (bitmasks, bit `0..4` = `A,C,G,T`): `shared = A∩B` contributes -/// `P_composition(x→x)` per shared base (diagonal — "stayed"); the -/// remaining `lost = A\B`, `gained = B\A` are parsimony-paired into -/// substitutions via [`best_pairing_cost`], and whatever's left over after -/// pairing (only possible on one side, since one side is fully consumed) -/// is *pure* cardinality change, priced once via `P_cardinality(|A|→|B|)` -/// — never chained through intermediate states. Row-normalised (`P(A→·)` -/// sums to `1` over all `B`), converted to a cost via `-ln`, then -/// symmetrised (`(cost(A,B)+cost(B,A))/2` — the row-normalised `P` is not -/// symmetric in general, but a Sankoff parsimony cost must be, so the -/// score is independent of where an unrooted tree gets rooted). -/// -/// `free_loss`: drop the `P_cardinality(|A|→|B|)` factor entirely (never -/// added to `log_p`) — the same low/incomplete-coverage argument that -/// justifies recoding whole-family non-detection as `?` (see -/// `DevDocMD/theory/evolutionary_distances.md`, "Locus dropout under -/// incomplete coverage") applies one level down: whether a genome shows 1 -/// vs 2 (etc.) detected members of a *present* family is exactly as -/// vulnerable to sampling failure as whether the family was detected at -/// all. Without this, `∅`-involving transitions are neutralised (via the -/// `?` recoding, bypassing this matrix's row/column 0 entirely) but -/// cardinality changes *between two otherwise-detected, non-empty* states -/// (e.g. `{A} -> {A,C}`) still carried the same calibrated -/// `P_cardinality` penalty as any other gain/loss — inconsistent with -/// `--free-loss`'s own rationale. With the factor dropped, cost is driven -/// only by composition matching (shared-base retention and paired -/// substitutions), never by a state pair's cardinality difference alone. -pub fn pairwise_cost_matrix(p_card: &[[f64; 5]; 5], p_comp: &[[f64; 4]; 4], free_loss: bool) -> [[f64; 16]; 16] { - let mut log_p = [[0.0f64; 16]; 16]; // ln(P), *before* row-normalisation - for a in 0u8..16 { - for b in 0u8..16 { - let shared = a & b; - let lost: Vec = (0..4).filter(|&i| a & (1 << i) != 0 && b & (1 << i) == 0).collect(); - let gained: Vec = (0..4).filter(|&i| b & (1 << i) != 0 && a & (1 << i) == 0).collect(); - - let mut lp = 0.0; // accumulate ln(P), so 0.0 = probability 1 - if !free_loss { - let card_a = a.count_ones() as usize; - let card_b = b.count_ones() as usize; - let p_c = p_card[card_a][card_b]; - lp += if p_c > 0.0 { p_c.ln() } else { f64::NEG_INFINITY }; - } - - for i in 0..4u8 { - if shared & (1 << i) != 0 { - let p = p_comp[i as usize][i as usize]; - lp += if p > 0.0 { p.ln() } else { f64::NEG_INFINITY }; - } - } - lp -= best_pairing_cost(&lost, &gained, p_comp); - - log_p[a as usize][b as usize] = lp; - } - } - - // Row-normalise and convert to cost entirely in log-space - // (log-sum-exp), never exponentiating a raw `log_p` value directly — - // for a transition reachable only through several low-probability - // steps, `log_p.exp()` can underflow to exactly `0.0` (anything below - // roughly `-709` does, in `f64`), silently turning a real, if small, - // probability into a hard `+∞` cost. Observed in practice: a - // calibrated matrix with many such exact-zero entries is numerically - // unstable for IQ-TREE's own likelihood-derivative computation - // ("Numerical underflow for lh-derivative"). `log_sum_exp` never - // exponentiates anything above `0` (every term is shifted by the - // row's own max first), so it stays accurate across the full range - // `f64` can represent, not just what survives a direct `exp()`. - let mut cost = [[0.0f64; 16]; 16]; - for a in 0..16 { - let row_max = log_p[a].iter().cloned().fold(f64::NEG_INFINITY, f64::max); - if row_max == f64::NEG_INFINITY { - // Every transition out of this state has probability 0 in the - // calibration data — genuinely unreachable, not underflow. - cost[a] = [f64::INFINITY; 16]; - continue; - } - let log_sum = row_max + log_p[a].iter().map(|&lp| (lp - row_max).exp()).sum::().ln(); - for b in 0..16 { - // `log_sum - log_p[a][b]` is `+∞` automatically when - // `log_p[a][b] == -∞` (IEEE 754: finite − (−∞) = +∞) — no - // separate branch needed for a genuinely zero-probability - // transition. - cost[a][b] = log_sum - log_p[a][b]; - } - } - - // Symmetrise (see doc comment: parsimony on an unrooted tree requires - // a symmetric cost matrix). - let mut sym = [[0.0f64; 16]; 16]; - for a in 0..16 { - for b in 0..16 { - sym[a][b] = if a == b { 0.0 } else { (cost[a][b] + cost[b][a]) / 2.0 }; - } - } - sym -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cardinality_probs_row_normalised() { - let mut counts = [[0u64; 5]; 5]; - counts[0][0] = 10; - counts[0][1] = 5; - counts[1][0] = 5; - counts[1][1] = 2; - let tally = CardinalityTally { counts }; - let p = cardinality_transition_probs(&tally); - assert!((p[0].iter().sum::() - 1.0).abs() < 1e-12); - assert!((p[1].iter().sum::() - 1.0).abs() < 1e-12); - assert!((p[0][0] - 10.0 / 15.0).abs() < 1e-12); - } - - #[test] - fn composition_probs_include_diagonal() { - let mut tally = BasePairTally { counts: [[0u64; 4]; 4], same: [0u64; 4] }; - tally.same[0] = 90; // A stays A 90 times - tally.counts[0][1] = 10; // A -> C 10 times - tally.counts[1][0] = 10; - let p = composition_transition_probs(&tally); - assert!((p[0][0] - 0.9).abs() < 1e-12); - assert!((p[0][1] - 0.1).abs() < 1e-12); - assert!((p[0].iter().sum::() - 1.0).abs() < 1e-12); - } - - #[test] - fn pairwise_cost_matrix_is_symmetric_and_zero_diagonal() { - // A plausible-looking pair of models (not calibrated from real - // data — just needs every state reachable with nonzero probability - // for this structural test). - let p_card = [ - [0.5, 0.3, 0.15, 0.04, 0.01], - [0.3, 0.4, 0.2, 0.08, 0.02], - [0.15, 0.2, 0.4, 0.2, 0.05], - [0.04, 0.08, 0.2, 0.5, 0.18], - [0.01, 0.02, 0.05, 0.18, 0.74], - ]; - let p_comp = [ - [0.7, 0.1, 0.15, 0.05], - [0.1, 0.7, 0.05, 0.15], - [0.15, 0.05, 0.7, 0.1], - [0.05, 0.15, 0.1, 0.7], - ]; - let cost = pairwise_cost_matrix(&p_card, &p_comp, false); - for a in 0..16 { - assert_eq!(cost[a][a], 0.0); - for b in 0..16 { - assert!((cost[a][b] - cost[b][a]).abs() < 1e-9, "cost[{a}][{b}]={} cost[{b}][{a}]={}", cost[a][b], cost[b][a]); - } - } - } - - #[test] - fn free_loss_ignores_cardinality_transition_probs() { - // A skewed cardinality model (cardinality change made artificially - // expensive) must have zero effect on the cost matrix once - // `free_loss` is set — the whole point of the flag. - let p_card_uniform = [[0.2f64; 5]; 5]; - let p_card_skewed = [ - [0.96, 0.01, 0.01, 0.01, 0.01], - [0.01, 0.96, 0.01, 0.01, 0.01], - [0.01, 0.01, 0.96, 0.01, 0.01], - [0.01, 0.01, 0.01, 0.96, 0.01], - [0.01, 0.01, 0.01, 0.01, 0.96], - ]; - let p_comp = [ - [0.7, 0.1, 0.15, 0.05], - [0.1, 0.7, 0.05, 0.15], - [0.15, 0.05, 0.7, 0.1], - [0.05, 0.15, 0.1, 0.7], - ]; - let cost_uniform = pairwise_cost_matrix(&p_card_uniform, &p_comp, true); - let cost_skewed = pairwise_cost_matrix(&p_card_skewed, &p_comp, true); - for a in 0..16 { - for b in 0..16 { - assert!( - (cost_uniform[a][b] - cost_skewed[a][b]).abs() < 1e-9, - "cost[{a}][{b}] differs between cardinality models under free_loss: {} vs {}", - cost_uniform[a][b], cost_skewed[a][b], - ); - } - } - } - - #[test] - fn underflow_prone_transition_gets_finite_cost_not_infinite() { - // Every off-diagonal composition probability is tiny (1e-200) but - // not exactly zero — small enough that the naive `log_p.exp()` - // (exponentiate before row-normalising) hard-underflows to `0.0` - // in `f64` well before the probability is truly zero, silently - // turning a real, if minuscule, probability into `+∞` cost. The - // log-sum-exp normalisation in `pairwise_cost_matrix` must keep - // this finite instead. - let p_card = [[0.2f64; 5]; 5]; - let tiny = 1e-200; - let mut p_comp = [[tiny; 4]; 4]; - for i in 0..4 { - p_comp[i][i] = 1.0 - 3.0 * tiny; - } - - let cost = pairwise_cost_matrix(&p_card, &p_comp, false); - - // {A,C} -> {G,T}: no shared bases, both members substituted — the - // shape most prone to underflow (several tiny-probability factors - // multiplied together). - let a = 0b0011u8; // A, C - let b = 0b1100u8; // G, T - assert!( - cost[a as usize][b as usize].is_finite(), - "cost must stay finite for a merely tiny (not exactly zero) probability, got {}", - cost[a as usize][b as usize] - ); - } -} diff --git a/src/obikphylo/src/siblings_old/cardinality.rs b/src/obikphylo/src/siblings_old/cardinality.rs deleted file mode 100644 index bc16372a..00000000 --- a/src/obikphylo/src/siblings_old/cardinality.rs +++ /dev/null @@ -1,132 +0,0 @@ -use std::sync::Arc; - -use ndarray::Array2; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::distance::RawSnpDistanceOutput; -use super::family_scan::{Selection, scan_layer_families}; - -/// See [`KmerIndex::cardinality_tally`]. -pub struct CardinalityTally { - /// `counts[a][b] == counts[b][a]` = number of family sites, pooled over - /// included genome pairs, where one genome's family cardinality - /// (popcount of its presence mask, `0..=4`) is `a` and the other's is - /// `b`. Diagonal is real data here (both genomes at the same - /// cardinality), unlike [`super::distance::BasePairTally::counts`]. - pub counts: [[u64; 5]; 5], -} - -/// Adds [`cardinality_tally`](Self::cardinality_tally) to `KmerIndex`. -pub trait CardinalityExt { - /// Cardinality co-occurrence, pooled only over genome pairs whose - /// overall SNP ratio in `raw` is at or below `ratio_ceiling` — same - /// saturation/no-data exclusion discipline as - /// [`base_pair_tally`](Self::base_pair_tally). Unlike - /// `scan_family_pairs` (which resolves each genome to a single form and - /// silently drops any genome carrying more than one member of the - /// family), this needs the *full* per-genome presence mask — a family - /// member count of 2, 3 or 4 is exactly the signal being tallied, not - /// noise to discard — so it re-implements the traversal rather than - /// reusing that helper. - /// - /// Restricted to variable families (`family_size() >= 2`), matching - /// `snp_pseudo_alignment`'s own scope — briefly removed, then - /// reinstated: without it, the diagonal is dominated by genome-wide - /// invariant background (family_size()<2 loci vastly outnumber the - /// ones that ever vary anywhere), which is inconsistent with the - /// `+ASC`-corrected alignment this matrix is ultimately used with — - /// `+ASC` exists specifically because the likelihood only ever sees - /// variable sites, so a rate model calibrated mostly from invariant - /// background sites doesn't describe the population it's applied to. - /// Verified empirically: removing the filter measurably worsened a - /// real IQ-TREE run (log-likelihood dropped, `NNI search needs - /// unusual large number of steps to converge` warnings appeared) — see - /// `DevDocMD/theory/evolutionary_distances.md` for the full account. - /// [`base_pair_tally`](Self::base_pair_tally)'s own diagonal (`same`) - /// gets the matching restriction via `scan_family_pairs`'s new - /// `variable` flag, rather than a `family_size()` check of its own (it - /// doesn't have direct access to the family's mask). - fn cardinality_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult; -} - -impl CardinalityExt for KmerIndex { - fn cardinality_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| { - if i == j { - return false; - } - let snp = raw.snp[[i, j]]; - let total = snp + raw.shared[[i, j]]; - total > 0 && (snp as f64 / total as f64) <= ratio_ceiling - }); - - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - - let pb = progress_bar("cardinality_tally", layer_dirs.len() as u64, "layers"); - // One layer at a time — see `snp_pseudo_alignment`'s comment for why - // `par_iter()` over layers would defeat `scan_layer_families`'s - // partition-grouped locality. Tallied straight into `total` as each - // family comes back — no per-layer buffer. - let mut total = [[0u64; 5]; 5]; - for layer_dir in &layer_dirs { - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::All, - |_family_idx, mask, genome_mask| { - if mask.family_size() < 2 { - // Fully invariant family (never varies anywhere in - // the index) — genome-wide background, not - // SNP-adjacent signal; would otherwise swamp the - // diagonal (`c=1/c=1` etc.), which needs to reflect - // the same variable-families-only population the - // `+ASC`-corrected alignment/likelihood actually - // models. See `base_pair_tally`'s `variable` gate - // on its own `same` diagonal for the matching fix. - return; - } - - for i in 0..n_genomes { - let card_i = genome_mask[i].count_ones() as usize; - for j in (i + 1)..n_genomes { - if !included[[i, j]] { - continue; - } - let card_j = genome_mask[j].count_ones() as usize; - total[card_i][card_j] += 1; - if card_i != card_j { - total[card_j][card_i] += 1; - } - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(CardinalityTally { counts: total }) - } -} diff --git a/src/obikphylo/src/siblings_old/distance.rs b/src/obikphylo/src/siblings_old/distance.rs deleted file mode 100644 index abd92012..00000000 --- a/src/obikphylo/src/siblings_old/distance.rs +++ /dev/null @@ -1,244 +0,0 @@ -use std::sync::Arc; - -use ndarray::Array2; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::family_scan::{Selection, scan_layer_families}; - -/// Raw p-distance restricted to loci that are single-copy in **both** -/// genomes of a pair — the "stringent / paralogy-aware" locus eligibility -/// rule (`DevDocMD/theory/evolutionary_distances.md`, "Locus eligibility"), -/// without the JC/K2P/LogDet finalisation math: `snp[i,j]` + `shared[i,j]` -/// is the number of eligible loci, `snp[i,j] / (snp[i,j] + shared[i,j])` is -/// `p_hat`. A quick, self-contained way to sanity-check the estimator -/// against a real index before the full `SnpTally` design is built. -/// -/// A locus (family, tallied once at its minorant) is eligible for pair -/// `(i, j)` iff genome `i` carries exactly one of the family's observed -/// forms **and** genome `j` carries exactly one (possibly a different one) -/// — presence-only: a genome carrying the same form twice (a same-allele -/// duplicate) is indistinguishable from carrying it once when only a -/// presence matrix is available, so such cases are not excluded here even -/// when a count index exists. See "Locus eligibility", stringent rule, for -/// why this matters and how a count index would close the gap — left as a -/// follow-up, not applied here. -pub struct RawSnpDistanceOutput { - /// n×n count of eligible loci where the two genomes' single forms differ. - pub snp: Array2, - /// n×n count of eligible loci where the two genomes' single forms agree. - pub shared: Array2, -} - -/// Shared traversal behind [`DistanceExt::raw_snp_distance`] and -/// [`DistanceExt::base_pair_tally`]: for every family (tallied once, at its -/// minorant) of every layer of the already-built sibling annex, resolves -/// each genome's single observed form (`None` if absent or -/// ambiguous/multi-copy), then calls `on_pair(acc, i, j, bi, bj, variable)` -/// for every genome pair `(i, j)` where both are unambiguous and -/// single-copy (`bi == bj` means shared at that locus, `bi != bj` means a -/// SNP). `variable` is the family's own `family_size() >= 2` (true if more -/// than one member is observed *anywhere* in the family, i.e. it isn't -/// fully invariant across the whole index) — `raw_snp_distance` ignores it -/// (a fully-invariant family is still legitimately "shared"), but callers -/// whose diagonal should only reflect genuine SNP-adjacent agreement, not -/// the genome-wide invariant background, need it (see -/// [`DistanceExt::base_pair_tally`]'s `same` field). Layers are processed -/// one at a time, not in parallel — see `snp_pseudo_alignment`'s comment -/// for why `par_iter()` over layers would defeat `scan_layer_families`'s -/// partition-grouped locality; each layer gets its own accumulator from -/// `zero()`, combined pairwise via `combine`. A free function, not a -/// `KmerIndex` method — called only from this file's `DistanceExt` impl. -fn scan_family_pairs( - index: &KmerIndex, - label: &str, - zero: impl Fn() -> Acc + Sync, - on_pair: F, - combine: C, -) -> OKIResult -where - Acc: Send, - F: Fn(&mut Acc, usize, usize, u8, u8, bool) + Sync, - C: Fn(Acc, Acc) -> Acc, -{ - let n_parts = index.n_partitions(); - let n_genomes = index.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = index.meta().config.with_counts; - let k = index.kmer_size(); - let cache = Arc::new(PartitionCache::build(index, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(index)?; - - let pb = progress_bar(label, layer_dirs.len() as u64, "layers"); - // One layer at a time — see `snp_pseudo_alignment`'s comment for why - // `par_iter()` over layers would defeat `scan_layer_families`'s - // partition-grouped locality. - let mut total = zero(); - for layer_dir in &layer_dirs { - let mut acc = zero(); - - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::All, - |_family_idx, mask, genome_mask| { - let variable = mask.family_size() >= 2; - - // Per genome: which single form (if exactly one) it - // carries — `None` (a `popcount != 1` mask) once a - // second form is seen, ambiguous/not single-copy, - // ineligible for either side of a pair. - let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - - for i in 0..n_genomes { - let Some(bi) = single_form(i) else { continue }; - for j in (i + 1)..n_genomes { - let Some(bj) = single_form(j) else { continue }; - on_pair(&mut acc, i, j, bi, bj, variable); - } - } - }, - )?; - - total = combine(total, acc); - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(total) -} - -/// Adds [`raw_snp_distance`](Self::raw_snp_distance) and -/// [`base_pair_tally`](Self::base_pair_tally) to `KmerIndex`. -pub trait DistanceExt { - /// Compute [`RawSnpDistanceOutput`] from an already-built sibling annex - /// (run [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) first). - fn raw_snp_distance(&self) -> OKIResult; - - /// Symmetric 6-category base-pair substitution tally (AC, AG, AT, CG, - /// CT, GT — indexed `0=A,1=C,2=G,3=T`), pooled only over genome pairs - /// whose overall SNP ratio in `raw` is at or below `ratio_ceiling` — - /// same saturation-exclusion discipline as - /// [`cardinality_tally`](super::cardinality::CardinalityExt::cardinality_tally), - /// for the same reason: a saturated pair's observed base-pair mix - /// trends toward neutral base composition, not the true point-mutation - /// spectrum. - /// - /// A second full pass over the annex, sharing - /// [`raw_snp_distance`](Self::raw_snp_distance)'s traversal (guided by - /// it, not a blind re-scan) — needed because `raw_snp_distance` only - /// keeps aggregate SNP/shared counts per genome pair, not which bases - /// were actually involved at each locus, and the ratio-ceiling filter - /// can only be evaluated once the aggregate counts are known. - fn base_pair_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult; -} - -impl DistanceExt for KmerIndex { - fn raw_snp_distance(&self) -> OKIResult { - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let (snp, shared) = scan_family_pairs( - self, - "raw_snp_distance", - || { - ( - Array2::::zeros((n_genomes, n_genomes)), - Array2::::zeros((n_genomes, n_genomes)), - ) - }, - |(snp, shared), i, j, bi, bj, _variable| { - if bi == bj { - shared[[i, j]] += 1; - shared[[j, i]] += 1; - } else { - snp[[i, j]] += 1; - snp[[j, i]] += 1; - } - }, - |(mut snp, mut shared), (s, sh)| { - snp += &s; - shared += &sh; - (snp, shared) - }, - )?; - Ok(RawSnpDistanceOutput { snp, shared }) - } - - fn base_pair_tally( - &self, - raw: &RawSnpDistanceOutput, - ratio_ceiling: f64, - ) -> OKIResult { - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| { - if i == j { - return false; - } - let snp = raw.snp[[i, j]]; - let total = snp + raw.shared[[i, j]]; - total > 0 && (snp as f64 / total as f64) <= ratio_ceiling - }); - - let (counts, same) = scan_family_pairs( - self, - "base_pair_tally", - || ([[0u64; 4]; 4], [0u64; 4]), - |(counts, same), i, j, bi, bj, variable| { - if !included[[i, j]] { - return; - } - if bi != bj { - counts[bi as usize][bj as usize] += 1; - counts[bj as usize][bi as usize] += 1; - } else if variable { - // Only count "stayed the same" from families that vary - // *somewhere* in the index — a fully invariant family - // (never varies anywhere) isn't a SNP-adjacent - // agreement, it's genome-wide background, and would - // otherwise swamp the diagonal (see - // `DevDocMD/theory/evolutionary_distances.md`, the - // ascertainment-bias regression this was reverting). - same[bi as usize] += 1; - } - }, - |(mut counts, mut same), (partial_counts, partial_same)| { - for a in 0..4 { - same[a] += partial_same[a]; - for b in 0..4 { - counts[a][b] += partial_counts[a][b]; - } - } - (counts, same) - }, - )?; - Ok(BasePairTally { counts, same }) - } -} - -/// See [`DistanceExt::base_pair_tally`]. -pub struct BasePairTally { - /// `counts[a][b] == counts[b][a]` = number of eligible loci, pooled - /// over included genome pairs, where the two genomes' single forms are - /// `a` and `b` (0=A, 1=C, 2=G, 3=T). Diagonal always `0` — an `a == b` - /// locus is counted in `same`, not here. - pub counts: [[u64; 4]; 4], - /// `same[a]` = number of eligible loci, pooled over included genome - /// pairs, where both genomes' single forms are `a` — the diagonal - /// `counts` omits, needed to build a proper row-stochastic composition - /// probability matrix (the "stay the same base" entries), not just the - /// substitution-cost off-diagonal. - pub same: [u64; 4], -} diff --git a/src/obikphylo/src/siblings_old/entropy.rs b/src/obikphylo/src/siblings_old/entropy.rs deleted file mode 100644 index 5bead80c..00000000 --- a/src/obikphylo/src/siblings_old/entropy.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! `--shannon` — per-family Shannon entropy over the 15 non-empty subsets -//! of `{A,C,G,T}`. See `DevDocMD/architecture/siblings.md`, "Entropy -//! definition — settled 2026-08-15" and "`--subsample`/`--shannon`", for -//! the design discussion this implements: the project's calibrated 16-state -//! Sankoff cost matrix (`cardcomp::pairwise_cost_matrix`) treats every -//! observed combination of bases as its own first-class state, so entropy -//! over that same 15-symbol space (state `0`/`∅` excluded — a family is -//! only informative among the genomes where it is actually observed) is the -//! consistent informativeness proxy for this project, not a 4-symbol -//! reduction. - -use std::io::{BufWriter, Write}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::cache::PartitionCache; -use super::entropy_annex::{ENTROPY_ANNEX_FILE_NAME, EntropyAnnexBuilder}; -use super::family_scan::{Selection, scan_layer_families}; -use super::subsample::EntropyBias; - -/// Shannon entropy (bits) of one family's per-genome states, over the 15 -/// non-empty subsets of `{A,C,G,T}` actually observed among the genomes -/// carrying it (`genome_mask[g] != 0`) — genomes where the family is absent -/// are excluded from the denominator, not scored as a 16th state. `None` -/// only if the family is absent from every genome (shouldn't happen for a -/// real minorant, guarded against rather than dividing by zero). Second -/// value: the denominator (`n_genomes_present`) used, for traceability. -pub(super) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> { - let mut freq = [0u32; 16]; // index 0 (∅) never incremented, kept for direct `m as usize` indexing - let mut present = 0u32; - for &m in genome_mask { - if m != 0 { - freq[m as usize] += 1; - present += 1; - } - } - if present == 0 { - return None; - } - let mut h = 0.0f64; - for &c in &freq[1..] { - if c == 0 { - continue; - } - let p = f64::from(c) / f64::from(present); - h -= p * p.log2(); - } - Some((h, present as usize)) -} - -/// Shannon entropy (bits) of one family's per-genome states, reduced to the -/// 4 plain nucleotide symbols instead of the 15-state space above — kept -/// alongside it (not instead of it) purely to measure how much the two -/// diverge on real data, per the discussion in -/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`". A genome -/// carrying several bases at once (`genome_mask[g]` with more than one bit -/// set) contributes to *each* of those bases' counts — counted once per -/// base present, not split fractionally, and not folded into a combined -/// state the way [`family_entropy`] does: a genome polymorphic for the -/// family is, by construction, present at more than one base, so it simply -/// counts more than once here. The denominator is therefore the total -/// number of base-occurrences summed over every present genome, not the -/// genome count — the two coincide only when no genome carries more than -/// one base. `None` under the same condition as [`family_entropy`]. -pub(super) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> { - let mut freq = [0u32; 4]; - let mut total = 0u32; - for &m in genome_mask { - for (base, count) in freq.iter_mut().enumerate() { - if m & (1 << base) != 0 { - *count += 1; - total += 1; - } - } - } - if total == 0 { - return None; - } - let mut h = 0.0f64; - for &c in &freq { - if c == 0 { - continue; - } - let p = f64::from(c) / f64::from(total); - h -= p * p.log2(); - } - Some((h, total as usize)) -} - -/// Adds [`shannon_entropy_csv`](Self::shannon_entropy_csv) to `KmerIndex`. -pub trait ShannonEntropyExt { - /// Writes a CSV - /// (`layer,family_idx,entropy15,entropy4,family_size,n_genomes_present`) - /// of per-family Shannon entropy — both [`family_entropy`] (15 non-empty - /// states, the project's settled definition) and [`family_entropy_4`] - /// (plain 4-symbol reduction, kept alongside for comparison) — one row - /// per non-monomorphic minorant visited, from an already-built sibling - /// annex (run - /// [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) - /// first). `layer` is this layer's ordinal among every layer carrying an - /// annex (not a partition/layer pair — an opaque but stable id, unique - /// together with `family_idx`); `family_idx` is the family's - /// iteration-order index within that layer, the same numbering - /// `--subsample`'s selection is drawn from. - /// - /// `subsample`/`entropy_bias`: same meaning as - /// [`SnpAlignmentExt::snp_pseudo_alignment`](super::SnpAlignmentExt::snp_pseudo_alignment)'s - /// — `subsample = None`, `entropy_bias = None` streams every - /// non-monomorphic minorant of the whole index (a time cost, not a - /// memory one: entropy is computed and written per-family as soon as it - /// resolves); `subsample = Some(n)` bounds the index's cross-partition - /// resolution work to ~`n` sampled families; `entropy_bias = Some(_)` - /// weights that sample by its own entropy (see - /// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection") — - /// useful here specifically to see the biased sample's own entropy - /// distribution, not just to speed up a distance computation. - fn shannon_entropy_csv( - &self, - path: &Path, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult<()>; -} - -impl ShannonEntropyExt for KmerIndex { - fn shannon_entropy_csv( - &self, - path: &Path, - subsample: Option, - entropy_bias: Option, - ) -> OKIResult<()> { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - let selections = - super::subsample::compute_selections(self, &layer_dirs, subsample, entropy_bias)?; - - let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?); - writeln!( - f, - "layer,family_idx,entropy15,entropy4,family_size,n_genomes_present" - ) - .map_err(OKIError::Io)?; - - let pb = progress_bar("shannon_entropy", layer_dirs.len() as u64, "layers"); - for (layer_ord, (layer_dir, layer_selection)) in - layer_dirs.iter().zip(selections.iter()).enumerate() - { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - let mut write_err = None; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |family_idx, mask, genome_mask| { - if write_err.is_some() { - return; - } - if mask.family_size() < 2 { - return; // monomorphic family — entropy trivially 0, no signal, skip - } - let Some((h15, n_present)) = family_entropy(genome_mask) else { - return; - }; - let h4 = family_entropy_4(genome_mask).map_or(0.0, |(h, _)| h); - if let Err(e) = writeln!( - f, - "{layer_ord},{family_idx},{h15:.6},{h4:.6},{},{n_present}", - mask.family_size() - ) { - write_err = Some(e); - } - }, - )?; - if let Some(e) = write_err { - return Err(OKIError::Io(e)); - } - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(()) - } -} - -/// Builds the entropy annex (`entropy_annex.rs`) for every layer in -/// `layer_dirs` that doesn't already have one — a no-op, cheap existence -/// check per layer, once every layer's annex file exists. Called from -/// [`super::subsample::compute_selections`] whenever entropy-biased -/// selection is requested; see `DevDocMD/architecture/siblings.md`, -/// "Entropy-biased selection", for why this is the one place that pays a -/// scan (every non-monomorphic minorant's `genome_mask` must be resolved -/// to know its entropy) — every later read of the resulting file is a -/// plain positional mmap lookup, not a re-scan. -/// -/// Bounded to non-monomorphic minorants only -/// ([`super::subsample::non_monomorphic_selection_layer`], a cheap -/// annex-only pass — no cross-partition resolution), not -/// `Selection::All`: monomorphism (`family_size() < 2`) is already known -/// from the annex bits alone, so there is no reason to pay -/// `scan_layer_families`'s expensive per-genome resolution for the ~98% -/// of minorants that are monomorphic only to discard the result. -pub(super) fn ensure_entropy_annexes(index: &KmerIndex, layer_dirs: &[PathBuf]) -> OKIResult<()> { - let missing: Vec = layer_dirs - .iter() - .enumerate() - .filter(|(_, dir)| !dir.join(ENTROPY_ANNEX_FILE_NAME).exists()) - .map(|(i, _)| i) - .collect(); - if missing.is_empty() { - return Ok(()); - } - - let n_parts = index.n_partitions(); - let n_genomes = index.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = index.meta().config.with_counts; - let k = index.kmer_size(); - let cache = Arc::new(PartitionCache::build(index, n_parts, with_counts)?); - let minorant_counts = super::subsample::minorant_counts(layer_dirs)?; - - let pb = progress_bar("entropy_annex_build", missing.len() as u64, "layers"); - for &li in &missing { - let layer_dir = &layer_dirs[li]; - let mut builder = EntropyAnnexBuilder::new( - minorant_counts[li] as usize, - &layer_dir.join(ENTROPY_ANNEX_FILE_NAME), - ) - .map_err(OKIError::Io)?; - let eligible = super::subsample::non_monomorphic_selection_layer(layer_dir)?; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::Some(&eligible), - |family_idx, mask, genome_mask| { - debug_assert!( - mask.family_size() >= 2, - "Selection::Some(eligible) must only visit non-monomorphic minorants" - ); - if let Some((h15, _)) = family_entropy(genome_mask) { - builder.set(family_idx, h15 as f32); - } - }, - )?; - builder.close().map_err(OKIError::Io)?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(()) -} diff --git a/src/obikphylo/src/siblings_old/entropy_annex.rs b/src/obikphylo/src/siblings_old/entropy_annex.rs deleted file mode 100644 index 1ac2457a..00000000 --- a/src/obikphylo/src/siblings_old/entropy_annex.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! Persisted, per-layer Shannon entropy (`entropy15`, see `entropy.rs`), -//! one value per **minorant family**, indexed by `family_idx` — the same -//! numbering `scan_layer_families`/`Selection`/`subsample::reservoir_sample_layer` -//! already use (every minorant of the layer, monomorphic ones included, -//! counted in iteration order; see `subsample.rs`'s module docs for why -//! `family_idx` and the raw annex slot are kept as two different counters). -//! -//! Exists so entropy-biased selection (`--entropy`/`--entropy-sd`) can -//! weight candidates without re-resolving each one's `genome_mask` (the -//! expensive cross-partition step) on every run — see -//! `DevDocMD/architecture/siblings.md`, "Entropy-biased selection": entropy is -//! only knowable once presence is resolved, so *building* this file still -//! costs a full, unsampled scan the first time (`ensure_entropy_annexes` in -//! `entropy.rs`), but every subsequent read is a plain positional mmap -//! lookup, restoring `Selection::Some`'s usual "skip resolving excluded -//! families entirely" speedup for every run after the first. -//! -//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat -//! per-entry data, read-only after build) but is not slot-indexed the way -//! `SiblingAnnex` is: `SiblingAnnex` is indexed by every k-mer of the layer -//! (needed because any slot might turn out to be a minorant), while this -//! file is built *from* an already-open `SiblingAnnex`, so it only ever -//! needs to be as long as the layer's minorant count — monomorphic -//! minorants still get an entry (a `-1.0` sentinel, entropy not meaningful -//! for them, never read), keeping the same family_idx numbering every -//! other consumer uses, but non-minorant k-mers (the vast majority of -//! slots) are never represented here at all. - -use std::fs::{File, OpenOptions}; -use std::io; -use std::path::Path; - -use memmap2::{Mmap, MmapMut}; - -const MAGIC: [u8; 4] = *b"PENT"; - -// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows. -const HEADER_SIZE: usize = 16; - -/// Sentinel: monomorphic minorant (no signal, never sampled) or otherwise -/// not applicable. Entropy is always `>= 0.0`, so any negative value is -/// unambiguous — chosen over `NaN` to avoid `NaN`-comparison footguns at -/// every read site. -const SENTINEL: f32 = -1.0; - -pub(super) const ENTROPY_ANNEX_FILE_NAME: &str = "entropy.pent"; - -pub(super) struct EntropyAnnex { - mmap: Mmap, - n: usize, -} - -impl EntropyAnnex { - pub(super) fn open(path: &Path) -> io::Result { - let mmap = unsafe { Mmap::map(&File::open(path)?)? }; - if mmap.len() < HEADER_SIZE { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PENT file too short")); - } - if mmap[0..4] != MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PENT magic")); - } - let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; - if mmap.len() < HEADER_SIZE + n * 4 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PENT file truncated")); - } - Ok(Self { mmap, n }) - } - - pub(super) fn len(&self) -> usize { self.n } - - /// `None` for the sentinel (monomorphic minorant, or not yet computed). - pub(super) fn get(&self, family_idx: usize) -> Option { - let off = HEADER_SIZE + family_idx * 4; - let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap()); - (v >= 0.0).then_some(v) - } -} - -pub(super) struct EntropyAnnexBuilder { - mmap: MmapMut, -} - -impl EntropyAnnexBuilder { - /// Create a new annex of `n` entries (a layer's total minorant count) at - /// `path`, pre-initialised to the sentinel — entries never written - /// (monomorphic minorants) stay the sentinel, not an accidental `0.0` - /// (a real, achievable entropy value). - pub(super) fn new(n: usize, path: &Path) -> io::Result { - let file_size = HEADER_SIZE + n * 4; - let file = OpenOptions::new() - .read(true).write(true).create(true).truncate(true) - .open(path)?; - file.set_len(file_size as u64)?; - let mut mmap = unsafe { MmapMut::map_mut(&file)? }; - mmap[0..4].copy_from_slice(&MAGIC); - mmap[4..8].copy_from_slice(&[0u8; 4]); - mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); - for i in 0..n { - let off = HEADER_SIZE + i * 4; - mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes()); - } - Ok(Self { mmap }) - } - - pub(super) fn set(&mut self, family_idx: usize, value: f32) { - debug_assert!(value >= 0.0, "entropy must be non-negative, got {value}"); - let off = HEADER_SIZE + family_idx * 4; - self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes()); - } - - pub(super) fn close(self) -> io::Result<()> { - self.mmap.flush() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn sentinel_is_negative_and_unset_entries_read_as_none() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let builder = EntropyAnnexBuilder::new(4, &path).unwrap(); - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - for i in 0..4 { - assert_eq!(annex.get(i), None); - } - } - - #[test] - fn roundtrip_including_zero_entropy() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let mut builder = EntropyAnnexBuilder::new(4, &path).unwrap(); - let values = [0.0f32, 1.5, 3.9, 0.0001]; - for (i, &v) in values.iter().enumerate() { - builder.set(i, v); - } - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - for (i, &v) in values.iter().enumerate() { - assert_eq!(annex.get(i), Some(v), "entry {i}"); - } - } - - #[test] - fn mixed_sentinel_and_real_values() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let mut builder = EntropyAnnexBuilder::new(3, &path).unwrap(); - builder.set(1, 2.0); - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - assert_eq!(annex.get(0), None); - assert_eq!(annex.get(1), Some(2.0)); - assert_eq!(annex.get(2), None); - assert_eq!(annex.len(), 3); - } -} diff --git a/src/obikphylo/src/siblings_old/family_scan.rs b/src/obikphylo/src/siblings_old/family_scan.rs deleted file mode 100644 index 59d4d368..00000000 --- a/src/obikphylo/src/siblings_old/family_scan.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! Shared per-layer family traversal, used by every sibling-annex consumer -//! (`snp_pseudo_alignment`, `cardinality_tally`, `scan_family_pairs`, -//! `sibling_annex_stats`) — resolves each minorant family's per-genome -//! base-presence, with the same locality discipline as -//! `build_sibling_annex` (see `cache.rs`/`build.rs`'s docs): cross-partition -//! lookups are grouped by destination partition and resolved in one -//! contiguous sweep per partition, instead of one lookup jumping between -//! partitions in family order. -//! -//! Two concerns, kept on two different mechanisms because they need -//! opposite things: -//! -//! - **Generating** each batch's cross-partition queries is pure CPU (bit -//! tests, one hash per variant) — cheap, and safe to run for several -//! batches at once. It runs on an `obipipeline::throttle` + `make_pipe!` -//! pipeline, the same mechanism `build_layer_sibling_annex` uses, for the -//! same reason: it overlaps this CPU work with the *previous* batch's -//! resolution instead of leaving cores idle between batches. -//! - **Resolving** those queries against `PartitionCache` is I/O (mmap page -//! faults on a real index far larger than RAM) — one batch at a time, -//! `rayon`-parallel *across partitions* (like `build_sibling_annex`'s own -//! `outgoing.par_iter()`), never several batches concurrently. An earlier -//! version resolved each batch fully inside its own pipeline worker, so -//! `n_workers` batches were resolved concurrently — each one spreading its -//! queries thin across every partition of the layer at once. Sampling a -//! real run showed the fix hadn't helped at all: 16 threads "busy" in -//! `find_presence`, but mostly blocked on page faults (~1.7 GB/s pagein), -//! because 16 concurrent sweeps across the same partition space is exactly -//! the scattering the grouping was meant to prevent — just spread over -//! threads instead of over layers this time. Resolving one batch's worth -//! at a time, with each of `rayon`'s threads owning one partition -//! contiguously until that batch is done, keeps only one partition set -//! "hot" at once, restoring the locality the batching is for. -//! -//! `obipipeline` does not guarantee output order, so generated batches carry -//! their own starting family index and are replayed through a small reorder -//! buffer before resolution — bounded by the throttle's own concurrency -//! (`n_workers` batches), not by the layer's size. -//! -//! [`FAMILY_BATCH`] bounds a single batch's memory to `FAMILY_BATCH * -//! genome_count` bytes, and is chosen large enough that a batch still gives -//! each partition a decent number of queries to resolve in one go — too -//! small a batch starves that density regardless of how the resolution step -//! is parallelised. An earlier version materialised the whole layer's -//! families up front before resolving anything: O(layer's family count × -//! genome count) memory, measured pushing a real run into heavy VM -//! compression/swap. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU8, Ordering}; - -use rayon::prelude::*; - -use obikindex::layer::meta::PartitionMeta; -use obikseq::CanonicalKmer; -use obipipeline::{ThrottleGuard, throttle}; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use obikindex::layer::KmerLayer; - -use super::cache::PartitionCache; -use super::helpers::central_base; -use super::iter::{SiblingEntry, SiblingLayerExt}; -use super::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex}; - -/// Families per batch — see the module docs for the memory-vs-per-partition- -/// density trade-off this picks a point on. At ~90 genomes and a few -/// hundred partitions, this keeps a batch's resolution memory in the low -/// tens of MB while still giving each partition on the order of a thousand -/// queries per batch to amortise against. -const FAMILY_BATCH: usize = 65536; - -/// Restricts [`scan_layer_families`] to a subset of a layer's families, -/// identified by their iteration-order index — see -/// `DevDocMD/architecture/siblings.md`, "`--subsample`/`--shannon`", for why: -/// bounding which families actually pay the expensive cross-partition -/// resolution cost is what makes a sampled run cheap on an index far larger -/// than the sample itself. `All` (the default for every pre-existing -/// caller) keeps today's behaviour exactly. -pub(super) enum Selection<'a> { - All, - Some(&'a std::collections::HashSet), -} - -/// Shared by the pipeline worker (via the `Arc`-cloned `Option`, -/// see below — a `Selection` itself doesn't cross the worker thread -/// boundary) and the final replay loop, so both sides agree on membership -/// without duplicating the match. -fn is_selected(selected: &Option>, family_idx: usize) -> bool { - match selected { - None => true, - Some(set) => set.contains(&family_idx), - } -} - -/// Every (partition, layer) directory carrying a sibling annex, checked -/// up front so a missing one is reported before any real work starts. -/// Shared by every sibling-annex consumer's extension-trait impl (`alignment`, -/// `build`, `cardinality`, `distance`, `stats`) — a free function, not a -/// `KmerIndex` method, since it is crate-internal only and `KmerIndex` lives -/// in `obikindex`, a foreign crate from here (orphan rule). -pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult> { - let n_parts = index.n_partitions(); - let mut layer_dirs = Vec::new(); - for part in 0..n_parts { - let index_dir = index.index_dir(part); - if !index_dir.exists() { - continue; - } - let n_layers = index.n_layers(part)?; - for l in 0..n_layers { - let this_layer_dir = index.layer_dir(part, l); - let annex_path = this_layer_dir.join(ANNEX_FILE_NAME); - if !annex_path.exists() { - return Err(OKIError::InvalidInput(format!( - "no sibling annex at {} — run build_sibling_annex first", - annex_path.display() - ))); - } - layer_dirs.push(this_layer_dir); - } - } - Ok(layer_dirs) -} - -/// Read-only state shared (via `Arc`) across every pipeline worker -/// generating this layer's batches — opened once, not per batch. No `cache` -/// here: generation never touches the cross-partition cache, only this -/// layer's own already-open matrix. No separate MPHF/`slot_kmer` either — -/// `mat` (a `TypedLayer`) already bundles the MPHF, and each `SiblingEntry` -/// arrives with its kmer and mask already in hand from `iter_minorants_batch`. -struct LayerCtx { - mat: KmerLayer, - n_parts: usize, - n_genomes: usize, - n_cols: usize, - k: usize, -} - -struct SourceBatch { - start_family_idx: usize, - /// One entry per minorant family in this batch, straight from - /// `iter_minorants_batch` — `order` (iteration-order index, not an MPHF - /// slot; see `DevDocMD/architecture/siblings.md`), `kmer`, and `mask` - /// already carried through, no second annex read. - entries: Vec, - _permit: ThrottleGuard, -} - -/// One batch's generated work: local presence already resolved (straight -/// from this layer's own matrix, no lookup needed), cross-partition queries -/// collected but not yet resolved against the cache. -struct GeneratedBatch { - start_family_idx: usize, - masks: Vec, - /// Flat `slots.len() * n_genomes` — `genome_mask[i * n_genomes + g]`. - genome_mask: Vec, - /// `outgoing[dest_partition]` = `(variant, family_idx_in_batch, base, - /// layer)` — `layer` is the annex-recorded destination layer - /// (`FamilyMask::family_members`'s `layer_value`, `0` if absent), - /// trustworthy only when `PartitionCache::fast_mode()` is true. - outgoing: Vec>, - _permit: ThrottleGuard, -} - -enum FamData { - Batch(SourceBatch), - Generated(GeneratedBatch), -} - -/// Visits every minorant family of one layer, in iteration order, batched -/// [`FAMILY_BATCH`] at a time — see the module docs for why generation and -/// resolution use different concurrency. `on_family` is called once per -/// family, in iteration order, with that family's iteration-order index -/// (the same numbering [`Selection`] indices are drawn from), its own annex -/// mask, and its per-genome base-presence (`genome_mask[g]`: bit `b` set -/// iff genome `g` carries the member whose own canonical central base is -/// `b`), backed by a scratch buffer reused across every call — callers that -/// need to keep data past the call must copy it themselves. -pub(super) fn scan_layer_families( - layer_dir: &Path, - n_parts: usize, - n_genomes: usize, - with_counts: bool, - k: usize, - cache: &Arc, - selection: &Selection, - mut on_family: impl FnMut(usize, FamilyMask, &[u8]), -) -> OKIResult<()> { - let index_dir = layer_dir - .parent() - .expect("layer_dir has a parent index dir"); - let meta = PartitionMeta::load(index_dir)?; - let annex = Arc::new(SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?); - - let mat = KmerLayer::open(layer_dir, &meta.mode, with_counts)?; - let n_cols = mat.n_cols().min(n_genomes); - - let ctx = Arc::new(LayerCtx { - mat, - n_parts, - n_genomes, - n_cols, - k, - }); - - // Streamed straight from `iter_minorants_batch` (zips this layer's own - // `iter_kmers()` with the annex, both in iteration order — never an - // MPHF slot; see `DevDocMD/architecture/siblings.md`) — never collected - // into a `Vec` first: a layer can hold billions of k-mers, so - // materialising every minorant family up front is exactly the memory - // blowup an earlier version of this traversal was rewritten to avoid - // (see the module docs). `.scan()` computes each batch's starting - // family index lazily, mirroring what the eager `chunks()`+running - // `offset` used to do. - let batches = - ctx.mat - .iter_minorants_batch(annex, FAMILY_BATCH) - .scan(0usize, |offset, batch| { - let start = *offset; - *offset += batch.len(); - Some((start, batch)) - }); - - let n_workers = obisys::effective_parallelism(); - let capacity = 4; - let throttled = throttle(batches, n_workers).map(|t| SourceBatch { - start_family_idx: t.item.0, - entries: t.item.1, - _permit: t.guard, - }); - - // `Selection::Some` borrows a `HashSet` whose lifetime doesn't span the - // pipeline's worker threads — cloned once into an `Arc` so every worker - // can share it cheaply instead of requiring `selection` itself to be - // `'static`. `None` means [`Selection::All`], checked with a plain - // `map_or` at each use instead of allocating an always-true set. - let selected: Arc>> = Arc::new(match selection { - Selection::All => None, - Selection::Some(set) => Some((*set).clone()), - }); - - let worker_ctx = Arc::clone(&ctx); - let worker_selected = Arc::clone(&selected); - let pipe = obipipeline::make_pipe! { - FamData : SourceBatch => GeneratedBatch, - | { - move |batch: SourceBatch| -> GeneratedBatch { - let ctx = &worker_ctx; - let n = batch.entries.len(); - let mut masks = Vec::with_capacity(n); - let mut bases = Vec::with_capacity(n); - let mut genome_mask = vec![0u8; n * ctx.n_genomes]; - let mut outgoing: Vec> = (0..ctx.n_parts).map(|_| Vec::new()).collect(); - - // Pass 1: cheap, no matrix access — own base and this - // batch's cross-partition queries. Kmer and mask already in - // hand from `iter_minorants_batch`, no second annex read, - // no slot lookup needed for this pass. `family_members` - // (not a hand-rolled `central_canonical_neighbors` + - // `mask.has` loop) already filters to present members and - // hands back each one's annex-recorded layer alongside. - // Families outside `selected` (sampling only) still get a - // mask/base entry — pass 2 indexes uniformly by `i` — but - // never an outgoing query: that's the expensive part a - // sampled run exists to avoid paying for every family. - for (i, entry) in batch.entries.iter().enumerate() { - let (kmer, mask) = (entry.kmer, entry.mask); - masks.push(mask); - let base = central_base(kmer, ctx.k); - bases.push(base); - let family_idx = batch.start_family_idx + i; - if !is_selected(&worker_selected, family_idx) { - continue; - } - for (member, layer) in mask.family_members(kmer, ctx.k) { - if member == kmer { - continue; // local — resolved below straight from `mat`, no lookup - } - let b = central_base(member, ctx.k); - let dest = member.partition(ctx.n_parts); - outgoing[dest].push((member, i, b, layer.unwrap_or(0))); - } - } - - // Pass 2: genome-major, not family-major — `mat` is stored - // one contiguous block per genome (column), slot as the - // offset within it (see `PartitionCache::find_presence_batch`'s - // docs for the full rationale). `fill_sub_matrix_carries` - // sorts the slots internally for a sequential mmap sweep per - // column, then restores this batch's order — no hand-rolled - // sort/genome-major loop needed here. The presence/count - // matrix is still MPHF-slot-indexed (unlike the annex), so - // each entry's kmer is mapped to its slot via `index_batch` - // — a pure MPHF lookup, no evidence check, since these are - // this layer's own kmers, known members by construction. - let kmers: Vec = batch.entries.iter().map(|e| e.kmer).collect(); - let slots = ctx.mat.hash_batch(&kmers); - let mut carries: Vec> = (0..ctx.n_cols).map(|_| Vec::new()).collect(); - ctx.mat.fill_sub_matrix_carries(&slots, &mut carries); - for (g, col) in carries.iter().enumerate() { - for (i, &carries_it) in col.iter().enumerate() { - if carries_it { - genome_mask[i * ctx.n_genomes + g] |= 1 << bases[i]; - } - } - } - - GeneratedBatch { start_family_idx: batch.start_family_idx, masks, genome_mask, outgoing, _permit: batch._permit } - } - } : Batch => Generated, - }; - - // Batches finish generation in whatever order their worker completes - // them, not submission order — replay in order through a buffer bounded - // by the throttle's own concurrency (`n_workers` batches can be in - // flight at once, so at most that many can be waiting here). - let mut pending: HashMap = HashMap::new(); - let mut next_expected = 0usize; - let mut scratch = vec![0u8; n_genomes]; - for generated in pipe.apply(throttled, n_workers, capacity) { - pending.insert(generated.start_family_idx, generated); - while let Some(batch) = pending.remove(&next_expected) { - let n = batch.masks.len(); - - // ── Resolve this one batch's cross-partition queries — the - // only place that touches `cache`. Parallel across partitions - // (one batch at a time, never several concurrently — see the - // module docs), each thread owning one partition's queries - // contiguously until this batch is done. - let genome_mask: Vec = - batch.genome_mask.into_iter().map(AtomicU8::new).collect(); - let fast_mode = cache.fast_mode(); - batch - .outgoing - .par_iter() - .enumerate() - .filter(|(_, q)| !q.is_empty()) - .for_each(|(dest, queries)| { - let on_hit = |i: usize, base: u8, g: usize| { - genome_mask[i * n_genomes + g].fetch_or(1 << base, Ordering::Relaxed); - }; - if fast_mode { - cache.find_presence_batch_fast(dest, queries, n_genomes, on_hit); - } else { - cache.find_presence_batch(dest, queries, n_genomes, on_hit); - } - }); - - for i in 0..n { - let family_idx = batch.start_family_idx + i; - if !is_selected(&selected, family_idx) { - continue; // not in the sample — never resolved above, nothing to report - } - for (g, dst) in scratch.iter_mut().enumerate() { - *dst = genome_mask[i * n_genomes + g].load(Ordering::Relaxed); - } - on_family(family_idx, batch.masks[i], &scratch); - } - next_expected += n; - } - } - debug_assert!( - pending.is_empty(), - "every generated batch must have been replayed" - ); - - Ok(()) -} diff --git a/src/obikphylo/src/siblings_old/helpers.rs b/src/obikphylo/src/siblings_old/helpers.rs deleted file mode 100644 index b11c7c52..00000000 --- a/src/obikphylo/src/siblings_old/helpers.rs +++ /dev/null @@ -1,23 +0,0 @@ -use obikseq::CanonicalKmer; - -use super::FamilyMask; - -/// Central-position base of a canonical k-mer, in the fixed 0=A/1=C/2=G/3=T -/// encoding — the mask's bit index. `k` must be odd (project invariant). -#[inline] -pub(super) fn central_base(kmer: CanonicalKmer, k: usize) -> u8 { - kmer.nucleotide((k - 1) / 2) -} - -/// Is `kmer` the minorant of its family, given the family's presence mask? -/// Regenerates the family's 4 canonical forms from `kmer` itself (cheap, no -/// lookup — see the design doc's "Definitions" section for why this is -/// always safe: the set of 4 forms is invariant regardless of which member -/// you start from), and compares the raw encodings of whichever are marked -/// present in `mask`. -pub(super) fn is_minorant(kmer: CanonicalKmer, mask: FamilyMask, k: usize) -> bool { - kmer.central_canonical_neighbors().into_iter().all(|other| { - other == kmer || !mask.has(central_base(other, k)) || kmer.raw() <= other.raw() - }) -} - diff --git a/src/obikphylo/src/siblings_old/iter.rs b/src/obikphylo/src/siblings_old/iter.rs deleted file mode 100644 index 8d134639..00000000 --- a/src/obikphylo/src/siblings_old/iter.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! Phylo/sibling-domain iteration over a layer — an extension trait, not a -//! new field on `MphfLayer`/`TypedLayer`: "family"/"minorant" are phylo -//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see -//! `DevDocMD/architecture/siblings.md`). -//! -//! The sibling annex is persisted in the same order as `iter_kmers()` -//! (`build_sibling_annex`, see `build.rs`), so pairing them is a plain zip — -//! no MPHF, no slot, no `kmer_at`. Both sides are already `Send + 'static` -//! (`KmerIter` owns an `Arc` clone; `SiblingAnnex` is -//! mmap-backed and handed in as an `Arc` by the caller), so `SiblingIter` -//! streams straight from disk and can be fed to `obipipeline` batch by -//! batch — never collected whole into memory (see the project's "no full -//! collect" rule). -//! -//! Four iterator types, deliberately mirroring `obikindex::layer`'s own -//! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the -//! minorant-filtered variant of each, since "all siblings" and "one row per -//! family" are both common cases: -//! -//! | | all entries | minorants only | -//! |------------|-------------------|---------------------| -//! | single | [`SiblingIter`] | [`MinorantIter`] | -//! | batch | [`SiblingBatchIter`] | [`MinorantBatchIter`] | -//! -//! No separate "enumerate" variant (unlike `KmerIter`/`enumerate_kmers`): -//! [`SiblingEntry`] already carries `order` for free, since pairing with -//! the annex requires it anyway. - -use std::sync::Arc; - -use obikindex::layer::{KmerIter, LayerData, TypedLayer}; -use obikseq::CanonicalKmer; - -use super::{FamilyMask, SiblingAnnex}; - -/// One layer entry: a k-mer's position in the layer's iteration order (the -/// same index the sibling annex is keyed on — not an MPHF slot), the k-mer -/// itself, and its family mask. -#[derive(Debug, Clone, Copy)] -pub struct SiblingEntry { - pub order: usize, - pub kmer: CanonicalKmer, - pub mask: FamilyMask, -} - -/// Streams `(order, kmer, mask)` triples for one layer, in iteration order. -/// Produced by [`SiblingLayerExt::iter_siblings`]. -pub struct SiblingIter { - kmers: KmerIter, - annex: Arc, - order: usize, -} - -impl Iterator for SiblingIter { - type Item = SiblingEntry; - - fn next(&mut self) -> Option { - loop { - let kmer = self.kmers.next()?; - let order = self.order; - self.order += 1; - // `None` means "not yet computed" (see `SiblingAnnex` module - // docs) — shouldn't happen against a fully-built annex, but - // skip rather than misalign the two streams if it does. - if let Some(mask) = self.annex.get(order) { - return Some(SiblingEntry { order, kmer, mask }); - } - } - } -} - -/// Batches of [`SiblingIter`]'s entries, `batch_size` at a time — the last -/// batch may be shorter. Produced by [`SiblingLayerExt::iter_siblings_batch`]. -pub struct SiblingBatchIter { - inner: SiblingIter, - batch_size: usize, -} - -impl Iterator for SiblingBatchIter { - type Item = Vec; - - fn next(&mut self) -> Option { - collect_batch(&mut self.inner, self.batch_size) - } -} - -/// Like [`SiblingIter`], filtered to the minorant of each family — the -/// common case, since a family is tallied once, at its minorant. Produced -/// by [`SiblingLayerExt::iter_minorants`]. -pub struct MinorantIter { - inner: SiblingIter, -} - -impl Iterator for MinorantIter { - type Item = SiblingEntry; - - fn next(&mut self) -> Option { - self.inner.by_ref().find(|e| e.mask.is_minorant()) - } -} - -/// Batches of [`MinorantIter`]'s entries, `batch_size` at a time — the last -/// batch may be shorter. Produced by -/// [`SiblingLayerExt::iter_minorants_batch`]. -pub struct MinorantBatchIter { - inner: MinorantIter, - batch_size: usize, -} - -impl Iterator for MinorantBatchIter { - type Item = Vec; - - fn next(&mut self) -> Option { - collect_batch(&mut self.inner, self.batch_size) - } -} - -/// Shared by [`SiblingBatchIter`] and [`MinorantBatchIter`] — pull up to -/// `batch_size` items, `None` once the source is exhausted with nothing left. -fn collect_batch(inner: &mut I, batch_size: usize) -> Option> { - let mut batch = Vec::with_capacity(batch_size); - for _ in 0..batch_size { - match inner.next() { - Some(item) => batch.push(item), - None => break, - } - } - if batch.is_empty() { None } else { Some(batch) } -} - -/// Adds phylo/sibling iteration to any `TypedLayer` — the extension that -/// turns a plain layer into a "sibling layer". Generic over `D` -/// (`LayerData`) rather than implemented once per matrix kind: kmer -/// iteration doesn't depend on the data payload, and `TypedLayer` already -/// delegates `iter_kmers`/`index`/`index_batch` to its inner MPHF for every -/// `D` — reusing that instead of going back through a separate `MphfLayer`. -pub trait SiblingLayerExt { - /// Zip this layer's k-mers with their sibling-annex entry, in iteration - /// order. `annex` must have been built from this same layer (its length - /// must match the layer's k-mer count). - fn iter_siblings(&self, annex: Arc) -> SiblingIter; - - /// Like [`iter_siblings`](Self::iter_siblings), yielding `batch_size` - /// entries at a time. - fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter; - - /// Like [`iter_siblings`](Self::iter_siblings), filtered to the - /// minorant of each family — the common case, since a family is - /// tallied once, at its minorant. - fn iter_minorants(&self, annex: Arc) -> MinorantIter; - - /// Like [`iter_minorants`](Self::iter_minorants), yielding `batch_size` - /// minorants at a time. - fn iter_minorants_batch( - &self, - annex: Arc, - batch_size: usize, - ) -> MinorantBatchIter; -} - -impl SiblingLayerExt for TypedLayer { - fn iter_siblings(&self, annex: Arc) -> SiblingIter { - SiblingIter { - kmers: self.iter_kmers(), - annex, - order: 0, - } - } - - fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter { - SiblingBatchIter { - inner: self.iter_siblings(annex), - batch_size, - } - } - - fn iter_minorants(&self, annex: Arc) -> MinorantIter { - MinorantIter { - inner: self.iter_siblings(annex), - } - } - - fn iter_minorants_batch( - &self, - annex: Arc, - batch_size: usize, - ) -> MinorantBatchIter { - MinorantBatchIter { - inner: self.iter_minorants(annex), - batch_size, - } - } -} - -/// Same extension, over `obikindex::layer::Layer` (the format-erased -/// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch -/// only, both arms return the same concrete iterator types (they don't -/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer` -/// itself can't implement this: "family"/"minorant" are phylo concepts, it -/// stays kmer/slot-mapping only (see the module docs above). -impl SiblingLayerExt for obikindex::layer::KmerLayer { - fn iter_siblings(&self, annex: Arc) -> SiblingIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_siblings(annex), - obikindex::layer::KmerLayer::Presence(l) => l.iter_siblings(annex), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_siblings() called on an Empty layer") - } - } - } - - fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_siblings_batch(annex, batch_size), - obikindex::layer::KmerLayer::Presence(l) => l.iter_siblings_batch(annex, batch_size), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_siblings_batch() called on an Empty layer") - } - } - } - - fn iter_minorants(&self, annex: Arc) -> MinorantIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_minorants(annex), - obikindex::layer::KmerLayer::Presence(l) => l.iter_minorants(annex), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_minorants() called on an Empty layer") - } - } - } - - fn iter_minorants_batch( - &self, - annex: Arc, - batch_size: usize, - ) -> MinorantBatchIter { - match self { - obikindex::layer::KmerLayer::Count(l) => l.iter_minorants_batch(annex, batch_size), - obikindex::layer::KmerLayer::Presence(l) => l.iter_minorants_batch(annex, batch_size), - obikindex::layer::KmerLayer::Empty { .. } => { - panic!("iter_minorants_batch() called on an Empty layer") - } - } - } -} diff --git a/src/obikphylo/src/siblings_old/mod.rs b/src/obikphylo/src/siblings_old/mod.rs deleted file mode 100644 index 2a59b795..00000000 --- a/src/obikphylo/src/siblings_old/mod.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Family presence-mask annex construction. -//! -//! See `DevDocMD/theory/evolutionary_distances.md`, "Definitions: family, and -//! the canonical form of a family" and "Step 2b", for the full design -//! discussion this implements. -//! -//! For each distinct k-mer of each layer of the (already built/merged) -//! index, computes a 4-bit presence mask for its "family" (the up to 4 -//! k-mers sharing its flanks, differing only at the central base — -//! well-defined for odd k): bit `b` set iff the family member whose own -//! canonical central base is `b` (0=A, 1=C, 2=G, 3=T) is observed anywhere -//! in the current multi-genome index — a property of the whole index, not -//! of any one genome. Sibling count and minorant are *derived* from the -//! mask by callers, not stored (see `FamilyMask` and -//! [`sibling_annex_stats`](stats::SiblingStatsExt::sibling_annex_stats) -//! below). -//! -//! Per layer, an `obipipeline` batch transform (throttled — see -//! `obipipeline::throttle`) generates a whole batch's central variants at -//! once (`BATCH_SIZE` source k-mers in, that batch's variants out as one -//! pipeline message), interleaved across many in-flight batches by the -//! scheduler's shared worker pool rather than processed on a single -//! thread. The actual cross-partition lookup reuses a `PartitionCache` of -//! every partition's already-open MPHF layers, built once for the whole -//! `build_sibling_annex` run, rather than reopening files per lookup or -//! per source layer. Two earlier, coarser-grained designs were tried and -//! measured (not guessed) to be worse, in order: (1) reopening/re-mmap'ing -//! every target partition's files on every single lookup — fine at toy -//! scale, ~90% system time against a real index; (2) a `Flat` pipeline -//! stage pushing one message per generated *variant* (up to 3 per source -//! k-mer) — cheaper than reopening files, but sampling a real run showed -//! most wall-clock time going into per-message channel send/notify -//! syscalls rather than the lookup itself, because a single k-mer's ≤3 -//! variants is far too fine a granularity to amortise a pipeline's -//! synchronisation cost over. See `DevDocMD/theory/evolutionary_distances.md`, -//! Step 2b, "Mechanism". -//! -//! Submodules, in the order data flows through them: [`cache`] (shared -//! whole-run partition cache), [`helpers`] (small pure functions used -//! throughout), [`build`] (annex construction), [`stats`] (family-size -//! diagnostics), [`distance`] (raw SNP distance + base-pair tally), -//! [`cardinality`] (cardinality co-occurrence), [`alignment`] (SNP-only -//! pseudo-alignment). - -mod alignment; -mod build; -mod cache; -mod cardcomp; -mod cardinality; -mod distance; -mod entropy; -mod entropy_annex; -mod family_scan; -mod helpers; -mod iter; -mod sankoff_bundle; -mod siblingannex; -mod stats; -mod subsample; - -#[cfg(test)] -mod tests; - -pub use alignment::{SnpAlignment, SnpAlignmentExt}; -pub use build::SiblingAnnexBuildExt; -pub use cardcomp::{cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix}; -pub use cardinality::{CardinalityExt, CardinalityTally}; -pub use distance::{BasePairTally, DistanceExt, RawSnpDistanceOutput}; -pub use entropy::ShannonEntropyExt; -pub use iter::{MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt}; -pub use sankoff_bundle::{SankoffBundle, SankoffBundleExt}; -pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; -pub use stats::{SiblingAnnexStats, SiblingStatsExt}; -pub use subsample::EntropyBias; - -pub(super) const ANNEX_FILE_NAME: &str = "siblings.psib"; diff --git a/src/obikphylo/src/siblings_old/sankoff_bundle.rs b/src/obikphylo/src/siblings_old/sankoff_bundle.rs deleted file mode 100644 index 9e969f88..00000000 --- a/src/obikphylo/src/siblings_old/sankoff_bundle.rs +++ /dev/null @@ -1,250 +0,0 @@ -//! Fused entry point for the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` -//! pipeline — see `DevDocMD/architecture/siblings.md`, "`--free-loss`/`--tnt` -//! pipeline: four independent scans, three of them unsampled" and "Wired -//! into `pack` and the sibling-annex build path" for the problem this -//! replaces: `raw_snp_distance`, `base_pair_tally`, `cardinality_tally` and -//! `snp_pseudo_alignment` used to each open their own `PartitionCache` and -//! independently re-scan the whole annex, three of the four ignoring -//! `--subsample` entirely. -//! -//! `base_pair_tally`/`cardinality_tally` both need `raw_snp_distance`'s -//! *complete* aggregate SNP/shared counts before they can derive -//! `included[i,j]` (the `ratio_ceiling` filter) — a genuine sequential -//! dependency, not an accidental one, so this is two passes, not one: -//! -//! - **Pass A**: aggregate SNP/shared counts per genome pair, over the -//! shared selection (computed once, [`super::subsample::compute_selections`] -//! — the *same* selection every other pass below uses, per the user's -//! own requirement: "il faut qu'on sélectionne tout le monde avec la -//! même sélection"). Excluded genomes (`--exclude-genome`) are zeroed -//! here, before `included` is derived from it — the ratio filter must -//! see the post-exclusion counts, not the raw ones. -//! - **Pass B**: base-pair tally, cardinality tally, and the pseudo- -//! alignment, fused into *one* scan over the same selection — the three -//! are mutually independent once `included` is known, so they are -//! accumulated together family-by-family instead of three more separate -//! scans. - -use std::sync::Arc; - -use ndarray::Array2; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::alignment::{SnpAlignment, iupac_code}; -use super::cache::PartitionCache; -use super::cardinality::CardinalityTally; -use super::distance::{BasePairTally, RawSnpDistanceOutput}; -use super::family_scan::{Selection, scan_layer_families, sibling_layer_dirs}; -use super::subsample::{EntropyBias, compute_selections}; - -/// Every output the `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline needs, -/// computed together from one shared, possibly-subsampled/entropy-biased -/// selection — see the module docs. -pub struct SankoffBundle { - pub raw: RawSnpDistanceOutput, - pub base_pair_tally: BasePairTally, - pub cardinality_tally: CardinalityTally, - pub alignment: SnpAlignment, -} - -/// Adds [`sankoff_bundle`](Self::sankoff_bundle) to `KmerIndex`. -pub trait SankoffBundleExt { - /// `subsample`/`entropy_bias`: see - /// [`SnpAlignmentExt::snp_pseudo_alignment`](super::SnpAlignmentExt::snp_pseudo_alignment). - /// `ratio_ceiling`: see - /// [`DistanceExt::base_pair_tally`](super::DistanceExt::base_pair_tally). - /// `exclude_mask[g]`: zero genome `g`'s row/column out of the aggregate - /// SNP/shared counts *before* `included[i,j]` is derived from them — - /// same semantics as `--exclude-genome` had on the old, separate - /// `raw_snp_distance` call. - fn sankoff_bundle( - &self, - subsample: Option, - entropy_bias: Option, - ratio_ceiling: f64, - exclude_mask: &[bool], - ) -> OKIResult; -} - -impl SankoffBundleExt for KmerIndex { - fn sankoff_bundle( - &self, - subsample: Option, - entropy_bias: Option, - ratio_ceiling: f64, - exclude_mask: &[bool], - ) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = sibling_layer_dirs(self)?; - // Computed once — every pass below iterates the exact same - // families, in the exact same layers, per the shared-selection - // requirement this module exists to satisfy. - let selections = compute_selections(self, &layer_dirs, subsample, entropy_bias)?; - - // ── Pass A: aggregate SNP/shared counts ──────────────────────────── - let pb = progress_bar("raw_snp_distance", layer_dirs.len() as u64, "layers"); - let mut snp = Array2::::zeros((n_genomes, n_genomes)); - let mut shared = Array2::::zeros((n_genomes, n_genomes)); - for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |_family_idx, _mask, genome_mask| { - let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - for i in 0..n_genomes { - let Some(bi) = single_form(i) else { continue }; - for j in (i + 1)..n_genomes { - let Some(bj) = single_form(j) else { continue }; - if bi == bj { - shared[[i, j]] += 1; - shared[[j, i]] += 1; - } else { - snp[[i, j]] += 1; - snp[[j, i]] += 1; - } - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - // `--exclude-genome`: zero excluded rows/columns before `included` - // is derived below — same effect the old standalone - // `zero_excluded_pairs` had, applied here since this is now the - // only place `included` gets computed for this pipeline. - for (i, &excluded) in exclude_mask.iter().enumerate() { - if !excluded { - continue; - } - for j in 0..n_genomes { - snp[[i, j]] = 0; - snp[[j, i]] = 0; - shared[[i, j]] = 0; - shared[[j, i]] = 0; - } - } - let raw = RawSnpDistanceOutput { snp, shared }; - - let included = Array2::from_shape_fn((n_genomes, n_genomes), |(i, j)| { - if i == j { - return false; - } - let s = raw.snp[[i, j]]; - let total = s + raw.shared[[i, j]]; - total > 0 && (s as f64 / total as f64) <= ratio_ceiling - }); - - // ── Pass B: base-pair tally + cardinality tally + alignment, fused ── - let pb = progress_bar("sankoff_pass_b", layer_dirs.len() as u64, "layers"); - let mut bp_counts = [[0u64; 4]; 4]; - let mut bp_same = [0u64; 4]; - let mut card_counts = [[0u64; 5]; 5]; - let mut sequences: Vec> = vec![Vec::new(); n_genomes]; - for (layer_dir, layer_selection) in layer_dirs.iter().zip(selections.iter()) { - let selection = match layer_selection { - None => Selection::All, - Some(set) => Selection::Some(set), - }; - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &selection, - |_family_idx, mask, genome_mask| { - let variable = mask.family_size() >= 2; - - // base-pair tally — same pairwise single-form resolution as pass A. - let single_form = |g: usize| -> Option { - let m = genome_mask[g]; - (m.count_ones() == 1).then(|| m.trailing_zeros() as u8) - }; - for i in 0..n_genomes { - let Some(bi) = single_form(i) else { continue }; - for j in (i + 1)..n_genomes { - let Some(bj) = single_form(j) else { continue }; - if !included[[i, j]] { - continue; - } - if bi != bj { - bp_counts[bi as usize][bj as usize] += 1; - bp_counts[bj as usize][bi as usize] += 1; - } else if variable { - bp_same[bi as usize] += 1; - } - } - } - - // cardinality tally — restricted to variable families, see - // `CardinalityExt::cardinality_tally`'s docs for why. - if variable { - for i in 0..n_genomes { - let card_i = genome_mask[i].count_ones() as usize; - for j in (i + 1)..n_genomes { - if !included[[i, j]] { - continue; - } - let card_j = genome_mask[j].count_ones() as usize; - card_counts[card_i][card_j] += 1; - if card_i != card_j { - card_counts[card_j][card_i] += 1; - } - } - } - } - - // pseudo-alignment — same variable-family gate as - // `snp_pseudo_alignment`. Every family the shared selection - // actually visits already satisfies this when the - // selection is `Selection::Some` (only non-monomorphic - // minorants are ever selected), but the explicit check - // still matters under `Selection::All` (no `--subsample`), - // which visits monomorphic minorants too. - if variable { - for (g, &m) in genome_mask.iter().enumerate() { - sequences[g].push(iupac_code(m)); - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(SankoffBundle { - raw, - base_pair_tally: BasePairTally { - counts: bp_counts, - same: bp_same, - }, - cardinality_tally: CardinalityTally { - counts: card_counts, - }, - alignment: SnpAlignment { sequences }, - }) - } -} diff --git a/src/obikphylo/src/siblings_old/siblingannex.rs b/src/obikphylo/src/siblings_old/siblingannex.rs deleted file mode 100644 index 0ec81d5e..00000000 --- a/src/obikphylo/src/siblings_old/siblingannex.rs +++ /dev/null @@ -1,433 +0,0 @@ -//! Family presence-mask annex: a compact, read-only-after-build, per-slot -//! derived value used by the central-position SNP distance estimator (see -//! `DevDocMD/theory/evolutionary_distances.md`, "Step 2b" and "Definitions: -//! family, and the canonical form of a family"). -//! -//! Two bytes are stored per MPHF slot of a partition/layer, packed as four -//! 3-bit fields (one per central base, in the fixed A/C/G/T = 0/1/2/3 -//! encoding) plus a minorant flag: -//! -//! - Each 3-bit field: `0` = the family member whose *own* central base — -//! in its own canonical orientation — is that base is absent from the -//! whole multi-genome index (a property of the whole index, not of any -//! one genome); a non-zero value `v` (`1..=7`) means it's present, and -//! *may* additionally encode `layer = v - 1` (the destination layer -//! within its partition) once a caller populates it via -//! [`with_layer`](FamilyMask::with_layer) — see that method's docs. -//! `FamilyMask` itself is policy-free about whether a given non-zero -//! value is a trustworthy layer index or just "present, no layer -//! recorded": that decision belongs to callers, who already have -//! `PartitionMeta::n_layers` in scope. -//! - The minorant bit: whether *this slot's own k-mer* is its family's -//! minorant (the smallest raw encoding among the family's observed -//! members) — computed once, when the whole family's mask is already -//! final (see `build::build_layer_sibling_annex`), and read back by every -//! consumer that would otherwise have to reconstruct this slot's k-mer -//! from `unitigs.bin` and hash it through the MPHF again just to ask the -//! same question. -//! -//! Both facts an even earlier design stored explicitly are still derived, -//! not stored: -//! - sibling count = `family_size() - 1`; -//! - which of the (up to 4) family members are present, and their canonical -//! form — [`FamilyMask::family_members`], regenerated from the slot's own -//! k-mer (`CanonicalKmer::central_canonical_neighbors`, cheap, no -//! lookup), not stored. -//! -//! Mask value 0 is logically unreachable as a real result (a slot's own -//! base is always present in its own family) and is reused as the "not yet -//! computed" sentinel: annex files are pre-initialised to all-zero, and a -//! real value is only ever written once, by the computation pass. -//! -//! Widened from the original 1-byte/slot design (4 presence bits + 1 -//! minorant bit) to carry a per-member layer number without an extra -//! lookup pass — see the project discussion this implements. An old -//! (1-byte/slot) `.psib` file is *not* silently misread by the new reader: -//! its length no longer matches `HEADER_SIZE + n * 2`, so `SiblingAnnex::open` -//! fails loudly ("PSIB file truncated") rather than producing garbage. -//! Annexes built before this change must be rebuilt (`--sibling-annex`). - -use std::fs::{File, OpenOptions}; -use std::io; -use std::path::{Path, PathBuf}; -use std::sync::atomic::AtomicU16; - -use memmap2::{Mmap, MmapMut}; - -use obikseq::CanonicalKmer; - -use super::helpers::central_base; - -const MAGIC: [u8; 4] = *b"PSIB"; - -// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (2 bytes/slot) follows. -const HEADER_SIZE: usize = 16; - -/// Width, in bits, of one base's field. -const FIELD_BITS: u32 = 3; -/// Mask for one base's field once shifted into position. -const FIELD_MASK: u16 = 0b111; -/// Highest raw field value a base can carry (`1..=MAX_FIELD_VALUE`, `0` = absent). -const MAX_FIELD_VALUE: u8 = 7; - -const MINORANT_BIT: u16 = 1 << 12; - -/// A family presence mask — see the module docs for the full bit layout. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct FamilyMask(u16); - -impl FamilyMask { - /// The empty mask — never a valid *computed* result (a slot's own base - /// is always present in its own family) — used only to build up a mask - /// via repeated [`with`](Self::with)/[`with_layer`](Self::with_layer) - /// calls before storing it. - pub const EMPTY: FamilyMask = FamilyMask(0); - - #[inline] - fn field_shift(base: u8) -> u32 { - debug_assert!(base < 4, "base out of range: {base}"); - base as u32 * FIELD_BITS - } - - #[inline] - fn field(self, base: u8) -> u8 { - ((self.0 >> Self::field_shift(base)) & FIELD_MASK) as u8 - } - - #[inline] - fn with_field(self, base: u8, value: u8) -> Self { - debug_assert!(value <= MAX_FIELD_VALUE, "field value out of range: {value}"); - let shift = Self::field_shift(base); - let cleared = self.0 & !(FIELD_MASK << shift); - FamilyMask(cleared | ((value as u16) << shift)) - } - - /// Raw bits to `fetch_or` into a shared `AtomicU16` accumulator to mark - /// `base` present at `layer`, without allocating a whole `FamilyMask` - /// per write — for `build::build_layer_sibling_annex`'s concurrent - /// construction, where each `(word, base)` field is written by exactly - /// one thread (see that module's docs), so a lock-free OR is safe. - #[inline] - pub(crate) fn layer_bits(base: u8, layer: usize) -> u16 { - debug_assert!(layer < MAX_FIELD_VALUE as usize, "layer out of range: {layer}"); - (layer as u16 + 1) << Self::field_shift(base) - } - - /// Same as [`layer_bits`](Self::layer_bits), presence-only (no layer - /// recorded) — the accumulator-side equivalent of [`with`](Self::with). - #[inline] - pub(crate) fn presence_bits(base: u8) -> u16 { - 1u16 << Self::field_shift(base) - } - - /// Mark the member with central base `base` (0=A, 1=C, 2=G, 3=T) as - /// present, without recording a layer (compat path: same observable - /// effect as the original 1-byte design's `with`). Use - /// [`with_layer`](Self::with_layer) instead when the destination layer - /// is already known. - #[inline] - pub fn with(self, base: u8) -> Self { - self.with_field(base, 1) - } - - /// Mark the member with central base `base` as present *and* record its - /// destination layer. `layer` must be `< 7` (`debug_assert`ed) — a - /// caller facing more layers than that has no compact field to record - /// them in and must fall back to [`with`](Self::with) instead (still - /// correct, just without the fast-path payoff); see the module docs on - /// why `FamilyMask` doesn't decide that threshold itself. - #[inline] - pub fn with_layer(self, base: u8, layer: usize) -> Self { - debug_assert!(layer < MAX_FIELD_VALUE as usize, "layer out of range: {layer}"); - self.with_field(base, layer as u8 + 1) - } - - /// Is the member with central base `base` (0..3) present? - #[inline] - pub fn has(self, base: u8) -> bool { - self.field(base) != 0 - } - - /// Raw stored value for `base`'s field, if present: `Some(v - 1)` where - /// `v` is the non-zero field value. This is the *raw* stored value, not - /// a validated layer index — a caller must cross-check it against - /// `PartitionMeta::n_layers` (`<= 7`) before trusting it as a real - /// layer, since a mask written via [`with`](Self::with) (no layer - /// known) also reads back as `Some(0)` here. - #[inline] - pub fn layer_value(self, base: u8) -> Option { - let v = self.field(base); - if v == 0 { None } else { Some(v - 1) } - } - - /// Number of family members observed anywhere in the index (1..=4). - #[inline] - pub fn family_size(self) -> u32 { - (0..4).filter(|&b| self.has(b)).count() as u32 - } - - /// Number of *other* members observed (0..=3) — `family_size() - 1`. - #[inline] - pub fn siblings(self) -> u32 { - self.family_size() - 1 - } - - /// Set or clear the minorant flag (see the module docs). - #[inline] - pub fn with_minorant(self, is_minorant: bool) -> Self { - if is_minorant { - FamilyMask(self.0 | MINORANT_BIT) - } else { - FamilyMask(self.0 & !MINORANT_BIT) - } - } - - /// Is this slot's own k-mer its family's minorant? Only meaningful once - /// `with_minorant` has been called with the family's *final* mask (i.e. - /// after construction) — see the module docs. - #[inline] - pub fn is_minorant(self) -> bool { - self.0 & MINORANT_BIT != 0 - } - - /// Raw presence bitmask (bit `b` = base `b` present, low 4 bits only — - /// never includes the minorant flag or any layer information) — for - /// callers that only need the old 1-bit-per-base view, e.g. to compare - /// against a previously-computed value built via their own bit - /// operations. - #[inline] - pub fn bits(self) -> u8 { - (0..4).fold(0u8, |acc, b| if self.has(b) { acc | (1 << b) } else { acc }) - } - - /// Construct from a raw presence bitmask (only the low 4 bits are kept - /// — the minorant flag is not part of this, use - /// [`with_minorant`](Self::with_minorant) separately). Each set bit is - /// recorded as "present, no layer known" — see [`with`](Self::with). - #[inline] - pub fn from_bits(bits: u8) -> Self { - (0..4).fold(FamilyMask::EMPTY, |mask, b| { - if bits & (1 << b) != 0 { mask.with(b) } else { mask } - }) - } - - /// This family's present members, in canonical form, paired with their - /// raw stored field value (see [`layer_value`](Self::layer_value) for - /// why it's not directly a validated layer index). `owner` is this - /// slot's own k-mer (any member works — the 4 canonical forms are - /// invariant regardless of which member you start from). - pub fn family_members(self, owner: CanonicalKmer, k: usize) -> impl Iterator)> { - owner - .central_canonical_neighbors() - .into_iter() - .filter_map(move |member| { - let base = central_base(member, k); - self.has(base).then(|| (member, self.layer_value(base))) - }) - } - - #[inline] - fn encode(self) -> u16 { - self.0 - } - - #[inline] - fn decode(word: u16) -> Option { - if word == 0 { - // Unreachable for a real result — reserved as the "not yet - // computed" sentinel. A real entry always has at least one - // non-zero base field, so word == 0 always means "nothing - // written yet", never a genuine minorant-only value. - return None; - } - Some(FamilyMask(word)) - } -} - -// ── SiblingAnnex (reader) ─────────────────────────────────────────────────── - -pub struct SiblingAnnex { - mmap: Mmap, - n: usize, - path: PathBuf, -} - -impl SiblingAnnex { - pub fn open(path: &Path) -> io::Result { - let mmap = unsafe { Mmap::map(&File::open(path)?)? }; - if mmap.len() < HEADER_SIZE { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short")); - } - if mmap[0..4] != MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic")); - } - let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; - if mmap.len() < HEADER_SIZE + n * 2 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated")); - } - Ok(Self { mmap, n, path: path.to_path_buf() }) - } - - pub fn path(&self) -> &Path { &self.path } - pub fn len(&self) -> usize { self.n } - pub fn is_empty(&self) -> bool { self.n == 0 } - - /// `None` means the slot has not (yet) been computed — see module docs. - pub fn get(&self, slot: usize) -> Option { - let off = HEADER_SIZE + slot * 2; - FamilyMask::decode(u16::from_le_bytes(self.mmap[off..off + 2].try_into().unwrap())) - } -} - -// ── SiblingAnnexBuilder (writer) ──────────────────────────────────────────── - -pub struct SiblingAnnexBuilder { - mmap: MmapMut, -} - -impl SiblingAnnexBuilder { - /// Create a new annex of `n` slots at `path`, pre-initialised to the - /// "not yet computed" sentinel (all-zero). - pub fn new(n: usize, path: &Path) -> io::Result { - let file_size = HEADER_SIZE + n * 2; - let file = OpenOptions::new() - .read(true).write(true).create(true).truncate(true) - .open(path)?; - file.set_len(file_size as u64)?; - let mut mmap = unsafe { MmapMut::map_mut(&file)? }; - mmap[0..4].copy_from_slice(&MAGIC); - mmap[4..8].copy_from_slice(&[0u8; 4]); - mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); - // Data region left at 0 by `set_len`/mmap — the sentinel value. - Ok(Self { mmap }) - } - - /// `None` means the slot has not (yet) been computed — see module - /// docs. Used to read back a slot's concurrently-accumulated value - /// (via [`atomic_slot`](Self::atomic_slot)) before finalising it with - /// [`set`](Self::set). - pub fn get(&self, slot: usize) -> Option { - let off = HEADER_SIZE + slot * 2; - FamilyMask::decode(u16::from_le_bytes(self.mmap[off..off + 2].try_into().unwrap())) - } - - /// Atomic handle to `slot`'s raw storage, for lock-free concurrent - /// construction straight into the mmap — no separate in-memory - /// accumulator needed (see `build::build_layer_sibling_annex`, the - /// only caller). Safe: every concurrent write goes through - /// `fetch_or`/atomic RMW, and by that caller's own invariant, a given - /// 3-bit field within a slot's word is written by exactly one thread — - /// only *different* fields of the same word can race, which `fetch_or` - /// already serialises correctly. Alignment holds: `HEADER_SIZE` is - /// even and every slot is 2 bytes, so `HEADER_SIZE + slot * 2` is - /// always even, and the mmap's own base address is page-aligned. - pub(crate) fn atomic_slot(&self, slot: usize) -> &AtomicU16 { - let off = HEADER_SIZE + slot * 2; - debug_assert!(off + 2 <= self.mmap.len()); - let ptr = unsafe { self.mmap.as_ptr().add(off) as *mut u16 }; - unsafe { AtomicU16::from_ptr(ptr) } - } - - /// Plain (non-atomic) store — for the sequential finalisation pass - /// only, once every concurrent write (via - /// [`atomic_slot`](Self::atomic_slot)) is done. - pub fn set(&mut self, slot: usize, mask: FamilyMask) { - let off = HEADER_SIZE + slot * 2; - self.mmap[off..off + 2].copy_from_slice(&mask.encode().to_le_bytes()); - } - - pub fn close(self) -> io::Result<()> { self.mmap.flush() } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.psib"); - let builder = SiblingAnnexBuilder::new(4, &path).unwrap(); - builder.close().unwrap(); - let annex = SiblingAnnex::open(&path).unwrap(); - for slot in 0..4 { - assert_eq!(annex.get(slot), None); - } - } - - #[test] - fn roundtrip_all_valid_masks() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.psib"); - let mut builder = SiblingAnnexBuilder::new(4, &path).unwrap(); - - let masks = [ - FamilyMask::EMPTY.with(0), // just A: family size 1 - FamilyMask::EMPTY.with(0).with(3), // A + T: size 2 - FamilyMask::EMPTY.with(1).with(2).with(3), // C+G+T: size 3 - FamilyMask::EMPTY.with(0).with(1).with(2).with(3), // all 4 - ]; - for (slot, mask) in masks.iter().enumerate() { - builder.set(slot, *mask); - } - builder.close().unwrap(); - let annex = SiblingAnnex::open(&path).unwrap(); - for (slot, mask) in masks.iter().enumerate() { - assert_eq!(annex.get(slot), Some(*mask)); - } - assert_eq!(annex.get(0).unwrap().siblings(), 0); - assert_eq!(annex.get(1).unwrap().siblings(), 1); - assert_eq!(annex.get(2).unwrap().siblings(), 2); - assert_eq!(annex.get(3).unwrap().siblings(), 3); - assert_eq!(annex.get(3).unwrap().family_size(), 4); - } - - #[test] - fn has_reflects_individual_bits() { - let mask = FamilyMask::EMPTY.with(0).with(2); - assert!(mask.has(0)); - assert!(!mask.has(1)); - assert!(mask.has(2)); - assert!(!mask.has(3)); - } - - #[test] - fn with_layer_roundtrips_and_leaves_absent_bases_none() { - let mask = FamilyMask::EMPTY.with_layer(0, 0).with_layer(2, 5); - assert_eq!(mask.layer_value(0), Some(0)); - assert_eq!(mask.layer_value(2), Some(5)); - assert_eq!(mask.layer_value(1), None); - assert_eq!(mask.layer_value(3), None); - assert!(mask.has(0) && mask.has(2)); - assert!(!mask.has(1) && !mask.has(3)); - assert_eq!(mask.family_size(), 2); - } - - #[test] - fn with_marks_present_without_a_layer() { - // Compat path: `with` still means "present", now reading back as - // `layer_value == Some(0)` (raw field value 1, i.e. "no layer - // recorded") — distinguishable from a real layer 0 only by a - // caller that already knows whether layers were ever wired in. - let mask = FamilyMask::EMPTY.with(1); - assert!(mask.has(1)); - assert_eq!(mask.layer_value(1), Some(0)); - } - - #[test] - fn family_members_reconstructs_present_canonical_forms() { - use obikseq::{Kmer, Sequence}; - - const K: usize = 11; - obikseq::params::set_k(K); - // Centre (index 5) is 'C' (base 1) in this k-mer's own orientation. - let owner = Kmer::from_ascii(b"AACCGCTTAAG").unwrap().canonical(); - let mask = FamilyMask::EMPTY.with_layer(1, 0).with_layer(2, 3); // C (own) + G present - - let members: Vec<_> = mask.family_members(owner, K).collect(); - assert_eq!(members.len(), 2, "only the 2 present members should be yielded"); - assert!(members.iter().any(|(k, layer)| *k == owner && *layer == Some(0))); - assert!(members.iter().any(|(k, layer)| *k != owner && *layer == Some(3))); - } -} diff --git a/src/obikphylo/src/siblings_old/stats.rs b/src/obikphylo/src/siblings_old/stats.rs deleted file mode 100644 index dbf24379..00000000 --- a/src/obikphylo/src/siblings_old/stats.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::sync::Arc; - -use rayon::prelude::*; - -use obisys::progress_bar; - -use obikindex::KmerIndex; -use obikindex::{OKIError, OKIResult}; - -use super::ANNEX_FILE_NAME; -use super::SiblingAnnex; -use super::cache::PartitionCache; -use super::family_scan::{Selection, scan_layer_families}; - -/// Distribution of family sizes (1-4), read back from an already-built -/// annex (see [`super::build::SiblingAnnexBuildExt::build_sibling_annex`]) -/// plus the index's presence/count data — a separate, occasional diagnostic -/// pass, not fused into construction. -/// -/// Every count here is **per family, not per slot**: a family with `F` -/// members occupies `F` annex slots (one per observed member), all sharing -/// the same mask. Counting every slot would count each family up to 4 -/// times over; only the minorant's slot is tallied (minorant is derived on -/// the fly — see `is_minorant` — not stored, but cheap: no lookup, pure -/// bit arithmetic on already-in-hand data). -#[derive(Debug, Clone, Default)] -pub struct SiblingAnnexStats { - /// `counts[s]` = number of *families* of size `s + 1` (index 0 = size 1, - /// i.e. 0 siblings, ... index 3 = size 4, i.e. 3 siblings). - pub counts: [u64; 4], - /// `per_genome[g][s]` = number of families of size `s + 1` for which - /// genome `g` (index into `KmerIndex::meta().genomes`) carries at least - /// one member. - pub per_genome: Vec<[u64; 4]>, -} - -/// Adds [`sibling_family_size_histogram`](Self::sibling_family_size_histogram) -/// and [`sibling_annex_stats`](Self::sibling_annex_stats) to `KmerIndex`. -pub trait SiblingStatsExt { - /// The global family-size histogram alone (`SiblingAnnexStats::counts`, - /// no `per_genome`) — reads only the already-built annex (`mask.siblings()` - /// + `mask.is_minorant()`, 1 byte/slot, mmap'd), nothing else: no - /// `unitigs.bin` scan, no MPHF lookup, no `PartitionCache`. Cost is a - /// linear scan of one small file per layer, independent of the rest of - /// the index's size or how much of it is paged in. An earlier version - /// re-derived minorant-ness per slot (re-reading `unitigs.bin`, hashing - /// every k-mer through the MPHF again to reconstruct what - /// `build_sibling_annex` already knew) — sampling a real run showed - /// `MphfLayer::find` alone at 71% of wall-clock time for what should be - /// a near-instant four-bucket count. `mask.is_minorant()` is that same - /// fact, computed once at construction (see `build_layer_sibling_annex`) - /// and stored in the annex's own spare bits — free to read back. - /// [`sibling_annex_stats`](Self::sibling_annex_stats) computes the same - /// `counts` but pays the full per-genome cross-partition resolution - /// cost to get there too; use this when only the global histogram is - /// needed. Requires an annex built after the minorant flag was added — - /// re-run `build_sibling_annex` if this reads as all-zero on an older one. - fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]>; - - /// Tally the family-size distribution of an already-built annex - /// (globally, and per genome), counting each family once (at its - /// minorant slot). Errors if - /// [`build_sibling_annex`](super::build::SiblingAnnexBuildExt::build_sibling_annex) - /// has not been run on this index first. - fn sibling_annex_stats(&self) -> OKIResult; -} - -impl SiblingStatsExt for KmerIndex { - fn sibling_family_size_histogram(&self) -> OKIResult<[u64; 4]> { - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - - // Each layer's annex file is independent — no shared `PartitionCache` - // and no `scan_layer_families` partition-grouped locality to protect - // (unlike `sibling_annex_stats`/`distance`/`cardinality_tally`), so - // layers can be scanned concurrently. - layer_dirs - .par_iter() - .try_fold( - || [0u64; 4], - |mut counts, layer_dir| -> OKIResult<[u64; 4]> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { - continue; - }; - if !mask.is_minorant() { - continue; // family tallied once, at its minorant - } - counts[mask.siblings() as usize] += 1; - } - Ok(counts) - }, - ) - .try_reduce( - || [0u64; 4], - |a, b| { - let mut sum = a; - for i in 0..4 { - sum[i] += b[i]; - } - Ok(sum) - }, - ) - } - - fn sibling_annex_stats(&self) -> OKIResult { - let n_parts = self.n_partitions(); - let n_genomes = self.meta().genomes().map_err(OKIError::Io)?.len(); - let with_counts = self.meta().config.with_counts; - let k = self.kmer_size(); - // Same whole-run cache as `build_sibling_annex` — see its docs for - // why re-opening per lookup (or per call to a batching helper) is - // not good enough on a real index. - let cache = Arc::new(PartitionCache::build(self, n_parts, with_counts)?); - let layer_dirs = super::family_scan::sibling_layer_dirs(self)?; - - // One layer at a time, not parallelised across layers — see - // `snp_pseudo_alignment`'s comment for why `par_iter()` over layers - // would defeat `scan_layer_families`'s partition-grouped locality. - let pb = progress_bar("sibling_annex_stats", layer_dirs.len() as u64, "layers"); - let mut stats = SiblingAnnexStats { - per_genome: vec![[0u64; 4]; n_genomes], - ..Default::default() - }; - for layer_dir in &layer_dirs { - scan_layer_families( - layer_dir, - n_parts, - n_genomes, - with_counts, - k, - &cache, - &Selection::All, - |_family_idx, mask, genome_mask| { - // "Genome g represents this family" means g carries - // *any* of its members, not just the minorant's own — - // `genome_mask[g] != 0` is exactly that. - let s = mask.siblings() as usize; - stats.counts[s] += 1; - for (g, &m) in genome_mask.iter().enumerate() { - if m != 0 { - stats.per_genome[g][s] += 1; - } - } - }, - )?; - pb.inc(1); - } - pb.finish_and_clear(); - - Ok(stats) - } -} diff --git a/src/obikphylo/src/siblings_old/subsample.rs b/src/obikphylo/src/siblings_old/subsample.rs deleted file mode 100644 index fc6b0a32..00000000 --- a/src/obikphylo/src/siblings_old/subsample.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! Proportional per-layer sampling of non-monomorphic minorants -//! (`family_size() >= 2`) — see `DevDocMD/architecture/siblings.md`, -//! "`--subsample`/`--shannon`", for the full design discussion. -//! -//! Three steps, in order, each cheaper than the one before because it -//! narrows what the next one has to touch: -//! -//! 1. [`non_monomorphic_counts`] — one cheap structural pass per layer -//! (annex bits only, no cross-partition resolution), giving `count_layer` -//! for every layer. -//! 2. [`sample_layers`] — given a target `n` and those counts, computes -//! `n_layer = round(n * count_layer / total_count)` per layer (never -//! exceeds `count_layer` as long as `n <= total_count`; if -//! `total_count <= n`, sampling is skipped entirely — every -//! non-monomorphic minorant of every layer is kept) and reservoir-samples -//! (Algorithm R, one more cheap structural pass, `O(n_layer)` memory) the -//! selected iteration-order indices. -//! 3. The result feeds [`super::family_scan::scan_layer_families`]'s -//! `Selection` parameter, which bounds the expensive cross-partition -//! resolution to just the selected families. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; - -use rand::RngExt; -use rayon::prelude::*; - -use obikindex::{KmerIndex, OKIError, OKIResult}; - -use super::ANNEX_FILE_NAME; -use super::SiblingAnnex; -use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME}; - -/// Gaussian-kernel entropy bias parameters — see -/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". `mu`/ -/// `sigma` are the `--entropy`/`--entropy-sd` CLI values (defaulted by the -/// caller, not here — this type carries only the resolved numbers). `pub` -/// (not `pub(super)`): part of the public signature of -/// `SnpAlignmentExt::snp_pseudo_alignment`, -/// `ShannonEntropyExt::shannon_entropy_csv`, and -/// `SankoffBundleExt::sankoff_bundle`, all consumed from `obikmer`. -#[derive(Clone, Copy)] -pub struct EntropyBias { - pub mu: f64, - pub sigma: f64, -} - -/// A family is non-monomorphic (informative-eligible) iff at least one -/// sibling is registered alongside it — `family_size() >= 2`, i.e. -/// `siblings() >= 1`. Mirrors the filter already applied post-hoc in -/// `snp_pseudo_alignment`, but checked here *before* any resolution work. -fn is_non_monomorphic_minorant(mask: super::FamilyMask) -> bool { - mask.is_minorant() && mask.siblings() >= 1 -} - -/// Step 1: per-layer count of non-monomorphic minorants — cheap, parallel -/// across layers, annex bits only (same shape as -/// `SiblingStatsExt::sibling_family_size_histogram`, but keeping the -/// per-layer breakdown instead of collapsing it into one index-wide sum, -/// since proportional sampling needs each layer's own share). -pub(super) fn non_monomorphic_counts(layer_dirs: &[PathBuf]) -> OKIResult> { - layer_dirs - .par_iter() - .map(|layer_dir| -> OKIResult { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut count = 0u64; - for slot in 0..annex.len() { - if annex.get(slot).is_some_and(is_non_monomorphic_minorant) { - count += 1; - } - } - Ok(count) - }) - .collect() -} - -/// Step 2, per layer: Algorithm-R reservoir sampling of `n_layer` -/// non-monomorphic minorant iteration-indices — single pass, no -/// cross-partition resolution, `O(n_layer)` memory regardless of the -/// layer's true size (never materialises the full non-monomorphic index -/// list, only the reservoir itself). -/// -/// The stored values are **`family_idx`, `scan_layer_families`'s own -/// numbering** — the index among *every* minorant of the layer (monomorphic -/// ones included, since `iter_minorants_batch` only filters on -/// `is_minorant()`), not the raw annex slot (`for slot in 0..annex.len()` -/// spans every k-mer, minorant or not) and not a counter over -/// non-monomorphic minorants alone. Conflating any of these with -/// `family_idx` silently drifts apart as soon as the layer contains a -/// monomorphic minorant — i.e. almost immediately, since ~98% of minorants -/// are monomorphic — so the two counters below are deliberately kept -/// separate: `family_idx` tracks every minorant (matching -/// `scan_layer_families`), `seen` tracks only the non-monomorphic ones -/// (what Algorithm R actually samples over). -fn reservoir_sample_layer(layer_dir: &Path, n_layer: usize) -> OKIResult> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut reservoir: Vec = Vec::with_capacity(n_layer); - let mut seen: u64 = 0; - let mut family_idx: usize = 0; - let mut rng = rand::rng(); - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { continue }; - if !mask.is_minorant() { - continue; // not a minorant at all — doesn't advance `family_idx` either - } - let this_family_idx = family_idx; - family_idx += 1; - if mask.siblings() < 1 { - continue; // monomorphic minorant — consumed a family_idx above, but not sample-eligible - } - if reservoir.len() < n_layer { - reservoir.push(this_family_idx); - } else { - let j = rng.random_range(0..=seen); - if j < n_layer as u64 { - reservoir[j as usize] = this_family_idx; - } - } - seen += 1; - } - Ok(reservoir.into_iter().collect()) -} - -/// Every non-monomorphic minorant's `family_idx`, for one layer — not a -/// sample, the *complete* set. Structural, annex-only, no cross-partition -/// resolution (same discipline as [`reservoir_sample_layer`], just keeping -/// everything instead of a bounded reservoir). Used to bound -/// [`super::entropy::ensure_entropy_annexes`]'s `scan_layer_families` call -/// to a `Selection::Some` of exactly the families that need their entropy -/// computed, instead of `Selection::All` — monomorphic minorants (~98% of -/// them, measured) are known to be ineligible straight from the annex bits -/// already, with no need to pay `scan_layer_families`'s expensive -/// per-genome resolution (`fill_sub_matrix_carries`) only to discard the -/// result once `family_size() < 2` is checked *after* resolving. -pub(super) fn non_monomorphic_selection_layer(layer_dir: &Path) -> OKIResult> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut selected = HashSet::new(); - let mut family_idx: usize = 0; - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { continue }; - if !mask.is_minorant() { - continue; // doesn't advance `family_idx` either - } - let this_family_idx = family_idx; - family_idx += 1; - if is_non_monomorphic_minorant(mask) { - selected.insert(this_family_idx); - } - } - Ok(selected) -} - -/// Total minorant count per layer (`is_minorant()`, monomorphic included) -/// — sizes the entropy annex (`entropy_annex.rs`), which needs exactly one -/// entry per minorant to keep its `family_idx` numbering aligned with -/// every other consumer (`scan_layer_families`, `Selection`, -/// `reservoir_sample_layer`'s own `family_idx` counter above). -pub(super) fn minorant_counts(layer_dirs: &[PathBuf]) -> OKIResult> { - layer_dirs - .par_iter() - .map(|layer_dir| -> OKIResult { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let mut count = 0u64; - for slot in 0..annex.len() { - if annex.get(slot).is_some_and(|m| m.is_minorant()) { - count += 1; - } - } - Ok(count) - }) - .collect() -} - -/// Entropy-biased alternative to [`reservoir_sample_layer`]: an -/// independent accept/reject draw per non-monomorphic minorant, probability -/// `p0 * w(entropy)` where `w(entropy) = exp(-(entropy-mu)^2 / (2*sigma^2))` -/// (the unnormalised Gaussian kernel — `1` exactly at `entropy == mu`, -/// smoothly decaying away from it, no hard cutoff) and `p0` is the same -/// uniform rate the plain reservoir targets in expectation (`n / total_count` -/// index-wide, or `1.0` when there is no size target at all — `--entropy` -/// given without `--subsample`, a pure soft entropy filter over the whole -/// index). See `DevDocMD/architecture/siblings.md`, "Entropy-biased -/// selection", for why this is a single streaming pass with no -/// cross-partition resolution: entropy is read from the already-built -/// entropy annex (`ensure_entropy_annexes`), not recomputed here. -fn entropy_biased_sample_layer(layer_dir: &Path, p0: f64, bias: EntropyBias) -> OKIResult> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME))?; - let entropy_annex = EntropyAnnex::open(&layer_dir.join(ENTROPY_ANNEX_FILE_NAME)).map_err(OKIError::Io)?; - let mut selected = HashSet::new(); - let mut family_idx: usize = 0; - let mut rng = rand::rng(); - for slot in 0..annex.len() { - let Some(mask) = annex.get(slot) else { continue }; - if !mask.is_minorant() { - continue; // not a minorant at all — doesn't advance `family_idx` either - } - let this_family_idx = family_idx; - family_idx += 1; - debug_assert!( - this_family_idx < entropy_annex.len(), - "entropy annex ({}) shorter than this layer's minorant count — built against a different annex?", - entropy_annex.len(), - ); - if mask.siblings() < 1 { - continue; // monomorphic minorant — no entropy, never eligible - } - let Some(entropy) = entropy_annex.get(this_family_idx) else { continue }; - let d = (entropy as f64 - bias.mu) / bias.sigma; - let w = (-0.5 * d * d).exp(); - if rng.random::() < p0 * w { - selected.insert(this_family_idx); - } - } - Ok(selected) -} - -/// Per-layer sampling outcome — `None` at a given layer means "keep every -/// non-monomorphic minorant of this layer" (either the `total_count <= n` -/// case, or a layer whose entire non-monomorphic set was already smaller -/// than its computed share and got fully absorbed by rounding). -pub(super) type LayerSelection = Option>; - -/// Full entry point for every caller (`snp_pseudo_alignment`, the Shannon -/// entropy report, `sankoff_bundle`): `n = None` and `entropy_bias = None` -/// means "no `--subsample`, no `--entropy`/`--entropy-sd`", i.e. keep every -/// non-monomorphic minorant of every layer — skips steps 1 and 2 entirely, -/// since there is nothing to proportion. -/// -/// `entropy_bias = Some(_)` switches selection from the plain uniform -/// reservoir to [`entropy_biased_sample_layer`] — see -/// `DevDocMD/architecture/siblings.md`, "Entropy-biased selection". `index` is -/// only used on that path, to build any missing per-layer entropy annex -/// first ([`super::entropy::ensure_entropy_annexes`]) — a one-time cost, -/// not paid again once every layer's annex file exists. The target size -/// `n` still proportions the *expected* rate across the whole index (a -/// single `p0 = n / total_count` applied uniformly at every layer already -/// yields each layer its proportional share, with no need to compute a -/// per-layer target the way [`sample_layers`] does for the uniform -/// reservoir) — but unlike the uniform reservoir, the result is -/// approximate, not exact (see [`entropy_biased_sample_layer`]'s docs). -pub(super) fn compute_selections( - index: &KmerIndex, - layer_dirs: &[PathBuf], - n: Option, - entropy_bias: Option, -) -> OKIResult> { - let Some(bias) = entropy_bias else { - return match n { - None => Ok(vec![None; layer_dirs.len()]), - Some(n) => { - let counts = non_monomorphic_counts(layer_dirs)?; - sample_layers(layer_dirs, &counts, n) - } - }; - }; - - super::entropy::ensure_entropy_annexes(index, layer_dirs)?; - let p0 = match n { - None => 1.0, - Some(n) => { - let counts = non_monomorphic_counts(layer_dirs)?; - let total_count: u64 = counts.iter().sum(); - if total_count == 0 { 0.0 } else { (n as f64 / total_count as f64).min(1.0) } - } - }; - layer_dirs - .par_iter() - .map(|layer_dir| -> OKIResult { - Ok(Some(entropy_biased_sample_layer(layer_dir, p0, bias)?)) - }) - .collect() -} - -/// Step 2 entry point: `n` is the total number of families requested across -/// the whole index (`--subsample N`); `layer_dirs` and `counts` must be the -/// same length and in the same order as returned by -/// [`super::family_scan::sibling_layer_dirs`] / -/// [`non_monomorphic_counts`]. -fn sample_layers(layer_dirs: &[PathBuf], counts: &[u64], n: usize) -> OKIResult> { - let total_count: u64 = counts.iter().sum(); - - if total_count <= n as u64 { - // Fewer non-monomorphic minorants in the whole index than requested - // — no sampling needed, keep everything everywhere. - return Ok(vec![None; layer_dirs.len()]); - } - - layer_dirs - .par_iter() - .zip(counts.par_iter()) - .map(|(layer_dir, &count_layer)| -> OKIResult { - if count_layer == 0 { - return Ok(Some(HashSet::new())); - } - let n_layer = ((n as u128 * count_layer as u128) / total_count as u128) as usize; - if n_layer == 0 { - return Ok(Some(HashSet::new())); - } - let selected = reservoir_sample_layer(layer_dir, n_layer)?; - Ok(Some(selected)) - }) - .collect() -} diff --git a/src/obikphylo/src/siblings_old/tests.rs b/src/obikphylo/src/siblings_old/tests.rs deleted file mode 100644 index 880e67af..00000000 --- a/src/obikphylo/src/siblings_old/tests.rs +++ /dev/null @@ -1,557 +0,0 @@ -use std::io::Write; -use std::path::Path; - -use obikseq::{CanonicalKmer, Kmer, Sequence}; -use obikindex::layer::MphfLayer; -use obisys::Reporter; -use tempfile::tempdir; - -use obikalgorithm::Algorithm; -use obikindexer::algorithms::counter::Counter; -use obikindexer::algorithms::dereplicator::Dereplicator; -use obikindexer::algorithms::layer_builder::LayerBuilder; -use obikindexer::algorithms::partitionner::PartitionRouter; -use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode}; - -use super::alignment::SnpAlignmentExt; -use super::build::SiblingAnnexBuildExt; -use super::cardinality::CardinalityExt; -use super::distance::DistanceExt; -use super::entropy::ShannonEntropyExt; -use super::entropy_annex::{EntropyAnnex, ENTROPY_ANNEX_FILE_NAME}; -use super::helpers::is_minorant; -use super::sankoff_bundle::SankoffBundleExt; -use super::stats::SiblingStatsExt; -use super::subsample::EntropyBias; -use super::{FamilyMask, SiblingAnnex, ANNEX_FILE_NAME}; - -// k must be >= 11 (project constraint, "k ∈ [11,31]"); k=11, level_max=1, -// theta=0.0 mirror `obiskbuilder`'s own tests (smaller k/level_max -// combinations trip an unrelated pre-existing bug in `obikentropy`'s -// sliding-window ring buffer — not this feature's concern). -const K: usize = 11; -const M: usize = 5; - -/// Build a single-genome index from one in-memory FASTA sequence, driving -/// the same primitives `obikmer`'s `scatter` step uses (minus the -/// multi-file `obipipeline` wrapper — a single sequence needs none of -/// that): normalise -> build superkmers -> route -> write. -/// `cargo test` doesn't install a `tracing` subscriber the way `obikmer`'s -/// CLI does, so `debug!`/etc. are silent no-ops by default — including the -/// `PartitionRunner` instrumentation that would matter most for -/// re-diagnosing a hang here. `try_init` is idempotent across concurrently -/// running tests (later calls just find a subscriber already installed). -fn init_tracing() { - let _ = tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .with_writer(std::io::stderr) - .try_init(); -} - -fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex { - init_tracing(); - let fasta_path = dir.join(format!("{label}.fasta")); - let mut f = std::fs::File::create(&fasta_path).unwrap(); - writeln!(f, ">{label}").unwrap(); - f.write_all(seq).unwrap(); - writeln!(f).unwrap(); - drop(f); - - let index_path = dir.join(format!("{label}.idx")); - let config = IndexConfig { - kmer_size: K, - minimizer_size: M, - n_bits: 0, // 1 partition — keeps the test deterministic and simple - with_counts: false, - evidence: obikindex::layer::IndexMode::Exact, - block_bits: 0, - }; - let idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label))) - .expect("create"); - - let stream = obiread::open_nuc_stream(fasta_path.to_str().unwrap(), K).expect("open fasta"); - let mut router = PartitionRouter::new(&idx); - for page in stream { - let batch = obiskbuilder::build_superkmers_page(page, K, /* level_max */ 1, /* theta */ 0.0); - router.write_batch(batch).expect("write_batch"); - } - router.close().expect("close partition writers"); // also marks scatter done - drop(router); // ends the borrow of `idx` early — `PartitionRouter`'s `Drop` impl would otherwise extend it to the end of scope - - Dereplicator::new(&idx).run().expect("dereplicate"); - Counter::new(&idx).run().expect("count_kmer"); // also writes the spectrum + marks counted - LayerBuilder::new(&idx) - .run() - .expect("build_layers"); // also marks indexed - idx -} - -fn canonical(ascii: &[u8]) -> CanonicalKmer { - Kmer::from_ascii(ascii).unwrap().canonical() -} - -/// Read back the annex entry for a given canonical k-mer from the merged -/// index's (single) partition/layer, asserting it was found at all. -fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask { - let meta = idx.partition_meta(0).unwrap(); - for l in 0..meta.n_layers { - let layer_dir = idx.layer_dir(0, l); - let mphf = MphfLayer::open(&layer_dir).unwrap(); - if let Some(slot) = mphf.find(kmer) { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).unwrap(); - return annex.get(slot).expect("slot must have a computed annex entry"); - } - } - panic!("kmer not found in any layer of partition 0"); -} - -fn merge_two(dir: &Path, g1: &KmerIndex, g2: &KmerIndex) -> KmerIndex { - let mut rep = Reporter::new(); - KmerIndex::merge( - &dir.join("merged.idx"), - &[g1, g2], - MergeMode::Presence, - false, - false, - 1.0, - &mut rep, - ) - .expect("merge") -} - -#[test] -fn sibling_annex_one_sibling_each() { - // k=11, centre = index 5 (0-based). Two genomes, each exactly one - // k-mer, sharing every base except the centre: - // g1 = "AACCGCTTAAG" (centre 'C', base index 1) - // g2 = "AACCGGTTAAG" (centre 'G', base index 2) - // Hand-verified: both stay forward-oriented under canonicalisation - // (each is lexicographically smaller than its own reverse - // complement, since both start with "AA"), and raw(g1) < raw(g2) - // (only differing base: C=0b01 < G=0b10 at the centre) — so g1 is - // the minorant, g2 is not. The mask is a family-wide value: both - // slots must read back the *same* presence bits (1 and 2 set) — but - // only g1's slot should carry the stored minorant flag. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let g1_kmer = canonical(b"AACCGCTTAAG"); - let g2_kmer = canonical(b"AACCGGTTAAG"); - let expected_mask = FamilyMask::EMPTY.with(1).with(2); - - let a = annex_info_for(&merged, g1_kmer); - assert_eq!(a.bits(), expected_mask.bits(), "AACCGCTTAAG"); - assert_eq!(a.siblings(), 1); - assert!(is_minorant(g1_kmer, a, K), "g1 should be the minorant"); - assert!(a.is_minorant(), "g1's stored minorant flag should be set at build time"); - - let b = annex_info_for(&merged, g2_kmer); - assert_eq!(b.bits(), expected_mask.bits(), "AACCGGTTAAG"); - assert_eq!(b.siblings(), 1); - assert!(!is_minorant(g2_kmer, b, K), "g2 should not be the minorant"); - assert!(!b.is_minorant(), "g2's stored minorant flag should not be set"); -} - -#[test] -fn sibling_annex_works_after_pack_sparse() { - // Same fixture as `sibling_annex_one_sibling_each`, but with - // `pack --sparse` (`KmerIndex::pack_matrices(true)`) run on the merged - // index before building the sibling annex — proves `PartitionCache`'s - // sparse-detection (`Mat::SparsePresence`, gated on `presence/ - // is_multi.prsb`) and the generic `TypedLayer` methods it relies on - // actually round-trip through the real build pipeline, not just the - // unit-level `TypedLayer` tests in - // `obikindex::layer`. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.pack_matrices(true).expect("pack_matrices(sparse)"); - - let index_dir = merged.index_dir(0); - assert!( - index_dir.join("layer_0").join("presence").join("is_multi.prsb").exists(), - "pack_matrices(true) must leave the sparse marker file behind" - ); - - merged.build_sibling_annex().expect("build_sibling_annex"); - - let g1_kmer = canonical(b"AACCGCTTAAG"); - let g2_kmer = canonical(b"AACCGGTTAAG"); - let expected_mask = FamilyMask::EMPTY.with(1).with(2); - - let a = annex_info_for(&merged, g1_kmer); - assert_eq!(a.bits(), expected_mask.bits(), "AACCGCTTAAG"); - assert_eq!(a.siblings(), 1); - assert!(is_minorant(g1_kmer, a, K), "g1 should be the minorant"); - assert!(a.is_minorant(), "g1's stored minorant flag should be set at build time"); - - let b = annex_info_for(&merged, g2_kmer); - assert_eq!(b.bits(), expected_mask.bits(), "AACCGGTTAAG"); - assert_eq!(b.siblings(), 1); - assert!(!is_minorant(g2_kmer, b, K), "g2 should not be the minorant"); - assert!(!b.is_minorant(), "g2's stored minorant flag should not be set"); -} - -#[test] -fn sibling_annex_zero_siblings_when_identical_and_no_variant_exists() { - // Same k-mer in both genomes, no other genome around to carry a - // variant -> 0 siblings, trivially its own minorant. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"GATTACAGATC"); - let g2 = build_single_genome_index(dir.path(), "g2", b"GATTACAGATC"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let kmer = canonical(b"GATTACAGATC"); - let mask = annex_info_for(&merged, kmer); - assert_eq!(mask.siblings(), 0, "GATTACAGATC"); - assert_eq!(mask.family_size(), 1); - assert!(is_minorant(kmer, mask, K)); -} - -#[test] -fn sibling_annex_stats_counts_each_family_once_and_per_genome() { - // Reuses the one-sibling-each fixture: a single family of size 2 - // (g1's centre-C form + g2's centre-G form), each genome carrying - // exactly one of the two members. Stats must report exactly one - // family of size 2 (`counts[1] == 1`, since index 1 = size 2), not - // two (which naively summing both slots would give), and both - // genomes represented at size 2, neither at any other size. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let stats = merged.sibling_annex_stats().expect("sibling_annex_stats"); - - assert_eq!(stats.counts, [0, 1, 0, 0], "one family of size 2, counted once"); - assert_eq!(stats.per_genome.len(), 2); - for g in 0..2 { - assert_eq!( - stats.per_genome[g], [0, 1, 0, 0], - "genome {g} should represent exactly one size-2 family" - ); - } -} - -#[test] -fn sibling_family_size_histogram_matches_full_stats_global_counts() { - // Same fixture as `sibling_annex_stats_counts_each_family_once_and_per_genome` - // — the cheap, annex-only histogram must agree with the `counts` half of - // the full (cross-partition) stats pass, without needing a `PartitionCache` - // at all. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let histogram = merged.sibling_family_size_histogram().expect("sibling_family_size_histogram"); - assert_eq!(histogram, [0, 1, 0, 0], "one family of size 2, counted once"); - - let stats = merged.sibling_annex_stats().expect("sibling_annex_stats"); - assert_eq!(histogram, stats.counts, "must agree with the full stats pass's global counts"); -} - -/// Exercises the four sibling-annex consumers that were rewritten to share -/// `family_scan::scan_layer_families` (partition-grouped lookups instead of -/// one lookup per family) — same one-sibling-each fixture as the tests -/// above (g1 carries the family's centre-C member, g2 the centre-G one), -/// hand-verified expected output for each. -#[test] -fn family_scan_consumers_agree_on_one_sibling_each() { - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - // Merge doesn't promise to preserve source order, so resolve each - // genome's index by label rather than assuming g1 -> 0, g2 -> 1. - let idx_of = |label: &str| merged.meta().genomes().unwrap().iter().position(|g| g.label == label).unwrap(); - let (i1, i2) = (idx_of("g1"), idx_of("g2")); - - // snp_pseudo_alignment: one variable family, one column — g1's row - // reads 'C' (its own member), g2's reads 'G'. - let alignment = merged.snp_pseudo_alignment(None, None).expect("snp_pseudo_alignment"); - assert_eq!(alignment.sequences[i1], vec![b'C']); - assert_eq!(alignment.sequences[i2], vec![b'G']); - - // raw_snp_distance: g1's single form (C) != g2's (G) at the family's - // one eligible locus -> a SNP, not a shared site. - let raw = merged.raw_snp_distance().expect("raw_snp_distance"); - assert_eq!(raw.snp[[i1, i2]], 1); - assert_eq!(raw.snp[[i2, i1]], 1); - assert_eq!(raw.shared[[i1, i2]], 0); - assert_eq!(raw.shared[[i2, i1]], 0); - - // cardinality_tally: both genomes carry exactly one member of the - // family (cardinality 1 each) -> one co-occurrence at [1][1]. - let cardinality = merged.cardinality_tally(&raw, 1.0).expect("cardinality_tally"); - assert_eq!(cardinality.counts[1][1], 1); - let total: u64 = cardinality.counts.iter().flatten().sum(); - assert_eq!(total, 1, "no other cardinality pair should be tallied"); - - // base_pair_tally: the one eligible, differing locus is C (base 1) vs - // G (base 2). - let base_pairs = merged.base_pair_tally(&raw, 1.0).expect("base_pair_tally"); - assert_eq!(base_pairs.counts[1][2], 1); - assert_eq!(base_pairs.counts[2][1], 1); - let total: u64 = base_pairs.counts.iter().flatten().sum(); - assert_eq!(total, 2, "no other base pair should be tallied"); - assert_eq!(base_pairs.same, [0, 0, 0, 0], "the two genomes never agree at this locus"); -} - -#[test] -fn sibling_annex_no_empty_masks_after_build() { - let dir = tempdir().unwrap(); - let seq = b"ACGTACGTACGT".repeat(500); - let g1 = build_single_genome_index(dir.path(), "g1", &seq); - g1.build_sibling_annex().expect("build_sibling_annex"); - - let n_layers = g1.n_layers(0).expect("partition meta"); - for l in 0..n_layers { - let layer_dir = g1.layer_dir(0, l); - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open"); - for slot in 0..annex.len() { - let mask = annex.get(slot).expect("slot must have an entry"); - assert!( - mask.family_size() >= 1, - "layer {l} slot {slot}: empty family mask (bits={:04b}) after build — \ - indicates batch-offset bug or unseeded mask slot", - mask.bits() - ); - } - } -} - -#[test] -fn sibling_histogram_does_not_panic_on_partial_last_batch() { - // Regression test for the bug where enumerate_kmers_batch().enumerate() - // used the batch index instead of the sequence offset, leaving the - // trailing slots of a non-multiple-of-BATCH_SIZE layer unseeded and - // producing the minorant-only sentinel (0x10) that caused - // sibling_family_size_histogram to panic with index u32::MAX. - let dir = tempdir().unwrap(); - let seq = b"ACGTACGTACGT".repeat(500); - let g1 = build_single_genome_index(dir.path(), "g1", &seq); - g1.build_sibling_annex().expect("build_sibling_annex"); - let hist = g1.sibling_family_size_histogram().expect("sibling_family_size_histogram"); - let total: u64 = hist.iter().sum(); - assert!(total > 0, "histogram should contain at least one family"); -} - -/// `sibling_annex_one_sibling_each`'s fixture happens to put g2's k-mer in -/// layer 0 and g1's in layer 1 (a fresh single-genome index, then one -/// merge that adds exactly one new layer for content absent from the -/// first — confirmed empirically, not assumed) — a real two-layer index, -/// not a contrived one, so it exercises both write paths -/// `build_layer_sibling_annex` now has to get right: the seeding pass (a -/// slot's own base gets *this* layer's index directly) and the -/// cross-partition/cross-layer resolution pass (`PartitionCache::find` -/// reporting which layer the other family member actually lives in, not -/// just that it exists). -#[test] -fn sibling_annex_records_the_real_layer_of_each_family_member() { - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let n_layers = merged.n_layers(0).unwrap(); - assert_eq!(n_layers, 2, "fixture assumption: one merge, one new layer"); - - let g1_kmer = canonical(b"AACCGCTTAAG"); // own base C (1), sibling base G (2) — lives in layer 1 - let g2_kmer = canonical(b"AACCGGTTAAG"); // own base G (2), sibling base C (1) — lives in layer 0 - - let a = annex_info_for(&merged, g1_kmer); - assert_eq!(a.layer_value(1), Some(1), "g1's own base: seeded with its own layer (1)"); - assert_eq!(a.layer_value(2), Some(0), "g1's sibling (g2's form): resolved to its real layer (0)"); - - let b = annex_info_for(&merged, g2_kmer); - assert_eq!(b.layer_value(2), Some(0), "g2's own base: seeded with its own layer (0)"); - assert_eq!(b.layer_value(1), Some(1), "g2's sibling (g1's form): resolved to its real layer (1)"); -} - -#[test] -fn subsample_and_shannon_on_one_variable_family() { - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let idx_of = |label: &str| merged.meta().genomes().unwrap().iter().position(|g| g.label == label).unwrap(); - let (i1, i2) = (idx_of("g1"), idx_of("g2")); - - // This fixture has exactly one non-monomorphic family (see - // `family_scan_consumers_agree_on_one_sibling_each`) — total_count = 1. - // Asking for far more than that must fall back to "keep everything", - // identical to the unsampled (`None`) result. - let sampled = merged.snp_pseudo_alignment(Some(1000), None).expect("snp_pseudo_alignment (subsample >> total)"); - assert_eq!(sampled.sequences[i1], vec![b'C']); - assert_eq!(sampled.sequences[i2], vec![b'G']); - - // Asking for 0 must keep nothing — an empty (but still one-row-per-genome) - // alignment, not an error. - let empty = merged.snp_pseudo_alignment(Some(0), None).expect("snp_pseudo_alignment (subsample 0)"); - assert!(empty.sequences[i1].is_empty()); - assert!(empty.sequences[i2].is_empty()); - - // shannon_entropy_csv: one row, both entropy15 and entropy4 = 1 bit - // exactly (2 genomes present, each carrying exactly one distinct base, - // uniform split -> -2*(0.5*log2(0.5)) = 1 — the two definitions - // coincide here since no genome carries more than one base). - let csv_path = dir.path().join("shannon.csv"); - merged.shannon_entropy_csv(&csv_path, Some(1000), None).expect("shannon_entropy_csv"); - let csv = std::fs::read_to_string(&csv_path).unwrap(); - let mut lines = csv.lines(); - assert_eq!(lines.next(), Some("layer,family_idx,entropy15,entropy4,family_size,n_genomes_present")); - let row = lines.next().expect("exactly one data row"); - assert!(lines.next().is_none(), "exactly one non-monomorphic family in this fixture"); - let fields: Vec<&str> = row.split(',').collect(); - let entropy15: f64 = fields[2].parse().unwrap(); - let entropy4: f64 = fields[3].parse().unwrap(); - assert!((entropy15 - 1.0).abs() < 1e-6, "entropy15 should be exactly 1 bit, got {entropy15}"); - assert!((entropy4 - 1.0).abs() < 1e-6, "entropy4 should be exactly 1 bit, got {entropy4}"); - assert_eq!(fields[4], "2", "family_size"); - assert_eq!(fields[5], "2", "n_genomes_present"); -} - -#[test] -fn sankoff_bundle_matches_old_separate_calls() { - // Same fixture as `sibling_annex_one_sibling_each`/ - // `subsample_and_shannon_on_one_variable_family`: one non-monomorphic - // family, g1='C' (minorant), g2='G'. Regression check that fusing - // raw_snp_distance/base_pair_tally/cardinality_tally/snp_pseudo_alignment - // into `sankoff_bundle`'s two passes produces identical results to the - // old four independent calls on `Selection::All` (no `--subsample`, - // no `--entropy`) — see `DevDocMD/architecture/siblings.md`, "Wired into - // `pack`...". - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let n_genomes = merged.meta().genomes().unwrap().len(); - let exclude_mask = vec![false; n_genomes]; - let ratio_ceiling = 0.5; - - let expected_raw = merged.raw_snp_distance().expect("raw_snp_distance"); - let expected_base = merged.base_pair_tally(&expected_raw, ratio_ceiling).expect("base_pair_tally"); - let expected_card = merged.cardinality_tally(&expected_raw, ratio_ceiling).expect("cardinality_tally"); - let expected_alignment = merged.snp_pseudo_alignment(None, None).expect("snp_pseudo_alignment"); - - let bundle = merged.sankoff_bundle(None, None, ratio_ceiling, &exclude_mask).expect("sankoff_bundle"); - - assert_eq!(bundle.raw.snp, expected_raw.snp); - assert_eq!(bundle.raw.shared, expected_raw.shared); - assert_eq!(bundle.base_pair_tally.counts, expected_base.counts); - assert_eq!(bundle.base_pair_tally.same, expected_base.same); - assert_eq!(bundle.cardinality_tally.counts, expected_card.counts); - assert_eq!(bundle.alignment.sequences, expected_alignment.sequences); -} - -#[test] -fn base_pair_tally_accumulates_base_a_diagnostic() { - // Diagnostic for a user-reported observation on real data: - // `composition_transitions`'s row/column for base 'A' (index 0) was - // *entirely* zero, including the diagonal (`same[0]`, "A stays A"), - // despite 'A' appearing at ~6% frequency in the pseudo-alignment - // itself. g1/g2 are identical (both central base 'A'), g3 differs - // (central base 'C') at the same family — makes the family variable - // (family_size=2) while g1/g2 form an eligible (ratio 0), ordinary - // pair both resolving `single_form` to `Some(0)` = 'A'. - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AAAAAAAAAAA"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AAAAAAAAAAA"); - let g3 = build_single_genome_index(dir.path(), "g3", b"AAAAACAAAAA"); - let mut rep = Reporter::new(); - let merged = KmerIndex::merge( - &dir.path().join("merged.idx"), - &[&g1, &g2, &g3], - MergeMode::Presence, - false, - false, - 1.0, - &mut rep, - ).expect("merge"); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let n_genomes = merged.meta().genomes().unwrap().len(); - let exclude_mask = vec![false; n_genomes]; - let bundle = merged.sankoff_bundle(None, None, 0.5, &exclude_mask).expect("sankoff_bundle"); - - assert!( - bundle.base_pair_tally.same[0] >= 1, - "base 'A' (index 0) must accumulate in BasePairTally.same for an identical, included g1/g2 \ - pair both resolving to 'A' — got same={:?}, counts={:?}", - bundle.base_pair_tally.same, bundle.base_pair_tally.counts, - ); -} - -#[test] -fn entropy_annex_builds_on_demand_and_biases_selection() { - // Same fixture, same single non-monomorphic family, known entropy15 = - // 1.0 bit exactly (see `subsample_and_shannon_on_one_variable_family`). - let dir = tempdir().unwrap(); - let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); - let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); - let merged = merge_two(dir.path(), &g1, &g2); - merged.build_sibling_annex().expect("build_sibling_annex"); - - let layer_dirs = super::family_scan::sibling_layer_dirs(&merged).expect("layer dirs"); - for layer_dir in &layer_dirs { - assert!( - !layer_dir.join(ENTROPY_ANNEX_FILE_NAME).exists(), - "entropy annex must not exist before any entropy-biased selection is requested" - ); - } - - // This fixture spans 2 layers (a single-genome bootstrap layer plus the - // merged one, see `merge_two`) — the one non-monomorphic family lives - // in whichever layer actually got a non-empty selection, not - // necessarily `layer_dirs[0]`. - let find_layer = |selections: &[super::subsample::LayerSelection]| { - selections.iter().position(|s| matches!(s, Some(set) if !set.is_empty())) - }; - - // mu == the family's real entropy, tiny sigma -> weight == 1 exactly, - // p0 == 1.0 (no `--subsample`) -> the family is selected with - // probability 1 (up to a measure-zero event on the uniform draw). - let bias_center = EntropyBias { mu: 1.0, sigma: 0.5 }; - let selections = super::subsample::compute_selections(&merged, &layer_dirs, None, Some(bias_center)) - .expect("compute_selections (entropy-biased)"); - for layer_dir in &layer_dirs { - assert!( - layer_dir.join(ENTROPY_ANNEX_FILE_NAME).exists(), - "compute_selections must build the entropy annex for every layer on first use" - ); - } - let li = find_layer(&selections).expect("mu at the true entropy must select the one non-monomorphic family, in some layer"); - assert_eq!(selections[li], Some(std::collections::HashSet::from([0]))); - - // The annex now exists — read it back directly and check the stored - // value matches `--shannon`'s own entropy15 computation exactly. - let entropy_annex = EntropyAnnex::open(&layer_dirs[li].join(ENTROPY_ANNEX_FILE_NAME)).expect("open entropy annex"); - let stored = entropy_annex.get(0).expect("family 0 has a real (non-sentinel) entropy value"); - assert!((stored as f64 - 1.0).abs() < 1e-6, "stored entropy should be 1 bit, got {stored}"); - - // mu far from any achievable entropy, tiny sigma -> weight underflows - // to exactly 0.0 -> never selected, deterministically, in any layer. - let bias_far = EntropyBias { mu: 100.0, sigma: 0.001 }; - let selections_far = super::subsample::compute_selections(&merged, &layer_dirs, None, Some(bias_far)) - .expect("compute_selections (entropy-biased, far)"); - assert!(find_layer(&selections_far).is_none(), "mu far from the true entropy must never select it, in any layer"); -} - -- 2.54.1 From 95bbc0b2ae4764001cb4017e0870a7f6d14193e4 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Fri, 18 Sep 2026 19:17:59 +0200 Subject: [PATCH 08/10] Implement sparse matrix optimizations and update architecture documentation This commit introduces significant functional improvements related to sparse matrix handling, including new storage layers, performance fixes for sparse operations, and bug corrections in matrix logic. Additionally, architectural documentation is cleaned up by removing obsolete entries and modifying structural text related to the Sibling Annex discussion. --- DevDoc/404.html | 2 +- .../index_architecture.refs/index.html | 2 +- .../index_architecture/index.html | 2 +- .../numa_partition_runner/index.html | 2 +- .../architecture/numa_worker_pools/index.html | 2 +- DevDoc/architecture/query.refs/index.html | 2 +- DevDoc/architecture/query/index.html | 2 +- DevDoc/architecture/rebuild_filter/index.html | 2 +- .../sequences/invariant.refs/index.html | 2 +- .../sequences/invariant/index.html | 2 +- DevDoc/architecture/siblings/index.html | 1413 +++-------------- .../benchmark_query_testing/index.html | 7 +- .../chunkreader.refs/index.html | 2 +- DevDoc/implementation/chunkreader/index.html | 2 +- .../evidence_elimination.refs/index.html | 2 +- .../evidence_elimination/index.html | 2 +- DevDoc/implementation/filtering/index.html | 2 +- DevDoc/implementation/kmer.refs/index.html | 2 +- DevDoc/implementation/kmer/index.html | 2 +- .../implementation/layer_tier.refs/index.html | 2 +- DevDoc/implementation/layer_tier/index.html | 4 +- DevDoc/implementation/merge.refs/index.html | 2 +- DevDoc/implementation/merge/index.html | 2 +- .../merge_parallelism/index.html | 2 +- DevDoc/implementation/mphf.refs/index.html | 2 +- DevDoc/implementation/mphf/index.html | 2 +- .../implementation/obicompactvec/index.html | 222 ++- .../obipipeline.refs/index.html | 2 +- DevDoc/implementation/obipipeline/index.html | 2 +- DevDoc/implementation/obitaxonomy/index.html | 2 +- .../partition_layer_cache/index.html | 2 +- .../persistent_bit_vec.refs/index.html | 2 +- .../persistent_bit_vec/index.html | 2 +- .../index.html | 2 +- .../persistent_compact_int_vec/index.html | 2 +- .../implementation/pipeline.refs/index.html | 2 +- DevDoc/implementation/pipeline/index.html | 2 +- DevDoc/implementation/select/index.html | 2 +- DevDoc/implementation/storage.refs/index.html | 2 +- DevDoc/implementation/storage/index.html | 2 +- .../implementation/superkmer.refs/index.html | 2 +- DevDoc/implementation/superkmer/index.html | 2 +- .../unitig_evidence.refs/index.html | 2 +- .../implementation/unitig_evidence/index.html | 2 +- DevDoc/index.html | 2 +- DevDoc/index.refs/index.html | 2 +- DevDoc/installation/index.html | 2 +- DevDoc/kmers.refs/index.html | 2 +- DevDoc/kmers/index.html | 2 +- DevDoc/sitemap.xml.gz | Bin 127 -> 127 bytes DevDoc/theory/encoding.refs/index.html | 2 +- DevDoc/theory/encoding/index.html | 2 +- DevDoc/theory/entropy.refs/index.html | 2 +- DevDoc/theory/entropy/index.html | 2 +- .../theory/evolutionary_distances/index.html | 598 ++++++- DevDoc/theory/indexing.refs/index.html | 2 +- DevDoc/theory/indexing/index.html | 2 +- DevDoc/theory/minimizer.refs/index.html | 2 +- DevDoc/theory/minimizer/index.html | 2 +- DevDocMD/architecture/siblings.md | 1184 ++------------ .../implementation/benchmark_query_testing.md | 5 +- DevDocMD/implementation/layer_tier.md | 2 +- DevDocMD/implementation/obicompactvec.md | 161 +- mkdocs.yml | 2 +- 64 files changed, 1374 insertions(+), 2330 deletions(-) diff --git a/DevDoc/404.html b/DevDoc/404.html index b04c63bb..5a9ae5cb 100644 --- a/DevDoc/404.html +++ b/DevDoc/404.html @@ -1119,7 +1119,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/index_architecture.refs/index.html b/DevDoc/architecture/index_architecture.refs/index.html index 78ae7534..f19ad23b 100644 --- a/DevDoc/architecture/index_architecture.refs/index.html +++ b/DevDoc/architecture/index_architecture.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/index_architecture/index.html b/DevDoc/architecture/index_architecture/index.html index 30ea415b..09497c21 100644 --- a/DevDoc/architecture/index_architecture/index.html +++ b/DevDoc/architecture/index_architecture/index.html @@ -1370,7 +1370,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/numa_partition_runner/index.html b/DevDoc/architecture/numa_partition_runner/index.html index c79dca08..653ab74a 100644 --- a/DevDoc/architecture/numa_partition_runner/index.html +++ b/DevDoc/architecture/numa_partition_runner/index.html @@ -1128,7 +1128,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/numa_worker_pools/index.html b/DevDoc/architecture/numa_worker_pools/index.html index 97421f9a..10674165 100644 --- a/DevDoc/architecture/numa_worker_pools/index.html +++ b/DevDoc/architecture/numa_worker_pools/index.html @@ -1130,7 +1130,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/query.refs/index.html b/DevDoc/architecture/query.refs/index.html index ef859997..141c734a 100644 --- a/DevDoc/architecture/query.refs/index.html +++ b/DevDoc/architecture/query.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/query/index.html b/DevDoc/architecture/query/index.html index 2ed91fb1..a2012d4a 100644 --- a/DevDoc/architecture/query/index.html +++ b/DevDoc/architecture/query/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/rebuild_filter/index.html b/DevDoc/architecture/rebuild_filter/index.html index a21d25de..04a44fdc 100644 --- a/DevDoc/architecture/rebuild_filter/index.html +++ b/DevDoc/architecture/rebuild_filter/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/sequences/invariant.refs/index.html b/DevDoc/architecture/sequences/invariant.refs/index.html index 2d5e0338..66ddfc59 100644 --- a/DevDoc/architecture/sequences/invariant.refs/index.html +++ b/DevDoc/architecture/sequences/invariant.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/sequences/invariant/index.html b/DevDoc/architecture/sequences/invariant/index.html index 38f3fb60..7ce2bce5 100644 --- a/DevDoc/architecture/sequences/invariant/index.html +++ b/DevDoc/architecture/sequences/invariant/index.html @@ -1140,7 +1140,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/architecture/siblings/index.html b/DevDoc/architecture/siblings/index.html index 8be09b91..74189217 100644 --- a/DevDoc/architecture/siblings/index.html +++ b/DevDoc/architecture/siblings/index.html @@ -23,7 +23,7 @@ - Sibling annex (discussion) - obikmer + Sibling annex - obikmer @@ -64,7 +64,7 @@
- + Skip to content @@ -100,7 +100,7 @@
- Sibling annex (discussion) + Sibling annex
@@ -1139,7 +1139,7 @@ - Sibling annex (discussion) + Sibling annex @@ -1157,7 +1157,7 @@ - Sibling annex (discussion) + Sibling annex @@ -1183,10 +1183,10 @@
  • - + - Two index spaces, uncorrelated + Data model @@ -1194,10 +1194,10 @@
  • - + - Two pipelines, never mixed + Construction @@ -1205,10 +1205,10 @@
  • - + - Sibling annex: an iteration-pipeline artifact only + Shared traversal @@ -1216,10 +1216,10 @@
  • - + - Pending work — done + Masking — computed once, shared everywhere @@ -1227,10 +1227,10 @@
  • - + - Performance: build_sibling_annex parallelism (2026-08-14) + Precomputed annexes (entropy, family overlap) @@ -1238,10 +1238,10 @@
  • - + - Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14) + Sampling and the shared scan trunk @@ -1249,10 +1249,10 @@
  • - + - Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15) + From a tally to distances and calibration @@ -1260,10 +1260,10 @@
  • - + - --subsample / --shannon — sampling strategy (decided 2026-08-15) + Public API surface @@ -1271,109 +1271,10 @@
  • - + - Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15) - - - - -
  • - -
  • - - - - --free-loss/--tnt pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below) - - - - -
  • - -
  • - - - - Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15) - - - - -
  • - -
  • - - - - Two entropy definitions kept side by side, for comparison (2026-08-15) - - - - -
  • - -
  • - - - - PersistentSparseBitMatrix — implemented and measured (2026-08-15) - - - - -
  • - -
  • - - - - Wired into pack and the sibling-annex build path (2026-08-15) - - - - -
  • - -
  • - - - - Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15) - - - - -
  • - -
  • - - - - query never benefits from sparse row-major access (found 2026-08-19, not implemented) - - - - -
  • - -
  • - - - - Implemented (2026-08-20) - - - - -
  • - -
  • - - - - PersistentCompactIntMatrix::Sparse — implemented (2026-08-26) + Known gaps @@ -1479,10 +1380,10 @@
    • - + - Two index spaces, uncorrelated + Data model @@ -1490,10 +1391,10 @@
    • - + - Two pipelines, never mixed + Construction @@ -1501,10 +1402,10 @@
    • - + - Sibling annex: an iteration-pipeline artifact only + Shared traversal @@ -1512,10 +1413,10 @@
    • - + - Pending work — done + Masking — computed once, shared everywhere @@ -1523,10 +1424,10 @@
    • - + - Performance: build_sibling_annex parallelism (2026-08-14) + Precomputed annexes (entropy, family overlap) @@ -1534,10 +1435,10 @@
    • - + - Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14) + Sampling and the shared scan trunk @@ -1545,10 +1446,10 @@
    • - + - Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15) + From a tally to distances and calibration @@ -1556,10 +1457,10 @@
    • - + - --subsample / --shannon — sampling strategy (decided 2026-08-15) + Public API surface @@ -1567,109 +1468,10 @@
    • - + - Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15) - - - - -
    • - -
    • - - - - --free-loss/--tnt pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below) - - - - -
    • - -
    • - - - - Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15) - - - - -
    • - -
    • - - - - Two entropy definitions kept side by side, for comparison (2026-08-15) - - - - -
    • - -
    • - - - - PersistentSparseBitMatrix — implemented and measured (2026-08-15) - - - - -
    • - -
    • - - - - Wired into pack and the sibling-annex build path (2026-08-15) - - - - -
    • - -
    • - - - - Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15) - - - - -
    • - -
    • - - - - query never benefits from sparse row-major access (found 2026-08-19, not implemented) - - - - -
    • - -
    • - - - - Implemented (2026-08-20) - - - - -
    • - -
    • - - - - PersistentCompactIntMatrix::Sparse — implemented (2026-08-26) + Known gaps @@ -1693,1008 +1495,151 @@ -

      Sibling annex — architecture (discussion)

      -

      Status: architecture decided (2026-08-14). Implementation not yet mandated.

      -

      Two index spaces, uncorrelated

      -

      Every kmer stored in a Layer lives in two independent index spaces:

      +

      Sibling annex & SNP-family pipeline — architecture

      +

      Current state of obikphylo::siblings (crate obikphylo, module siblings). +Historical rationale, dated findings, and superseded designs live in jj +history, not here.

      +

      Data model

      +

      A family is the up-to-4 k-mers sharing all flanks and differing only at +the central base (well-defined for odd k). FamilyMask (siblingannex.rs) +is a 4-bit presence mask (bit b set iff the family member whose own +canonical central base is b is observed anywhere in the index) plus a +minorant flag; family_size()/siblings() derive member count from it. +SiblingAnnex/SiblingAnnexBuilder persist one FamilyMask per k-mer, +keyed by iteration order (the physical layout of unitigs.bin), never by +MPHF slot — this lets every reader zip-iterate a layer's own k-mer stream +against the annex file directly, no MPHF/slot indirection.

      +

      Construction

      +

      algorithms::annex::build_layer_sibling_annex builds one layer's annex: +seeds each k-mer's own central base, generates its up-to-3 central-base +variants, resolves each cross-partition (IndexCache::find_in_partition, +grouped and chunked per destination partition for mmap locality), writes the +final mask (with the minorant flag) in a second streamed pass. +SiblingExt::build_sibling_annex drives this for every layer of every +partition, then invalidates and rebuilds the entropy and family-overlap +annexes (their family_idx numbering depends on the sibling annex).

      +

      Shared traversal

      +

      algorithms::family_scan::scan_layer_families is the only place that +resolves a family's full per-genome presence (genome_mask: &[u8], +cross-partition) — every other consumer in this module is built on it, never +reimplements the resolution. Two decoupled concurrency mechanisms: batch +generation (CPU-only) runs on an obipipeline throttle/pipe; per-batch +resolution (I/O-bound, mmap page faults) runs rayon-parallel across +destination partitions, one generated batch at a time (never several +concurrently, to keep one partition's pages hot instead of scattering +access). fast_mode (is_fast_mode, <= 7 layers per partition) picks +between trusting a family member's recorded layer index outright or probing +every layer of the destination partition.

      +

      Selection<'a> bounds which families of a layer are resolved:

        -
      • Iteration order: its position when enumerating unitigs.bin (the - superkmer file), deterministic but arbitrary with respect to slot.
      • -
      • MPHF slot: MphfLayer::index(kmer), the number the MPHF assigns.
      • +
      • All — every minorant, monomorphic included (SiblingExt::shannon_entropy_csv, + sibling_annex_stats).
      • +
      • Some(&HashSet<usize>) — a caller-picked subset, skipping the expensive + cross-partition resolution entirely for excluded families + (subsample::sample_layer's accepted-candidates batch; + entropy::ensure_layer_entropy_annex's non-monomorphic-only restriction).
      -

      The two are not correlated by any formula. Converting from one to the other -requires either recomputing the MPHF (kmer → slot) or scanning the iteration -stream (kmer → order). There is no slot → kmer operation: the MPHF is a -one-way function, not an invertible bijection with a stored inverse. Any -method that reconstructs a kmer from a bare slot number is wrong by -construction, regardless of the mechanism used (MPHF re-hash, or evidence -decode + direct unitig read). See MphfLayer::kmer_at -(obikindex/src/layer/mphf_layer.rs) — flagged for removal, currently called -from obikphylo/siblings/build.rs and family_scan.rs (since removed — see -"Pending work" status below).

      -

      Two pipelines, never mixed

      - - - - - - - - - - - - - - - - - - - - - - - -
      origin of the kmermembership known?correct mapping
      query pipelineexternal (caller-supplied)noquery/find/find_strict — MPHF + evidence check
      iteration pipelineenumerated from this layer's own unitigs.binyes, by constructionindex/index_batch — MPHF only, no evidence
      -

      Evidence exists solely to answer "is this external kmer a member of the -layer" for the query pipeline. Using it (or the MPHF) to go the other way — -recover a kmer from a slot, or re-verify a kmer that was just produced by -iterating the layer — is a conceptual error: evidence can be probabilistic -(Approx mode), so any slot→kmer attempt is unsound in general, and -pointless even in Exact/Hybrid mode since the kmer was already known.

      -

      Sibling annex: an iteration-pipeline artifact only

      -

      The sibling annex (FamilyMask/SiblingAnnex, .psib, -obicompactvec/src/siblingannex.rs) records, per kmer, whether it is a -family minorant and which family members are present in the index. Its only -consumers (obikphylo/siblings/stats.rs, family_scan.rs) enumerate it -exhaustively (0..annex.len()); no query-pipeline code path touches it.

      -

      Decision: the annex must be persisted in iteration order, not slot -order. This lets readers zip-iterate Layer::iter_kmers() and the annex -file directly — one linear, cache-friendly pass, no MPHF/slot indirection, -no kmer_at. It also enables specialized iterators building on this zip: -minorants-only iteration, batch-of-kmers → batch-of-family-members, etc.

      -

      Today the annex is built and stored in slot order -(build_layer_sibling_annex, siblings/build.rs): slot_kmer is populated -via (0..n_slots).map(|slot| mphf.kmer_at(slot)), and the origin slot is -threaded through the whole cross-partition reconciliation pipeline (variant -generation, query_partition_with, final mask[slot].fetch_or(...)). This -must change to iterating iter_kmers()/enumerate_kmers() and threading -the iteration index instead of the slot end to end — eliminating -kmer_at from the build path entirely, not just the read path. No -slot-indexed intermediate is needed even during construction; the -iteration-order id is sufficient throughout.

      -

      The cross-partition side of the same pipeline is unaffected: checking -whether a generated family-variant kmer exists in another partition is a -genuine query-pipeline operation (the variant's membership in the target -partition is unknown) and must keep going through -KmerPartition::query_partition_with (MPHF + evidence), never a raw -index().

      -

      Pending work — done

      -

      The plan above shipped: obikphylo (a new crate — phylo-domain extension -traits over obikindex::KmerIndex/obikindex::layer::Layer<D>, replacing the -old obikindex::siblings module) builds and reads the annex purely in -iteration order (SiblingLayerExt::iter_siblings/iter_minorants, both with -batch variants, mirroring Layer<D>'s own KmerIter/KmerBatchIter -shape). MphfLayer::kmer_at has no remaining callers.

      -

      A separate, unrelated bug surfaced during this work and was fixed -(2026-08-14): MphfLayer::enumerate_kmers_batch computed its -batch_start_index via the stdlib .enumerate() adapter, which counts -batches (0, 1, 2…), not the cumulative k-mer offset the annex is actually -keyed on — every batch past the first wrote its mask/annex entries at the -wrong iteration-order position. Fixed by tracking a running offset instead; -regression tests added (sibling_annex_no_empty_masks_after_build, -sibling_histogram_does_not_panic_on_partial_last_batch).

      -

      Performance: build_sibling_annex parallelism (2026-08-14)

      -

      Investigated on a real multi-genome run (phyloskims_sal_vac, k=31/m=11). -Baseline: mostly one active core, with short multi-core bursts — average -~3 cores.

      -

      Fixes that helped, kept:

      +

      minorant_selection::eligible_bitset_layer/non_monomorphic_reindex_layer +provide the cheap, annex-bits-only (no cross-partition resolution) filters +these restricted scans are built from: monomorphism (family_size() < 2) is +knowable straight from the annex, so nothing pays scan_layer_families's +resolution cost just to discard a monomorphic result.

      +

      Masking — computed once, shared everywhere

      +

      algorithms::masking::masked_state(m, free_loss, no_ambiguity) is the single +function that turns a raw per-genome 4-bit state into the masked view every +consumer needs (None for absence under --free-loss or ambiguity under +--no-ambiguity). It is called exactly once per (family, genome), in +subsample::sample_layer, immediately after scan_layer_families resolves +the raw state — never recomputed downstream. survives_masking (does the +family still carry ≥2 distinct masked states among included genomes?) and +SurvivingFamily::masked: Vec<Option<u8>> both consume this same +precomputed array, which is also what AlignmentReducer/PairwiseReducer +read (see below) — this is what guarantees the alignment and the pairwise +tally can never disagree about a genome's state at a given site.

      +

      Precomputed annexes (entropy, family overlap)

      +

      algorithms::entropy::ensure_layer_entropy_annex builds EntropyAnnex +(entropy_annex.rs, one f32 Shannon-entropy value per non-monomorphic +minorant, mmap-backed) and/or feeds FamilyOverlapAccumulator +(family_overlap.rs) from one shared Selection::Some scan — both need the +same genome_mask resolution, so a missing one never pays for it twice. +family_entropy (15 non-empty states of {A,C,G,T}'s powerset, ∅ +excluded from the denominator — the project's settled informativeness +definition, matching the calibrated 16-state Sankoff cost matrix) and +family_entropy_4 (plain 4-symbol reduction, kept only for comparison) are +pure functions of genome_mask. extensions::SiblingBuilder::ensure_entropy_and_overlap_annexes +drives this for every cached layer, lazily (only what's missing).

      +

      Sampling and the shared scan trunk

      +

      algorithms::subsample::sample_index draws up to n non-monomorphic +families index-wide, proportionally per layer: for each layer, a target +quota sized from that layer's share of the index-wide eligible count, filled +by a circular walk with a Bernoulli(p) draw per candidate — p optionally +weighted by an unnormalized Gaussian kernel over the candidate's entropy +(EntropyBias { mu, sigma }, read from the entropy annex). Accepted +candidates are resolved in batches (scan_layer_families) and checked +against survives_masking; rejected-after-masking candidates don't count +toward quota, triggering further "topup" rounds (bounded, MAX_TOPUP_ROUNDS += 5) rather than silently under-filling. Every SurvivingFamily reaching a +caller has already survived both the draw and post-masking informativeness — +no second filter pass needed downstream.

      +

      algorithms::scan::sampled_scan is the single entry point every sampled +computation goes through: one sample_index pass, fanned out to whichever +LayerReducer implementations the caller actually needs, each receiving the +exact same per-layer SurvivingFamily batch (an Arc, cloned not copied). +This is what guarantees several outputs requested together +(--pseudo-alignment and --sankoff and a snp-* --distance in one CLI +invocation) come from one random sample, not independently-drawn ones — +there is no other driver of sample_index anywhere in this crate.

      +

      Two LayerReducer implementations exist today:

        -
      • CanonicalKmerOf::minimizer() (obikseq/src/kmer.rs) — a direct O(k) - bit-arithmetic minimiser for a single isolated k-mer, replacing a - RollingStat instance fed byte-by-byte through an ASCII round-trip (used - by helpers::partition_of, called for every generated family variant). - ~3x wall-clock improvement on its own, confirmed by sampling - (obiskbuilder::rolling_stat/obikentropy frames disappeared from the - hot path). CanonicalKmerOf::partition() added alongside it (wraps - minimizer().seq_hash() & mask, the same routing rule - KmerPartition/RoutableSuperKmer use).
      • -
      • Cross-partition resolution (outgoing.par_iter() in - build_layer_sibling_annex) parallelised at the partition level — one - Rayon task per non-empty outgoing[dest] bucket. For k=31/m=11, a - central-base substitution changes the winning minimiser (and thus the - destination partition) only when that window overlaps the central base: - ~11 of the 21 possible windows do, so ~10/21 (≈48%) of generated variants - route right back to the partition already being built. That self bucket - ends up far larger than any other, so the per-partition split pinned one - thread to it alone while the rest of the pool finished instantly — - confirmed by sampling: one thread solid in MphfLayer::find, everyone - else idle. Fixed by splitting each non-empty bucket into - total_queries / n_workers (capped 4096) chunks before par_iter(), - preserving per-partition mmap locality (each chunk stays contiguous - within one partition) while letting Rayon spread an oversized bucket - across several threads. Net effect of both fixes together: ~3 cores - average → ~10-13 cores average on the same run, and a projected total - build time of ~1h15 down to ~30min on the real phyloskims_sal_vac run - this was measured against.
      • -
      • TracedBar's ETA (obisys/src/progress.rs) was silently starved: the - custom progress message and the self-computed ETA text used to share one - pb.set_message() slot, with the ETA holding off for 2s after any custom - message — fine when custom messages are rare, broken once - build_sibling_annex's per-partition callback fires more often than - that. Fixed by keeping the two texts in separate fields, composed - together on every render instead of one overwriting the other.
      • +
      • AlignmentReducer (alignment.rs) — accumulates SnpAlignment (one + IUPAC-coded row per included genome), reading family.masked directly.
      • +
      • PairwiseReducer (pairwise.rs) — accumulates PairwiseTally/ + PartitionDispersion, also reading family.masked directly (substitution/ + agreement counts and cardinality both derive from the same masked state + the alignment uses).
      -

      Tried and reverted — do not repeat blindly:

      +

      From a tally to distances and calibration

      +

      PairwiseTally stores per-genome-pair substitution/agreement/cardinality +counts, unfiltered, from which every downstream output is cheap O(n²) +post-processing (never a second scan):

        -
      • Parallelising the outer partition loop in build_sibling_annex with - obikindex::PartitionRunner (already used by merge/build_layers), - splitting a fixed core budget between outer (partition) and inner - (pipeline + resolution) concurrency so their product wouldn't exceed the - budget. Measured worse: throughput dropped over time (26 - partitions/5min → 38/11-12min) and peak resolution concurrency fell from - ~11-12 cores to ~7-8. Cause: this capped the resolution burst — which - scales very well on its own — to make room for outer concurrency, and - running several partitions' resolution at once scatters access across - multiple partitions' mmap regions at once, working against the - locality outgoing's per-partition grouping exists for. PartitionRunner - stayed exported from obikindex (new_capped too) since it's - general-purpose, but nothing in obikphylo calls it.
      • -
      • Splitting resolution chunks even finer (/(n_workers*8), cap 1024, - instead of /n_workers, cap 4096) to smooth the residual sawtooth. - Measured ~10% slower, wider dips, not narrower. Reverted to the - original chunk sizing.
      • +
      • algorithms::snp_distance::distance_matrix — closed-form snp-* + corrections (raw p-distance, JC69, K80, K81, F81, T92, TN93, transversions- + only), optionally gamma-corrected (GammaShape, PartitionDispersion::estimate_alpha + for Auto).
      • +
      • algorithms::sankoff::SankoffBundle::finish — assembles the + --sankoff/--tnt/--phyg/--iqtree pipeline's inputs + (RawSnpDistanceOutput, BasePairTally, CardinalityTally) from an + already-scanned AlignmentReducer/PairwiseReducer pair, plus + SankoffBundle::snp_distance to reuse that exact sample for a snp-* + --distance in the same invocation without a second scan.
      • +
      • algorithms::cardcomp — pure functions turning CardinalityTally/ + BasePairTally into row-stochastic transition probability matrices, then + the calibrated 16-state Sankoff cost matrix (pairwise_cost_matrix).
      -

      Known remaining limitation, not yet worth fixing: within one layer, the -four stages (sequential unitigs.bin read → parallel generation → -parallel resolution → sequential annex write) never overlap — confirmed by -1s-interval sampling: generation alone occupies ~17 threads evenly, but the -next layer's read/generation never starts until the current layer's -resolution and write are both done. This produces a real, periodic (~layer -duration) alternation between "many cores" and "few cores" that neither of -the fixes above touches, since both operate within one layer's resolution -step. The only remaining lever is overlapping consecutive layers (e.g. a -depth-2 pipeline: start layer N+1's read/generation while layer N's -resolution/write is still running) — a real restructuring, not a parameter -tweak, and explicitly not to be combined with the reverted -budget-capping idea above (let each phase use however many cores it -naturally wants; only the scheduling needs to overlap). Deferred, not -started.

      -

      Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14)

      -

      family_scan.rs::scan_layer_families (shared by snp_pseudo_alignment, -sibling_annex_stats, cardinality_tally, scan_family_pairs) already -implements most of a dispatch/accumulate/resolve pipeline: generation -(cheap, CPU-only — builds outgoing[dest_partition] from FamilyMask and -buckets cross-partition queries) runs on an obipipeline::throttle + -make_pipe! stage, decoupled from resolution (I/O-bound, rayon::par_iter -across partitions, one generated batch resolved at a time, never several -concurrently — this ordering is deliberate, see the module's own docs on a -reverted concurrent-batch-resolution attempt that scattered mmap access). -The fast/slow mode gate (PartitionCache::fast_mode, cache.rs:162-163) -already exists: n_layers <= 7 (checked once from the first non-empty -partition's PartitionMeta::n_layers, documented as identical across every -partition of an index — a structural, build-time property, never a -per-partition state) decides whether FamilyMask's recorded layer_value -can be trusted to skip straight to the right layer -(find_presence_batch_fast) or must fall back to scanning every layer of -the destination partition (find_presence_batch).

      -

      Real gap, confirmed not implemented: resolution is triggered by the -source batch finishing (FAMILY_BATCH = 65536 minorants read from the -scanned layer), not by an output accumulator filling up. Since most -central-base variants of a family route back to the same partition being -scanned (~48% per the k=31/m=11 measurement above), a FAMILY_BATCH's -outgoing[dest] is large for the local/self partition and thin for the -other ~255 (or however many) destination partitions — each of those gets -resolved at low query density every batch instead of being accumulated -across several source batches until resolving it is worthwhile. This is -distinct from, and not fixed by, the fast/slow layer gate above.

      -

      Redesign sketched (not built): per-destination accumulators decoupled from -FAMILY_BATCH, flushed on reaching a size threshold instead of on source-batch -completion — a "hot" accumulator for the partition being scanned (sharded -one-per-generation-worker, no lock, since all n_workers pipeline workers -write to it concurrently — this differs from an earlier, simpler mental -model of "one thread owns one layer's local collector," which doesn't hold -here since n_workers threads cooperate on scanning one layer at a time, -not one thread per layer) and "cold" mutex-per-partition accumulators for -the rest, low contention expected since traffic to any single cold -destination is a small fraction of total.

      -

      This breaks the current strict-iteration-order delivery of on_family -(today: a reorder buffer keyed by batch, since a whole FAMILY_BATCH -resolves atomically). With cross-batch accumulation, a family only becomes -complete once every accumulator holding one of its outgoing queries has -flushed, at unpredictable, independent times — no longer streamable -strictly in order without a large, unbounded pending buffer. Resolution -sketched: replace order-dependent consumers with coordinate-addressed -writes instead of order-dependent appends (see PseudoAlignment idea -below) wherever possible, since sibling_annex_stats's reduction (plain -counts) is already order-independent and needs nothing here.

      -

      Superseded 2026-08-15 by the --subsample/--shannon design below, -which sidesteps the accumulator redesign for now: bounding the number of -families actually resolved per layer (via sampling) keeps per-layer -resolution volume small enough that the batch-density problem above stops -mattering in practice for these two consumers. The accumulator redesign -remains relevant for a future unsampled, full-index run, but is not -required to ship --subsample/--shannon.

      -

      Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15)

      -

      The reference run (phyloskims_sal_vac-scale bacterial test set, -iqtree.fasta) produced a dense alignment for 13 genomes × 383,965 sites -(4.8 MB) — trivially small. The in-progress plant index is expected to -carry on the order of 9 billion minorant families; a dense byte-per-cell -alignment at that column count is unbuildable regardless of genome count -(hundreds of GB even at a handful of genomes). Long-term ambition is 6,000– -8,000 genomes on a large machine, which makes the per-cell cost dominant in -the other dimension too. Pruning the retained family set before -materializing anything is mandatory, not an optimization.

      -

      Already free: family_size() < 2 (no sibling variant registered at -all) is a zero-cost structural filter, read directly off FamilyMask bits, -already applied in snp_pseudo_alignment. Insufficient alone — per the -~80-85% mono-family estimate from earlier discussion, this only brings 9 -billion down to roughly 1.3-1.8 billion, still unusable.

      -

      Entropy definition — settled 2026-08-15, correcting an earlier wrong -turn. The project does not encode families as IUPAC ambiguity codes -interpreted the classical way (Fitch-parsimony subset-compatibility, or -ML's "one true state, uncertain which"); see -docmd/theory/evolutionary_distances.md ("Why the IUPAC/DNA encoding used -for the first --snp test was wrong") and the Sankoff resolution that -followed it. The real model is a genuine 16-state alphabet (the powerset of -{A,C,G,T}, ∅ included as a real state) scored with a calibrated -pairwise cost matrix (obikphylo::cardcomp::pairwise_cost_matrix, -cmd/phylo/sankoff.rs), not a compatibility/subset relation between -states. Under that model, each of the 16 states — including multi-bit ones -like AC — is a first-class, independently-costed state, not an -uncertainty encoding of a single true base. So: entropy over the 15 -non-empty states (∅ excluded, matching the earlier decision to exclude -genomes where the family is absent) is the correct informativeness -measure for this project — not a 4-symbol reduction, which would discard -exactly the cardinality/composition information the calibrated cost matrix -is built to exploit.

      -

      --subsample / --shannon — sampling strategy (decided 2026-08-15)

      -

      Goal: make both the pseudo-alignment (--snp) and a Shannon-entropy -diagnostic usable at any index scale, from the 13-genome bacterial -reference run up to the 9-billion-family plant index, without requiring the -batched-accumulator redesign above.

      -

      --subsample N (integer, families to retain): bounds the pseudo-alignment -to N minorant families, sampled proportionally per layer among -non-monomorphic minorants (family_size >= 2) — this sidesteps the need -for a true global reservoir merge across layers while still approximating a -uniform sample over the whole index, and directly answers the earlier open -question of global-vs-per-layer selection scope.

      -

      Three passes, in order:

      -
        -
      1. Global count (cheap, structural, parallel across layers — same shape - as the existing sibling_family_size_histogram, extended to report a - per-layer breakdown rather than one index-wide aggregate): for each - layer, count_layer = number of non-monomorphic minorants. Gives - total_count = Σ count_layer.
      2. -
      3. Per-layer proportional reservoir sampling (cheap, structural, one - pass per layer, no cross-partition resolution): N_layer = round(N × - count_layer / total_count). Since N_layer is a proportion of - count_layer, it can never exceed it as long as N <= total_count — the - one edge case is total_count <= N, in which case sampling is skipped - entirely and every non-monomorphic minorant of every layer is kept - (no reservoir needed, N was never a real constraint). Otherwise: - Algorithm-R reservoir sampling over the layer's non-monomorphic minorant - indices, producing N_layer iteration-order indices directly, no - intermediate full list ever materialized.
      4. -
      5. Filtered resolution (the expensive step, the existing - scan_layer_families engine, unchanged): re-scan the layer, generating - and resolving cross-partition queries only for the indices selected - in step 2 (cheap membership test against a small per-layer index set) — - this is what keeps --subsample cheap even on an unsampled-scale index, - since the cross-partition resolution volume is bounded by N, not by - the layer's true size.
      6. -
      -

      Steps 2 and 3 cannot be merged into one pass: true single-pass reservoir -sampling would waste step-3's expensive resolution work on candidates later -evicted by the reservoir. Step 1 must fully complete (every layer) before -step 2 can start for any layer, since total_count is a global quantity.

      -

      --shannon (no argument): emits a CSV of per-family Shannon entropy -(15 non-empty states, ∅/absent genomes excluded from the denominator, per -the settled definition above). Independent of --subsample — entropy is -computed and written per family as soon as its genome_mask resolves, -O(1) memory per family, so it streams fine even unsampled at full index -scale (a time cost, not a memory one). Combined with --subsample N, it -delivers the original exploratory diagnostic (e.g. --subsample 1000000 ---shannon) directly from this general machinery, rather than a -purpose-built one-off script.

      -

      Validated end-to-end (2026-08-15) against real data: --sibling-hist on -phyloskims_sal_vac (91 real genomes, k=31/m=11, 256 partitions × 2 -layers) confirms the ~9-billion-family estimate almost exactly (8,925,068,238 -total, 97.9% monomorphic — a sharper mono fraction than the ~80-85% earlier -guess, corrected here). --subsample/--shannon on the smaller 20-genome -bacterial reference (benchmark/global_index_presence) produced a sample -size within rounding of the request (99,742/100,000) and a -[0.8,1.2)-bucket share (40.7%) matching the full unsampled population -(41.6%) — the two histograms only diverged wildly (5‰ vs 41.6%) under a -real bug in reservoir_sample_layer (see next section), now fixed.

      -

      Bug found and fixed (2026-08-15): family_idx numbering mismatch. -scan_layer_families's family_idx counts every minorant of a layer -(monomorphic ones included, since iter_minorants_batch filters only on -is_minorant()), not the raw annex slot (SiblingAnnex::get(slot) spans -every k-mer, minorant or not) and not a counter over non-monomorphic -minorants alone. subsample.rs's reservoir_sample_layer originally -stored raw slot numbers in its HashSet<usize> selection, which drifts -away from family_idx as soon as any monomorphic minorant is seen — -i.e. almost immediately, since ~98% of minorants are monomorphic. Fixed by -tracking two separate counters: family_idx (every minorant, matching -scan_layer_families) and seen (non-monomorphic minorants only, what -Algorithm R actually samples over) — only family_idx values are ever -stored in the selection set. The existing unit test never exercised this -(its fixture has exactly one non-monomorphic family, always hitting the -"keep everything" shortcut) — a stronger fixture with several interleaved -monomorphic/non-monomorphic families would be needed to catch a regression -here automatically; not yet written.

      -

      Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15)

      -

      Motivation: on real data (bacterial reference, full unsampled run), only -~5‰ of non-monomorphic minorants fall in the [0.5, 1.5] bit band judged -phylogenetically interesting (entropy too low = uninformative near-invariant -site; too high = saturated/noisy, see family_entropy's 15-state -discussion) — roughly 1 in 10,000 minorants overall. Computing exact -entropy for every candidate just to discard 99.99% of them is wasteful at -the full 9-billion scale.

      -

      The idea: PersistentBitMatrix::col_view(c) gives a genome's whole -presence column as a BitSliceView (sequential, no MPHF, no cross-partition -routing — purely local to one layer's own matrix). Accumulating -TempCompactIntVecBuilder::inc_present(col) (already exists in -obicompactvec/src/builder.rs:121, along with add/min/max/diff on -IntSliceView — no new low-level API needed) over every column of a layer -produces coverage[slot]: how many genomes carry each exact k-mer, in one -sequential per-layer pass, entirely decoupled from family/sibling -structure. Persisted once per layer, a family's members' coverage could -then be looked up via a plain local MPHF index() (cheap) instead of a -full cross-partition presence resolution (find_presence_batch) — the -expensive part today is specifically the cross-partition/cross-layer -routing to a sibling's own matrix, not the bit-reading itself, and -coverage[slot] sidesteps that routing entirely by moving the cost into a -one-time, purely local, embarrassingly-parallel build step.

      -

      Why not implemented: coverage[slot] is a per-member marginal — -summing members' coverages to approximate a family's entropy silently -assumes no genome carries more than one member at once. It cannot -represent or detect joint co-occurrence (a genome carrying both A and -C at once, i.e. a combined 15-symbol state) at all, which is exactly the -phenomenon family_entropy's 15-state definition exists to capture (see -CardinalityTally/cardinality_transition_probs, the project's own -existing machinery for this same co-occurrence structure, built for the -Sankoff matrix calibration). A family that is in truth uniformly AC -across every carrying genome would look like a well-balanced 2-state split -under the marginal approximation (entropy ≈ 1) while its true 15-state -entropy is 0 — i.e. the marginal proxy's failure mode lands on exactly the -"saturated, uninformative" tail this pre-filter would need to catch, -undermining the point. It stays plausible as a coarse filter for the -low tail only (a dominant single member's marginal share reliably -predicts low true entropy too), but not as a stand-in for the high tail — -not pursued further for now.

      -

      --free-loss/--tnt pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below)

      -

      Measured on phyloskims_sal_vac (91 genomes): obikmer phylo --subsample 500000 ---free-loss --tnt logs four sequential stages — -raw_snp_distance (1413s), base_pair_tally (1456s), -cardinality_tally (2078s), snp_pseudo_alignment (143s). Reading the -code (obikmer/src/cmd/phylo/mod.rs:210-242, -obikphylo/src/siblings/distance.rs, cardinality.rs, alignment.rs) -surfaced two compounding problems, not one:

      -
        -
      1. Four separate full scans of the annex, each opening its own - KmerPartition/PartitionCache and calling scan_family_pairs/ - scan_layer_families independently — nothing computed in one stage is - reused by another. base_pair_tally is explicitly documented - (distance.rs:138-143) as a second full pass over the same - traversal raw_snp_distance already did, needed only because - raw_snp_distance doesn't keep the resolved bases, only aggregate - counts. cardinality_tally and snp_pseudo_alignment are each a - third and fourth independent full pass. Per the module's own earlier - profiling note (family_scan.rs:26-28, cited already above), this - traversal is page-fault/mmap-bound, not compute-bound — the ~10-12% - CPU efficiency ("contention" status) observed on these three slow - stages is consistent with I/O stalls scaled by repeated full scans, - not lock contention (there are no Mutex/RwLock anywhere in - siblings/*.rs; shared writes use per-slot AtomicU8::fetch_or).
      2. -
      3. Worse: raw_snp_distance and cardinality_tally don't honor - --subsample at all — both call scan_layer_families with - Selection::All hardcoded, and args.subsample isn't even threaded - into their function signatures (mod.rs:212, mod.rs:226). Only - snp_pseudo_alignment(args.subsample) builds a real reservoir-sampled - Selection::Some(set) (via compute_selections, alignment.rs:89). - So today, --subsample 500000 only bounds the pseudo-alignment step — - the SNP-distance matrix, the Sankoff base-pair calibration, and the - cardinality histogram are always computed over the full, unsampled - index regardless of the flag. This is not merely "different subsamples - per stage" (which would already be a problem worth fixing) — it's that - three of the four stages never subsample, which explains most of the - ~10x runtime gap against snp_pseudo_alignment on its own.
      4. -
      -

      Decided requirement: all four stages consume one shared selection, -computed once, not each stage either scanning everything or drawing its -own independent sample. Per-genome-pair SNP counts, the Sankoff base-pair -calibration, the cardinality histogram, and the pseudo-alignment all -describe the same set of families — the calibration and the alignment it -calibrates are now guaranteed to agree on which sites exist.

      -

      Correction to the "single pass" framing above: base_pair_tally/ -cardinality_tally both need raw_snp_distance's complete aggregate -SNP/shared counts before they can derive included[i,j] (the -ratio_ceiling filter) — a genuine sequential dependency (included -can't be known until every pair's aggregate count is final), not an -artifact of the old code's structure. So the fix is two passes over -the shared selection, not one: pass A computes the aggregate counts (and -derives included); pass B fuses base_pair_tally + cardinality_tally -+ the pseudo-alignment (mutually independent once included is known) -into a single scan. Still a 4→2 reduction, and — per the clarification -that settled this — pass A itself now runs over the same shared -selection pass B uses (not the full unsampled index): "les ratios, on -les fait sur les sites sélectionnés, c'est tout, les autres sites -n'existent pas" — once a selection is chosen, both passes are bounded by -it, so on a real --subsample/--entropy run pass A is cheap too, not -just pass B.

      -

      Entropy-biased selection's own resolution to the "forward-looking -complication" (entropy must be known before selection, but selection -happens during the same scan that would resolve it): see "Entropy-biased -selection" below — resolved via a persisted per-layer entropy annex, not -by restructuring the scan into an inline pre-pass.

      -

      Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15)

      -

      Refines the "forward-looking complication" above with a concrete -mechanism. Instead of a hard [low, high] entropy band (or any other -exact cutoff) deciding which non-monomorphic minorant families are -eligible, selection is weighted by an unnormalized Gaussian kernel -centered on a target entropy: w(entropy) = exp(-(entropy - μ)² / (2σ²)) -— deliberately not the normalized Gaussian density (which would peak -below 1 and complicate the "probability" reading) — this kernel form -equals 1 exactly at entropy = μ and decays smoothly to 0 away from it, -so it reads directly as an acceptance weight: the further a family's -entropy from the target, the less likely it is picked, with no hard -in/out boundary — a few "bad" sites can still get in, by design. μ -(default ~1.0) and σ (default ~0.5) are meant to be user-tunable.

      -

      Mechanism: joint probability, not a weighted reservoir. Not -Efraimidis–Spirakis weighted reservoir sampling (an earlier, more complex -proposal, superseded before implementation) — instead, a single -independent accept/reject draw per qualifying candidate: draw -u ~ Uniform(0,1), accept iff u < p₀ · w(entropy). p₀ is a single -index-wide rate, N / total_count (total_count = the sum of -non_monomorphic_counts across every layer, N = --subsample's -target), applied identically at every layer — this alone already gives -each layer its proportional share (the same effect the old uniform -reservoir's explicit per-layer n_layer = N · count_layer / total_count -computation achieved, but without needing to compute it: applying one -rate uniformly is mathematically the same as proportioning per layer). -p₀ = 1.0 when there is no --subsample at all — --entropy alone is a -pure soft entropy filter over the whole index, no size target. Properties: -(1) strictly generalizes the existing uniform sampler — with σ large -enough that w ≈ 1 everywhere, this reduces to the old uniform N/total_count -draw; (2) yields "approximately N", not exactly N — expected accepted -count is N · mean(w), always ≤ N — an intentional relaxation, matching -"sous-échantillonnage à environ n" rather than the old reservoir's exact-N -guarantee; (3) one streaming pass, one random draw per candidate, no -reservoir state.

      -

      Resolving "entropy must be known before selection, but selection -happens during the same resolving scan": solved with a persisted, -per-layer entropy annex (obikphylo/src/siblings/entropy_annex.rs, -EntropyAnnex/EntropyAnnexBuilder), not by restructuring the scan. -Mirrors SiblingAnnex's mmap-backed, read-only-after-build convention, -but indexed by family_idx (every minorant of the layer, monomorphic -included — the same numbering Selection/scan_layer_families already -use), one f32 entropy15 value per entry, -1.0 sentinel for monomorphic/ -not-yet-computed. First use of --entropy/--entropy-sd on an index -pays a one-time cost (ensure_entropy_annexes in entropy.rs) — every -later run (any μ/σ, any command) reads the file positionally, no -re-scan, restoring the usual Selection::Some "skip resolving excluded -families" speedup that a naive "weigh during the resolving scan" design -would have permanently forfeited.

      -

      Bug found and fixed (2026-08-15): ensure_entropy_annexes scanned with -Selection::All instead of bounding to non-monomorphic minorants. -Monomorphism (family_size() < 2) is knowable directly from the annex -bits alone, no per-genome resolution needed — but the original -implementation called scan_layer_families with Selection::All -anyway, so fill_sub_matrix_carries (the expensive per-genome -resolution) ran for every minorant, ~98% of which are monomorphic -(measured elsewhere in this doc) and had their genome_mask immediately -discarded once the callback checked family_size() < 2. Fixed by adding -[subsample::non_monomorphic_selection_layer] — a cheap, annex-only, -non-sampling pass (same shape as reservoir_sample_layer, but keeping -every non-monomorphic minorant's family_idx instead of a bounded -reservoir) — and passing Selection::Some(&eligible) instead of -Selection::All, so the expensive resolution now runs only for the ~2% -of minorants that can actually produce a real entropy value. A -debug_assert!(mask.family_size() >= 2, ...) inside the -scan_layer_families callback guards the invariant.

      -

      Resolved: the existing hard "non-monomorphic minorant" eligibility -filter stays a hard gate upstream of the Gaussian weighting — only -qualifying families ever get a stored entropy value or a weighted draw.

      -

      CLI, implemented: two phylo options, --entropy <μ> and ---entropy-sd <σ> (obikmer/src/cmd/phylo/args.rs). The entropic filter -activates as soon as either is given (mod.rs, computed once into an -Option<EntropyBias> threaded through --snp/--family-overlap/ ---shannon/the fused sankoff pipeline below). If active but one or both -are unset, defaults are μ = 1.0, σ = 0.5.

      -

      Two entropy definitions kept side by side, for comparison (2026-08-15)

      -

      --shannon's CSV carries both entropy15 (family_entropy — the settled -15-non-empty-state definition, see above) and entropy4 (family_entropy_4 -— plain nucleotide reduction), computed from the same already-resolved -genome_mask, not from the marginal approximation above. A genome carrying -several bases at once contributes to each base's count (counted once per -base present, not fractionally split, not folded into one combined state) -— a genome polymorphic for the family is present at more than one base by -construction, so it is expected to count more than once; the denominator is -the total base-occurrence count, not the genome count (the two coincide -only when no genome carries more than one base). Kept side by side -specifically to measure, on real data, how much the two diverge — not yet -analyzed.

      -

      PersistentSparseBitMatrix — implemented and measured (2026-08-15)

      -

      A row-major (k-mer-major), deduplicated sparse alternative to -obicompactvec::PersistentBitMatrix, motivated by the same sparsity that -drove --subsample/--shannon above, but pursued as a foundational -storage-layer change rather than an index-level workaround. Full design -history, rationale, and rejected alternatives (external Elias-Fano crates, -cacheline-ef, a single unsplit dict_id array) are in the dedicated -implementation plan (vivid-mapping-tiger.md at the time of writing — the -content below is the durable summary, not a pointer to a session-scoped -file). Also directly informed by Alanko, Bille, Gørtz, Navarro, Puglisi, -"Compact Data Structures for Collections of Sets" (2025, -biblio/Alanko et al. - Compact Data Structures for Collections of -Sets.pdf) — this design implements only their exact-duplicate special -case (a plain dedup dictionary), not their full subset-containment -hierarchy.

      -

      Design: four on-disk components, each mmap-backed, built once per -layer (matching how the rest of the build pipeline already works — never -the whole multi-billion-row index at once): an is_multi rank-capable -flag per row (singleton vs. multi-genome), a fixed-bit-width array for -singleton rows (genome index directly, ceil(log2(n_cols)) bits), a -separate fixed-bit-width array for multi-genome rows (dict_id, -ceil(log2(n_distinct_multi_sets)) bits — kept apart from the singleton -array specifically because n_distinct_multi_sets can be large in -absolute terms even when multi-genome rows are a small fraction of all -rows, and a single shared array would force every row, singletons -included, to pay the wider width), and a deduplicated dictionary of -distinct multi-genome sets (Elias-Fano-encoded byte offsets + a -varint-encoded values blob). New low-level primitives added to -obicompactvec to build this: PersistentFixedIntVec (arbitrary, -runtime-parameterized bit width, width 0 included — needed once a real -bug surfaced, see below), PersistentRankSelectBitVec (rank1/rank0/select1 -on top of the crate's existing count_ones, using -common_traits::SelectInWord), EliasFano (composes the two). A new -BinaryMatrix trait (n, n_cols, row/fill_row, fill_sub_matrix, -count_ones) unifies dense and sparse at the one call site that needs -both interchangeably (obikphylo::siblings::cache::Mat) — column-oriented -methods (col, col_view, the partial_*_dist_matrix family) stay -dense-only.

      -

      Two real bugs caught by tests, not by inspection: (1) EliasFano::open -re-derived its low-bits width from the persisted low-vector file's own -width byte; the zero-width case was built with a dummy 1-bit placeholder -(the builder rejected true width 0), so every reopened value silently -doubled. Fixed by making PersistentFixedIntVec genuinely support width 0 -(no storage, get always 0) instead of working around the limitation in -EliasFano. (2) An empty row (cardinality 0 — not expected on a real -built index, but not guarded against either) was recorded as a singleton -at genome 0, indistinguishable on read-back from a real singleton at -genome 0. Fixed by routing cardinality-0 rows through the dictionary path -(a genuine empty entry) instead of the singleton shortcut. Both caught by -obicompactvec's test suite (142 tests, including disk-reopen round-trips -that drop every builder/mmap before reopening fresh), not by manual -review — worth remembering next time a "this edge case can't happen in -practice" shortcut is tempting.

      -

      Measured on real data (layer_1 of phyloskims_sal_vac's -part_00018, 30,246,774 rows, 91 genomes — #[ignore]d benchmarks in -obikphylo/src/siblings/tests.rs):

      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      densesparseratio
      on-disk size328.1MB43.7MB7.5x smaller
      build time / peak RSS—4.26s / 628MB(per-layer, in-memory construction — comfortable)
      row access, sequential (2M reads)43ns/row32ns/rowsparse faster (smaller structure, better cache fit)
      row access, random (2M reads)409ns/row85ns/rowsparse ~4.8x faster (the real --shannon/family-lookup shape)
      column access, one full column (30.2M rows)11.5ms993mssparse 86x slower (no native column method — every read decodes a full row to keep one bit)
      -

      The row-access wins (both directions) weren't the design's stated goal — -compactness was — but turn out real: dense's genome-major layout scatters -a single row read across a much bigger file, which costs more than -sparse's rank/select/varint decode once the file is this much smaller. -The column-access cost is the flip side of the same layout choice, and is -exactly what the next item below exists to fix.

      -

      Next, not yet planned: rewrite partial_jaccard_dist_matrix/ -partial_hamming_dist_matrix/etc. (obicompactvec/src/bitmatrix/pairwise.rs) -as a row-major co-occurrence accumulation (O(Σ_rows k²), per-row -increments into an NxN genome-pair counter — the known alternative to -today's column-fold, plausibly cheaper on data this sparse, not just a -fallback) so obikindex's --metric/distance-matrix path can use the -sparse type without the measured 86x column-access penalty. Needs its own -design pass (in particular how it plugs into the BitPartials/ -ColumnWeights traits so both matrix types keep serving --metric) -before implementation — not just "port the loop", a genuinely different -algorithm.

      -

      Also still deferred, unchanged from the implementation plan: full Alanko -et al. subset-hierarchy compression (only the exact-duplicate special case -is built), a sparse PersistentCompactIntMatrix (count matrices), and -BRWT-style column-correlation exploitation.

      -

      Wired into pack and the sibling-annex build path (2026-08-15)

      -

      PersistentSparseBitMatrix went from a validated but unused type to a -real, selectable on-disk format:

      +

      Public API surface

      +

      extensions::SiblingExt (CLI-facing trait on IndexCache) covers only the +operations with no cross-invocation sharing concern: build_sibling_annex, +shannon_entropy_csv, sibling_family_size_histogram/sibling_annex_stats, +family_overlap. The sampled SNP path (sampled_scan, AlignmentReducer, +PairwiseReducer, distance_matrix, SankoffBundle) is not wrapped in +trait methods — the CLI (obikmer::cmd::phylo) composes them directly, +because it alone knows the union of reducers a given invocation needs +(--pseudo-alignment/Sankoff group/snp-* --distance can combine in one +call). A per-output trait method would have to either duplicate that +composition logic or reintroduce the "two independent scans" problem this +design replaced.

      +

      Known gaps

        -
      • Generic Layer<D>: obikindex::layer::Layer<D>'s presence-only methods - (n_cols, sub_matrix, fill_sub_matrix) are generic over any - D: LayerData<Item = Box<[bool]>> + BinaryMatrix, not hardcoded to - PersistentBitMatrix — PersistentSparseBitMatrix implements - LayerData (open/read) the same way. find_slot/index_batch were - already generic over any D: LayerData, so they needed no change. - Verified by obikindex::layer's - presence_layer_generic_over_sparse_matches_dense test: build a dense - presence layer, convert it to sparse via build_from_dense, open both - as Layer<PersistentBitMatrix>/Layer<PersistentSparseBitMatrix> on - the same directory, assert n_cols/sub_matrix/find_slot agree. - (This test must stay at k=4 with mutually non-colliding canonical - 4-mers across its input sequences — K/M are process-wide - AtomicUsizes in test builds, not thread-local, so a test using a - different k races every other test in the same crate binary; a k=11 - version of this test passed alone but failed under the full - obikindex::layer suite for exactly that reason before being fixed.)
      • -
      • obikphylo::siblings::cache::Mat gained a third variant, - SparsePresence(Layer<PersistentSparseBitMatrix>), alongside Count - and Presence — every method (find_slot, index_batch, - iter_minorants_batch, n_cols, fill_sub_matrix_carries) dispatches - to it identically to Presence, since both go through the same generic - Layer<D> code. PartitionCache::build picks the variant per layer by - checking for presence/is_multi.prsb (the sparse format's own marker - file, see the design section above) before falling back to the dense - open path.
      • -
      • pack_sparse_bit_matrix (new, obicompactvec::bitmatrix::sparse): - pack --sparse's entry point. Idempotent (checks is_multi.prsb - first); packs to dense matrix.pbmx first if that hasn't happened yet - (the dense→sparse transpose needs random row access, which only the - packed/columnar dense forms give), then build_from_denses the sparse - form into the same directory and deletes matrix.pbmx — old-format - files are removed only after the new format is fully written, mirroring - pack_bit_matrix's own crash-safety convention.
      • -
      • CLI: obikmer pack --sparse threads a sparse: bool through - KmerIndex::pack_matrices (all other call sites — select, merge, - finalize_indexed — pass false, unchanged dense behaviour). Count - matrices are untouched by --sparse (no sparse PersistentCompactIntMatrix - — see "still deferred" above).
      • -
      • End-to-end coverage: obikphylo::siblings::tests:: - sibling_annex_works_after_pack_sparse builds a two-genome index, packs - it --sparse, asserts is_multi.prsb exists, then runs - build_sibling_annex and checks the resulting FamilyMasks match the - dense-path test (sibling_annex_one_sibling_each) exactly — proves the - sparse format round-trips through the real build pipeline - (PartitionCache sparse-detection included), not just the - obicompactvec/obikindex::layer unit layers below it.
      • -
      -

      Full workspace cargo test (all crates, unit + doc tests) green after -this change.

      -

      Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15)

      -

      Replaces the "four independent scans" problem above and implements -"Entropy-biased selection" above, end to end:

      -
        -
      • SankoffBundleExt::sankoff_bundle (new, - obikphylo/src/siblings/sankoff_bundle.rs) — the --sankoff/--tnt/ - --phyg/--iqtree block in obikmer/src/cmd/phylo/mod.rs now calls - this once instead of three separate raw_snp_distance/ - base_pair_tally/cardinality_tally calls. One PartitionCache, one - shared (possibly subsampled/entropy-biased) selection computed once via - compute_selections, two scans over it: pass A (aggregate SNP/shared - counts, --exclude-genome zeroing, then included[i,j]), pass B - (base_pair_tally + cardinality_tally + the pseudo-alignment, fused - into one scan_layer_families call per layer, all three read off the - same resolved genome_mask). snp_pseudo_alignment/ - shannon_entropy_csv (still used standalone by --snp/ - --family-overlap/--shannon) both gained an entropy_bias parameter - too, so entropy-biased selection isn't sankoff-specific.
      • -
      • Regression proof: sankoff_bundle_matches_old_separate_calls - (obikphylo/src/siblings/tests.rs) asserts sankoff_bundle's four - outputs are bit-identical to calling the old, separate - raw_snp_distance/base_pair_tally/cardinality_tally/ - snp_pseudo_alignment on the same fixture with no subsample — the - fusion is a performance change, not a behavior change.
      • -
      • EntropyAnnex/EntropyAnnexBuilder (new, - obikphylo/src/siblings/entropy_annex.rs) and ensure_entropy_annexes - (entropy.rs) implement the persisted-entropy mechanism from - "Entropy-biased selection" above. entropy_annex_builds_on_demand_and_biases_selection - (tests.rs) proves, on a fixture with one known-entropy family: the - annex file doesn't exist before any entropy-biased call; compute_selections - builds it on first use; μ set to the family's exact entropy with - p₀ = 1.0 selects it deterministically (u < 1.0 always, for - u ∈ [0,1)); μ set far away with tiny σ deterministically excludes - it (w underflows to exactly 0.0); the persisted value matches - --shannon's own family_entropy computation to 1e-6.
      • -
      • EntropyBias (pub, obikphylo::siblings::EntropyBias { mu, sigma }) - is the one new public type threading --entropy/--entropy-sd through - every Option<EntropyBias>-accepting method — resolved once in - obikmer/src/cmd/phylo/mod.rs from args.entropy/args.entropy_sd - (activation: either given; defaults 1.0/0.5 for whichever is unset).
      • -
      -

      Full workspace cargo test green after this change (167 unit tests in -obicompactvec+obikindex::layer+obikphylo alone, plus every other -crate's suite, no regressions).

      -

      Still open, not part of this change (per "Correction to the 'single -pass' framing" above): --raw-snp-distance/--raw-snp-counts (the -standalone diagnostic flags, not the --sankoff pipeline) still always -scan the full unsampled index — never threaded --subsample/--entropy, -out of scope here since the reported problem was specifically about the ---sankoff/--tnt pipeline's redundant/inconsistent scans, not these -two standalone flags.

      -

      query never benefits from sparse row-major access (found 2026-08-19, not implemented)

      -

      Benchmarked obikmer query against global_index_presence (dense-packed) -vs. global_index_presence_sparse (pack --sparse), 100k simulated reads -× 2 specimens (benchmark/, see -benchmark_query_testing.md). -Correctness: 0 mismatches — sparse and dense return bit-identical query -results. Performance: sparse consistently slower than dense (~30-50%, -reproducible across two runs with warm disk cache), the opposite of -pack --sparse's stated intent ("faster for single-row access... like -query").

      -

      Root cause, read from source, not measured in isolation: -KmerPartition::query_partition_with (obikindex/src/partition/query_layer.rs:155-220) -is architecturally column-major: stage 2 walks for g in 0..n_cols { for -slot in hit_slots { layer.col_value(g, slot) } }, documented (correctly) -as the right locality strategy for the packed/columnar formats, where -col_value → PersistentBitMatrix::get is a genuine O(1) mmap'd column -read (persistent.rs:110-113).

      -

      For Self::Sparse, that same get(c, slot) (persistent.rs:114-118) -allocates a full n_cols-wide buffer and calls fill_row — materializing -the entire row — just to return one cell. Called from inside the -column-major double loop, this reconstructs the same row once per genome -column touched: O(hits × n_cols) full-row rebuilds instead of O(hits). -PersistentSparseBitMatrix's own native row-major decode -(for_each_genome_in_row, sparse.rs:164-177, used correctly by its own -row/fill_row/fill_sub_matrix) is never reached from the query path -at all.

      -

      fill_sub_matrix (the existing BinaryMatrix trait primitive, -traits.rs:13-37) is not the right replacement for query either, even -once its own dispatch bug is fixed (see next section) — its output shape -is inherently column-dense: out[col] gets an entry for every column, -including columns with zero hits among the requested slots. On real -sparse data (a hit typically touching a handful of genomes out of dozens) -that's still O(n_cols) output regardless of true sparsity. What query -actually wants is the sparse triple stream (slot, col, value) it already -consumes as QueryHit::Value — not a materialized sub-matrix.

      -

      Proposed primitive (design only, not implemented — explicit ask: keep -count matrices not excluded, even though effort right now is -presence/absence only):

      -

      Not a closure-driven for_each — a real Iterator, one concrete struct -per matrix format, so the traversal state (current position in the sorted -slot list, current column, permutation, sparse-row decode cursor…) lives -in named struct fields instead of being threaded implicitly through -recursion or a captured closure. RPITIT (stable since 1.75, and this -workspace is edition 2024) means the trait method can return it without -naming or boxing the concrete type:

      -
      /// Yields every nonzero cell among `slots`, in implementation-defined order.
      -fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a;
      -// item: (idx into `slots`, col, value)
      -
      -

      This is the one foundational primitive per format — both fill_sub_matrix -and a for_each-style callback become trivial, free consumers of it -(.for_each(f) is already Iterator::for_each from std; fill_sub_matrix -becomes "drain the iterator, scatter into out[][]"), instead of two -independently-maintained traversals that can silently diverge (see the bug -below — this is exactly how it happened).

      -
        -
      • On PersistentSparseBitMatrix: the struct is nearly free to write — - it wraps the existing (currently private) for_each_genome_in_row - per-row decode, advancing to the next slots entry on exhaustion. O(Σ - row nnz), zero n_cols-wide allocation.
      • -
      • On PersistentBitMatrix::{Packed,Columnar}: revised — cheaper than - first thought, by reusing the same split already used for - fill_matrix's own implementation instead of hand-writing a resumable - state machine at the matrix level. The base-vector layer - (BitSliceView, views.rs) already separates the two concerns: - fill_batch_sorted (views.rs:55-60, sorted-slot batch lookup) and a - genuine per-bit Iterator (BitSliceIter, views.rs:94+) sit side by - side there, one level below the matrix. Adding a - "positions among sorted_slots where the bit is set" iterator at that - same vector level is a filter over the existing get() — no new - state machine, since std::iter::Filter already is one, generated by - the compiler: -
        fn nonzero_among_sorted<'s>(&'s self, sorted_slots: &'s [usize]) -> impl Iterator<Item = usize> + 's {
        -    sorted_slots.iter().copied().filter(move |&slot| self.get(slot))
        -}
        -
        - The matrix-level nonzero_iter then composes these per column with - flat_map over 0..n_cols (each column's hits, tagged with c, - slot mapped back through the sort permutation fill_batch/ - fill_batch_sorted already carry) — again a combinator chain, not a - hand-rolled struct. Same algorithm, same mmap/sort locality as today's - fill_sub_matrix; just assembled from std iterator adaptors instead - of a loop body writing into a buffer, mirroring the vector/matrix split - the codebase already uses for fill_batch_sorted rather than - introducing a new shape.
      • -
      • Implicit: trivial (slots.iter().map(|&i| (i, 0, 1)), one column, - always present).
      • -
      • On PersistentCompactIntMatrix (counts): same treatment as - Packed/Columnar — no sparse count format exists yet ("Explicitly - deferred" per traits.rs:9-12), so no native low-effort case the way - Sparse has one, but not excluded either: the iterator's Item is - already (usize, usize, u32), not bool, specifically so presence - (0/1) and counts (arbitrary u32) share one primitive instead of a - bool/u32 split forcing counts out of the design. Ready for a native - sparse-count struct later without a signature change.
      • -
      -

      Would let query_partition_with's stage 2 collapse to one -for (i, g, v) in layer.matrix().nonzero_iter(&hit_slots) { on_event(...) } -per layer, format-agnostic, each backend's struct deciding the actual -traversal.

      -

      This also closes the existing dispatch bug for free, by construction: -PersistentBitMatrix::fill_sub_matrix (persistent.rs:190-215, the enum -wrapper backing BinaryMatrix's trait impl) today does not delegate to -PersistentSparseBitMatrix::fill_sub_matrix for Self::Sparse — it -reimplements the same naive per-(column, slot) fill_row_bool loop -instead, bypassing the efficient native method one file over -(sparse.rs:249-258). obikphylo::siblings::cache::Mat -(cache.rs:138-145) independently built its own parallel enum wrapper -that dispatches correctly — evidence this was worked around rather than -fixed at the source: two hand-written traversals for the same format, -free to drift apart, and they did. If fill_sub_matrix itself is -rewritten as "drain nonzero_iter, scatter into out[][]", there is only -one traversal per format left to get right — the bug class doesn't just -get fixed once, it stops being possible to reintroduce.

      -

      Implemented (2026-08-20)

      -

      Built as designed above, with one deviation from the original sketch: -nonzero_iter ended up Box<dyn Iterator<...>>, not a bare impl -Iterator, because Columnar/Packed/Sparse/Implicit are genuinely -different concrete types and this method isn't on a trait (kept off -BinaryMatrix deliberately — that trait is used as dyn BinaryMatrix in -tests/sparse.rs, and RPITIT methods aren't dyn-compatible). One Box -per nonzero_iter call, not per cell — negligible next to what it -replaces.

      -
        -
      • BitSliceView::nonzero_among_sorted / IntSliceView::nonzero_among_sorted - (obicompactvec/src/views.rs): the vector-level filter/filter_map - primitive, exactly as sketched — no new state machine, std's own.
      • -
      • PersistentSparseBitMatrix::nonzero_iter (bitmatrix/sparse.rs): native, - std::iter::from_fn over one buffered row at a time via the existing - for_each_genome_in_row — no n_cols-wide allocation, ever.
      • -
      • PersistentBitMatrix::nonzero_iter (bitmatrix/persistent.rs): dispatches - to the above for Sparse; for Columnar/Packed, loops columns, - collects each column's nonzero_among_sorted hits via .extend() (not - flat_map — a flat_map closure can't lazily return something - borrowing its own captured sort permutation across separate calls - without either boxing per-column or fighting the borrow checker; eager - collection into one Vec sidesteps it, at zero cost since - fill_sub_matrix already fully materialized anyway). Implicit trivial.
      • -
      • PersistentBitMatrix::fill_sub_matrix and sub_matrix rewritten to - drain nonzero_iter — the dispatch bug is gone because there is now - only one traversal per format, not because the old one was patched. - PersistentCompactIntMatrix::nonzero_iter added the same way (counts - not excluded, per the earlier ask) — no native low-effort case, since no - sparse count format exists, but on the same primitive, ready for one.
      • -
      • KmerPartition::query_partition_with (obikindex/src/partition/query_layer.rs): - stage 2's column-major for g { for slot { col_value } } replaced by one - layer.nonzero_iter(&slot_list) call per layer, format-agnostic.
      • -
      • Tests: nonzero_iter_matches_dense, nonzero_iter_matches_row, and — - the one that actually targets the dispatch bug rather than each type's - own correctness — enum_wrapper_dispatches_to_native_sparse (builds - PersistentBitMatrix::Sparse(...) directly, not through open, since - open only auto-detects Sparse from a presence/ dir layout). - cargo test --workspace: green, no regressions.
      • -
      -

      Measured: re-ran the benchmark/ query branch (100k reads × 2 -specimens, same setup as the original finding). Correctness still 0 -mismatches. The dense/sparse performance gap is gone — previously sparse -~30-50% slower than dense, reproducibly; now within ~1-3% either way -(7.42s dense vs 7.60s sparse for Escherichia_coli--K-12_MG1655; 5.25s vs -5.30s for Saccharolobus_islandicus--M.16.4) — noise-level, not a -systematic gap. pack --sparse's claimed query win isn't confirmed -outright by this (sparse should arguably now beat dense on truly sparse -real data, not just tie), but the pathological regression is fixed.

      -

      PersistentCompactIntMatrix::Sparse — implemented (2026-08-26)

      -

      Closes the gap flagged throughout this document ("no sparse count format -exists yet", traits.rs:9-12's "Explicitly deferred"): obicompactvec -already had PersistentSparseCompactIntMatrix (row-major, built on top of -PersistentSparseBitMatrix as its "which columns are non-zero" support, -values not deduplicated — see that struct's own doc comment), but it was -never wired into PersistentCompactIntMatrix, the dense-dispatching enum -every real consumer (TypedLayer<PersistentCompactIntMatrix>, -KmerLayer::Count) actually holds. Concretely: kmer_index.rs:: -pack_matrices(sparse=true) already called pack_sparse_compact_int_matrix -on every layer's counts/ — but PersistentCompactIntMatrix::open had no -code path back to what that just wrote, so a Count layer became -unreadable ("no count matrix found ... run 'obikmer upgrade'") the moment -anyone ran pack --sparse on an index with count layers. Root cause, not a -workaround: add the missing Sparse variant.

      -
        -
      • -

        Enum + dispatch (intmatrix.rs): PersistentCompactIntMatrix::Sparse - (PersistentSparseCompactIntMatrix), detected in open/detect_storage - via a singleton_values.pciv marker (mirrors PersistentBitMatrix's own - sparse_meta.json check), reported via storage_kind(). col/ - col_view/col_persist panic/Unsupported on Sparse, same convention - as the bit side. sub_matrix/fill_sub_matrix and nonzero_iter - unified the same way PersistentBitMatrix's already are (drain - nonzero_iter, one traversal per format — see "Implemented - (2026-08-20)" above); nonzero_iter had to become Box<dyn Iterator<...>> - for the same reason (Columnar/Packed/Sparse are different concrete - types). No change needed in obikindex at all — KmerLayer::Count - already only ever holds TypedLayer<PersistentCompactIntMatrix>, so the - enum absorbing Sparse fixes the unreadable-layer bug for free, same as - PersistentBitMatrix::Sparse already did on the presence side.

        -
      • -
      • -

        CountPartials, non-naive (sparse_intmatrix.rs): unlike - PersistentSparseBitMatrix's dict-driven col_weights_and_pair_counts, - values here aren't deduplicated (two rows can share the same non-zero - column set via the same dict_id while carrying different counts), so - the "weight by how many rows share a dict entry" shortcut doesn't carry - over. What does: a single row-major pass (row_major_pairwise, decodes - each row once via for_each_cell_in_row, nests over that row's own - co-present columns) — O(Σ k̄²) over populated rows instead of the naive - O(n_cols² × n) column-pair rescan, same complexity class as the bit - side minus the dict multiplicity discount. Kernels used: min(a,b) - (bray, relfreq-bray — both vanish when either side is absent, so no - correction needed), a·b and √(a·b) (euclidean/relfreq-euclidean and - hellinger — these do need a correction, reconstructed from per-column - marginals via Σ(a-b)² = Σa²+Σb²-2Σab, since (a-0)² = a² ≠ 0 unlike - the min-based formulas). threshold_jaccard(1) shortcuts straight to - support's own BitPartials::partial_jaccard (threshold 1 is exactly - presence); threshold_jaccard(0) is closed-form (every u32 is ≥ 0).

        -
      • -
      • -

        Two pre-existing bugs found and fixed while wiring the threshold==1 - shortcut (bitmatrix/sparse.rs, BitPartials for - PersistentSparseBitMatrix, present since the 2026-08-15 implementation - above, never caught because no test compared Sparse's raw partial_* - output against dense on real data — only the diagonal-blind - jaccard_dist_matrix/hamming_dist_matrix finalisations were tested):

        -
      • -
      • partial_jaccard's diagonal was (0, 2×col_weights[i]) instead of a - genuine self-comparison (col_weights[i], col_weights[i]) — - col_weights_and_pair_counts's inter never pairs a column with - itself by construction.
      • -
      • partial_hamming's off-diagonal formula itself was wrong: total - - union (count of rows where neither column is present) instead of - the actual Hamming distance col_weights[i] + col_weights[j] - - 2×inter[i,j] (symmetric-difference size). Only coincides with the - correct value when col_weights[i] + col_weights[j] == total, so - small/synthetic test data could easily have hidden it.
      • -
      -

      Neither surfaced through jaccard_dist_matrix/hamming_dist_matrix - (both explicitly zero their own diagonal at finalisation, and the - off-diagonal partial_hamming bug had gone untested against dense - entirely) — only visible to a caller of the raw partial_* methods - directly, which is exactly what partial_threshold_jaccard(1)'s new - shortcut became. Fixed at the source, not patched around at the call - site; regression test added: - tests::sparse::partial_jaccard_and_hamming_match_dense_including_diagonal.

      -
        -
      • Tests: tests::intmatrix::sparse_roundtrip_matches_columnar/ - sparse_roundtrip_from_packed (the open-dispatch fix, both build - paths); tests::intmatrix::sparse_count_partials_match_dense (all six - CountPartials formulas, thresholds 0/1/2/3, against Columnar on - asymmetric-presence data — this is what caught the diagonal gap in the - int side's own new code before it shipped, the same way it exposed the - two pre-existing bit-side bugs above); obikindex's - count_layer_transparently_reads_sparse_after_pack — the actual - end-to-end regression test for the original "layer unreadable after - pack --sparse" bug, built → packed sparse → reopened, compared against - the pre-pack dense read. cargo test -p obicompactvec -p obikindex: - green, no regressions (180 + 12 tests).
      • +
      • sample_index's skip_layers parameter (checkpointed-resume support) is + never called with a non-zero value by any current caller — --session + persistence was removed pending a redesign on top of sampled_scan + (single shared trunk, not per-driver checkpoint plumbing). The parameter + stays as the seam a future resume mechanism would use.
      • +
      • No cheap entropy pre-filter exists yet for the very high (saturated) tail — + a row-marginal coverage approximation was considered and rejected (cannot + represent joint multi-base co-occurrence, which is exactly what the + 15-state entropy definition exists to capture).
      diff --git a/DevDoc/implementation/benchmark_query_testing/index.html b/DevDoc/implementation/benchmark_query_testing/index.html index c8e7c979..14817889 100644 --- a/DevDoc/implementation/benchmark_query_testing/index.html +++ b/DevDoc/implementation/benchmark_query_testing/index.html @@ -1265,7 +1265,7 @@ - Sibling annex (discussion) + Sibling annex @@ -1485,8 +1485,9 @@ this pipeline.

      by verify_presence/verify_merge_presence, which only check index content against the .npz truth, never the query API.
    • pack --sparse produces a presence-matrix format documented (see - siblings.md) as faster for single-row - access (query) and slower for column-oriented access (phylo --metric). + obicompactvec.md, "Sparse matrix formats") as faster + for single-row access (query) and slower for column-oriented access + (phylo --metric). global_index_presence/ built by merge_presence.sh is always packed dense (packing is a stage inside merge, not a separate pack invocation) — there was no dense/sparse regression check.
    • diff --git a/DevDoc/implementation/chunkreader.refs/index.html b/DevDoc/implementation/chunkreader.refs/index.html index d19f2081..7bb38ea2 100644 --- a/DevDoc/implementation/chunkreader.refs/index.html +++ b/DevDoc/implementation/chunkreader.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/chunkreader/index.html b/DevDoc/implementation/chunkreader/index.html index b1ed11a9..6c386b97 100644 --- a/DevDoc/implementation/chunkreader/index.html +++ b/DevDoc/implementation/chunkreader/index.html @@ -579,7 +579,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/evidence_elimination.refs/index.html b/DevDoc/implementation/evidence_elimination.refs/index.html index 1fa176b3..b29cbf5d 100644 --- a/DevDoc/implementation/evidence_elimination.refs/index.html +++ b/DevDoc/implementation/evidence_elimination.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/evidence_elimination/index.html b/DevDoc/implementation/evidence_elimination/index.html index 70621d90..7db92264 100644 --- a/DevDoc/implementation/evidence_elimination/index.html +++ b/DevDoc/implementation/evidence_elimination/index.html @@ -1265,7 +1265,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/filtering/index.html b/DevDoc/implementation/filtering/index.html index 6fdab235..781161fe 100644 --- a/DevDoc/implementation/filtering/index.html +++ b/DevDoc/implementation/filtering/index.html @@ -1388,7 +1388,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/kmer.refs/index.html b/DevDoc/implementation/kmer.refs/index.html index b73ef4d2..898475b5 100644 --- a/DevDoc/implementation/kmer.refs/index.html +++ b/DevDoc/implementation/kmer.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/kmer/index.html b/DevDoc/implementation/kmer/index.html index f367ae72..ca7782a3 100644 --- a/DevDoc/implementation/kmer/index.html +++ b/DevDoc/implementation/kmer/index.html @@ -1265,7 +1265,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/layer_tier.refs/index.html b/DevDoc/implementation/layer_tier.refs/index.html index 92d01fcf..aff4e5af 100644 --- a/DevDoc/implementation/layer_tier.refs/index.html +++ b/DevDoc/implementation/layer_tier.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/layer_tier/index.html b/DevDoc/implementation/layer_tier/index.html index 2dc88b7a..048b4831 100644 --- a/DevDoc/implementation/layer_tier/index.html +++ b/DevDoc/implementation/layer_tier/index.html @@ -1504,7 +1504,7 @@ - Sibling annex (discussion) + Sibling annex @@ -2208,7 +2208,7 @@ pub fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<bool>> pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>])
-

Column-first to match the on-disk column-major layout. fill_sub_matrix sorts slots once, then calls each column's fill_batch in turn — no redundant per-column sort. On PersistentSparseBitMatrix (k-mer-major, no column method) this degrades to a row-by-row decode; see siblings.md.

+

Column-first to match the on-disk column-major layout. fill_sub_matrix sorts slots once, then calls each column's fill_batch in turn — no redundant per-column sort. On PersistentSparseBitMatrix (k-mer-major, no column method) this goes through nonzero_iter instead, decoding one row at a time; see obicompactvec.md, "Sparse matrix formats".


LayeredStore\<S> and aggregation traits

LayeredStore<S> is a generic aggregation wrapper over Vec<S>. It propagates three traits from obicompactvec::traits up the hierarchy via blanket impls:

diff --git a/DevDoc/implementation/merge.refs/index.html b/DevDoc/implementation/merge.refs/index.html index 8f4e288c..22d6d050 100644 --- a/DevDoc/implementation/merge.refs/index.html +++ b/DevDoc/implementation/merge.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/merge/index.html b/DevDoc/implementation/merge/index.html index 9f91a463..b1050f4e 100644 --- a/DevDoc/implementation/merge/index.html +++ b/DevDoc/implementation/merge/index.html @@ -1348,7 +1348,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/merge_parallelism/index.html b/DevDoc/implementation/merge_parallelism/index.html index b7395e98..d78249bb 100644 --- a/DevDoc/implementation/merge_parallelism/index.html +++ b/DevDoc/implementation/merge_parallelism/index.html @@ -1332,7 +1332,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/mphf.refs/index.html b/DevDoc/implementation/mphf.refs/index.html index 331707e7..b48c8bda 100644 --- a/DevDoc/implementation/mphf.refs/index.html +++ b/DevDoc/implementation/mphf.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/mphf/index.html b/DevDoc/implementation/mphf/index.html index 010a5215..67587caa 100644 --- a/DevDoc/implementation/mphf/index.html +++ b/DevDoc/implementation/mphf/index.html @@ -1343,7 +1343,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/implementation/obicompactvec/index.html b/DevDoc/implementation/obicompactvec/index.html index 9f6c2fff..d1f0eea5 100644 --- a/DevDoc/implementation/obicompactvec/index.html +++ b/DevDoc/implementation/obicompactvec/index.html @@ -494,7 +494,7 @@ - Sibling annex (discussion) + Sibling annex @@ -632,6 +632,46 @@
  • + + + + Sparse matrix formats + + + + +
  • +
  • @@ -1191,7 +1231,7 @@ Pass 1 — byte max, SIMD-vectorizable, O(n) Sparse bit: sparse_meta.json + PFIV/Elias-Fano component files, row-major. Int: same support files (built on PersistentSparseBitMatrix internally) plus singleton_values.pciv/multi_values.pciv/multi_offsets for the per-row, non-deduplicated values -pack --sparse; see siblings.md for the sparse-vs-dense access-pattern trade-off +pack --sparse; see "Sparse matrix formats" below for the sparse-vs-dense access-pattern trade-off Implicit (bit only) @@ -1201,7 +1241,6 @@ Pass 1 — byte max, SIMD-vectorizable, O(n)

    PersistentBitMatrix::open(layer_dir) auto-detects the variant, in order: matrix.pbmx → Packed, presence/meta.json → Columnar, presence/sparse_meta.json → Sparse, layer_meta.json (no presence dir at all) → Implicit. PersistentCompactIntMatrix::open(layer_dir) mirrors the same priority order minus Implicit (there's no implicit count matrix — counts always have at least one on-disk column): matrix.pcmx → Packed, counts/meta.json → Columnar, counts/singleton_values.pciv → Sparse. col_view/col/sub_matrix panic on Sparse/Implicit where the operation has no direct-slice equivalent (Sparse is k-mer-major, not column-major; Implicit has no backing storage) — callers needing per-column data on those variants go through row/fill_row.

    -

    Unlike the bit side, PersistentSparseCompactIntMatrix's values are not deduplicated across rows — two rows can share the same non-zero column set (same dict_id in the shared support) while carrying different counts — so its CountPartials impl can't reuse the support's dict-multiplicity shortcut the way BitPartials for PersistentSparseBitMatrix does. It still avoids the naive O(n_cols² × n) column-pair scan via a single row-major pass (row_major_pairwise in sparse_intmatrix.rs), reconstructing the squared-difference formulas (euclidean/relfreq_euclidean/hellinger) from per-column marginals via Σ(a-b)² = Σa²+Σb²-2Σab — see siblings.md's "PersistentCompactIntMatrix::Sparse — implemented" entry for the full derivation.

    col_view(c) returns the appropriate view directly:

    // PersistentBitMatrix
     pub fn col_view(&self, c: usize) -> BitSliceView<'_>
    @@ -1212,6 +1251,183 @@ Pass 1 — byte max, SIMD-vectorizable, O(n)
     

    No wrapper enums (BitColView, IntColView): the caller receives a Copy view struct immediately usable with any view method or bulk builder method.

    pack_compact_int_matrix and pack_bit_matrix convert columnar → packed format.


    +

    Sparse matrix formats

    +

    Motivated by real k-mer-family data being extremely row-sparse (most rows — +k-mers — present in only a handful of genome columns out of dozens to +thousands): a row-major, deduplicated alternative to the dense column-major +formats above, built once per layer like the rest of the pipeline (never the +whole multi-billion-row index at once). Implements the exact-duplicate +special case of Alanko, Bille, Gørtz, Navarro, Puglisi, "Compact Data +Structures for Collections of Sets" (2025, +biblio/Alanko et al. - Compact Data Structures for Collections of +Sets.pdf) — a plain dedup dictionary, not their full subset-containment +hierarchy.

    +

    PersistentSparseBitMatrix

    +

    Four mmap-backed on-disk components: an is_multi rank-capable flag per row +(singleton vs. multi-genome), a fixed-bit-width array for singleton rows +(genome index directly, ceil(log2(n_cols)) bits), a separate fixed-width +array for multi-genome rows (dict_id, ceil(log2(n_distinct_multi_sets)) +bits — kept apart from the singleton array because n_distinct_multi_sets +can be large in absolute terms even when multi-genome rows are a small +fraction of all rows, and a shared array would force every row, singletons +included, to pay the wider width), and a deduplicated dictionary of distinct +multi-genome sets (Elias-Fano-encoded byte offsets + a varint-encoded values +blob). Supporting primitives: PersistentFixedIntVec (arbitrary, +runtime-parameterised bit width, width 0 included — needed once an +EliasFano reopen bug surfaced, see below), PersistentRankSelectBitVec +(rank1/rank0/select1 on top of the crate's existing count_ones, via +common_traits::SelectInWord), EliasFano (composes the two). BinaryMatrix +(n, n_cols, row/fill_row, fill_sub_matrix, count_ones) is the +trait unifying dense and sparse at the call sites that need both +interchangeably; column-oriented methods (col, col_view, the +partial_*_dist_matrix family) stay dense-only — Sparse/Implicit panic +there (see the PersistentBitMatrix::open note above).

    +

    Measured on real data (one layer, 30.2M rows, 91 genomes):

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    densesparseratio
    on-disk size328.1MB43.7MB7.5x smaller
    build time / peak RSS—4.26s / 628MBper-layer, in-memory construction
    row access, sequential (2M reads)43ns/row32ns/rowsparse faster (smaller structure, better cache fit)
    row access, random (2M reads)409ns/row85ns/rowsparse ~4.8x faster — the --shannon/family-lookup access shape
    column access, one full column (30.2M rows)11.5ms993mssparse 86x slower — no native column method, every read decodes a full row to keep one bit
    +

    The row-access wins (compactness was the stated design goal, not speed) come +from dense's genome-major layout scattering a single row read across a much +bigger file — costlier than sparse's rank/select/varint decode once the file +is this much smaller. Column access is the flip side of the same layout +choice; nonzero_iter (below) is what removes it from the hot path that +actually matters (query).

    +

    Two reopen-correctness bugs, now fixed at the source: (1) EliasFano::open +re-derived its low-bits width from the persisted low-vector's own width +byte; the width-0 case was built with a dummy 1-bit placeholder (the +builder used to reject true width 0), silently doubling every reopened +value. Fixed by making PersistentFixedIntVec genuinely support width 0 +(no storage, get always 0). (2) An empty row (cardinality 0) used to be +recorded as a singleton at genome 0, indistinguishable on read-back from a +real singleton at genome 0. Fixed by routing cardinality-0 rows through +the dictionary path (a genuine empty entry).

    +

    Still deferred: full Alanko et al. subset-hierarchy compression (only the +exact-duplicate case is built), a sparse PersistentCompactIntMatrix +variant beyond what's described below, BRWT-style column-correlation +exploitation, and a row-major rewrite of +partial_jaccard_dist_matrix/partial_hamming_dist_matrix/etc. as a +per-row co-occurrence accumulation (O(Σ_rows k̄²)) so obikindex's +--metric distance-matrix path stops paying the column-access penalty on +Sparse — needs its own design pass (how it plugs into +BitPartials/ColumnWeights so both matrix types keep serving --metric), +not just "port the loop".

    +

    nonzero_iter — the one traversal per format

    +

    fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box<dyn Iterator<Item = (usize, usize, u32)> + 'a> +(item: index into slots, column, value) is the single foundational +per-format traversal primitive every consumer builds on — fill_sub_matrix +and a for_each-style callback are both trivial, free consumers of it +(.for_each(f) is Iterator::for_each; fill_sub_matrix drains the +iterator and scatters into out[][]), instead of two independently +hand-maintained traversals that can silently diverge. Box<dyn Iterator>, +not a bare impl Iterator: Columnar/Packed/Sparse/Implicit are +genuinely different concrete types and the method isn't on a +dyn-compatible trait (BinaryMatrix is used as dyn BinaryMatrix +elsewhere, and RPITIT methods aren't dyn-compatible) — one Box per +nonzero_iter call, not per cell.

    +

    Per-format implementation: PersistentSparseBitMatrix decodes natively, +buffered one row at a time (std::iter::from_fn over the existing +for_each_genome_in_row, no n_cols-wide allocation ever). Columnar/ +Packed loop columns, collecting each column's hits via +BitSliceView::nonzero_among_sorted/IntSliceView::nonzero_among_sorted +(a filter/filter_map over the vector-level get(), no new state +machine — std's own Filter iterator already is one) into one Vec via +.extend() (not flat_map: a flat_map closure can't lazily return +something borrowing its own captured sort permutation across separate calls +without either boxing per-column or fighting the borrow checker — eager +collection sidesteps it at zero extra cost, since fill_sub_matrix already +fully materialises anyway). Implicit is trivial (slots.iter().map(|&i| (i, 0, 1))).

    +

    This closed a real dispatch bug by construction, not by patching it: +PersistentBitMatrix::fill_sub_matrix/sub_matrix used to reimplement a +naive per-(column, slot) loop instead of delegating to +PersistentSparseBitMatrix's own efficient native method for Self::Sparse +— two hand-written traversals for the same format, free to drift apart, and +they had (obikphylo's own cache layer had independently built a parallel +enum wrapper that did dispatch correctly, evidence the bug was worked +around rather than fixed at the source). With fill_sub_matrix/sub_matrix +rewritten to drain nonzero_iter, there is only one traversal per format +left to get right. KmerPartition::query_partition_with +(obikindex/src/partition/query_layer.rs) — previously column-major +(for g { for slot { col_value } }, which for Sparse rebuilt a full row +per single cell touched, O(hits × n_cols) instead of O(hits) — the original +motivation for this primitive) — now issues one layer.nonzero_iter(&slot_list) +call per layer, format-agnostic. Dense/sparse query performance parity +confirmed on real data (previously sparse 30-50% slower than dense, +reproducibly; now within ~1-3% either way — noise level).

    +

    PersistentSparseCompactIntMatrix

    +

    The count-matrix analogue, built on PersistentSparseBitMatrix as its +"which columns are non-zero" support, values not deduplicated (unlike +the bit side, two rows can share the same non-zero column set — same +dict_id — while carrying different counts, so the dict-multiplicity +shortcut BitPartials for PersistentSparseBitMatrix uses doesn't carry +over). CountPartials here: a single row-major pass (row_major_pairwise +in sparse_intmatrix.rs, decodes each row once via +for_each_cell_in_row, nests over that row's own co-present columns) — +O(Σ k̄²) over populated rows, avoiding the naive O(n_cols² × n) +column-pair rescan. min(a,b)-based kernels (bray, relfreq_bray) need +no correction — they vanish when either side is absent. a·b and √(a·b) +kernels (euclidean/relfreq_euclidean/hellinger) do need one, since +(a-0)² = a² ≠ 0 unlike the min-based formulas — reconstructed from +per-column marginals via Σ(a-b)² = Σa²+Σb²-2Σab. +threshold_jaccard(1) shortcuts to BitPartials::partial_jaccard +(threshold 1 is exactly presence); threshold_jaccard(0) is closed-form +(every u32 is ≥ 0).

    +

    Wiring the threshold==1 shortcut surfaced two pre-existing formula bugs in +BitPartials for PersistentSparseBitMatrix (present since that type's own +introduction, never caught because no test had compared Sparse's raw +partial_* output against dense on real data — only the diagonal-blind +jaccard_dist_matrix/hamming_dist_matrix finalisations were tested), +fixed at the source:

    +
      +
    1. partial_jaccard's diagonal used to be (0, 2×col_weights[i]) instead + of a genuine self-comparison (col_weights[i], col_weights[i]) — + col_weights_and_pair_counts's inter never pairs a column with itself + by construction, so the diagonal needs its own case.
    2. +
    3. partial_hamming's off-diagonal formula itself was wrong: it computed + total - union (rows where neither column is present) instead of the + actual Hamming distance col_weights[i] + col_weights[j] - 2×inter[i,j] + (symmetric-difference size) — the two only coincide when + col_weights[i] + col_weights[j] == total, which small/synthetic test + data can easily satisfy by accident.
    4. +

    Aggregation traits (matrix level)

    ColumnWeights

    trait ColumnWeights: Send + Sync {
    diff --git a/DevDoc/implementation/obipipeline.refs/index.html b/DevDoc/implementation/obipipeline.refs/index.html
    index 46dace87..ba0cda60 100644
    --- a/DevDoc/implementation/obipipeline.refs/index.html
    +++ b/DevDoc/implementation/obipipeline.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/obipipeline/index.html b/DevDoc/implementation/obipipeline/index.html
    index edebe6b6..8c864f3f 100644
    --- a/DevDoc/implementation/obipipeline/index.html
    +++ b/DevDoc/implementation/obipipeline/index.html
    @@ -1293,7 +1293,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/obitaxonomy/index.html b/DevDoc/implementation/obitaxonomy/index.html
    index b1886a24..a9d00b9e 100644
    --- a/DevDoc/implementation/obitaxonomy/index.html
    +++ b/DevDoc/implementation/obitaxonomy/index.html
    @@ -1305,7 +1305,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/partition_layer_cache/index.html b/DevDoc/implementation/partition_layer_cache/index.html
    index 1cce3095..07dd753c 100644
    --- a/DevDoc/implementation/partition_layer_cache/index.html
    +++ b/DevDoc/implementation/partition_layer_cache/index.html
    @@ -1563,7 +1563,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/persistent_bit_vec.refs/index.html b/DevDoc/implementation/persistent_bit_vec.refs/index.html
    index 452c7db1..bccfdd30 100644
    --- a/DevDoc/implementation/persistent_bit_vec.refs/index.html
    +++ b/DevDoc/implementation/persistent_bit_vec.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/persistent_bit_vec/index.html b/DevDoc/implementation/persistent_bit_vec/index.html
    index 3e40d825..a8701785 100644
    --- a/DevDoc/implementation/persistent_bit_vec/index.html
    +++ b/DevDoc/implementation/persistent_bit_vec/index.html
    @@ -1405,7 +1405,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/persistent_compact_int_vec.refs/index.html b/DevDoc/implementation/persistent_compact_int_vec.refs/index.html
    index 9e306acb..7d6b5deb 100644
    --- a/DevDoc/implementation/persistent_compact_int_vec.refs/index.html
    +++ b/DevDoc/implementation/persistent_compact_int_vec.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/persistent_compact_int_vec/index.html b/DevDoc/implementation/persistent_compact_int_vec/index.html
    index 05ea4d9f..ab5d8aa2 100644
    --- a/DevDoc/implementation/persistent_compact_int_vec/index.html
    +++ b/DevDoc/implementation/persistent_compact_int_vec/index.html
    @@ -1388,7 +1388,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/pipeline.refs/index.html b/DevDoc/implementation/pipeline.refs/index.html
    index 3ab74967..985e28c7 100644
    --- a/DevDoc/implementation/pipeline.refs/index.html
    +++ b/DevDoc/implementation/pipeline.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/pipeline/index.html b/DevDoc/implementation/pipeline/index.html
    index a697eee7..fc08d198 100644
    --- a/DevDoc/implementation/pipeline/index.html
    +++ b/DevDoc/implementation/pipeline/index.html
    @@ -1254,7 +1254,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/select/index.html b/DevDoc/implementation/select/index.html
    index c98f84e1..c3cb4add 100644
    --- a/DevDoc/implementation/select/index.html
    +++ b/DevDoc/implementation/select/index.html
    @@ -1382,7 +1382,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/storage.refs/index.html b/DevDoc/implementation/storage.refs/index.html
    index 03bfe3b3..06d67528 100644
    --- a/DevDoc/implementation/storage.refs/index.html
    +++ b/DevDoc/implementation/storage.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/storage/index.html b/DevDoc/implementation/storage/index.html
    index 7fd77097..ce4b99a7 100644
    --- a/DevDoc/implementation/storage/index.html
    +++ b/DevDoc/implementation/storage/index.html
    @@ -1326,7 +1326,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/superkmer.refs/index.html b/DevDoc/implementation/superkmer.refs/index.html
    index 946e4da1..6622b610 100644
    --- a/DevDoc/implementation/superkmer.refs/index.html
    +++ b/DevDoc/implementation/superkmer.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/superkmer/index.html b/DevDoc/implementation/superkmer/index.html
    index d5267a43..6d6069fe 100644
    --- a/DevDoc/implementation/superkmer/index.html
    +++ b/DevDoc/implementation/superkmer/index.html
    @@ -1232,7 +1232,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/unitig_evidence.refs/index.html b/DevDoc/implementation/unitig_evidence.refs/index.html
    index 1dfbaa9c..4118486a 100644
    --- a/DevDoc/implementation/unitig_evidence.refs/index.html
    +++ b/DevDoc/implementation/unitig_evidence.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/implementation/unitig_evidence/index.html b/DevDoc/implementation/unitig_evidence/index.html
    index a66adbd3..32ef44a3 100644
    --- a/DevDoc/implementation/unitig_evidence/index.html
    +++ b/DevDoc/implementation/unitig_evidence/index.html
    @@ -1354,7 +1354,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/index.html b/DevDoc/index.html
    index e6629e0a..06cc62b0 100644
    --- a/DevDoc/index.html
    +++ b/DevDoc/index.html
    @@ -1228,7 +1228,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/index.refs/index.html b/DevDoc/index.refs/index.html
    index 5f0a7f61..b229367c 100644
    --- a/DevDoc/index.refs/index.html
    +++ b/DevDoc/index.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/installation/index.html b/DevDoc/installation/index.html
    index 42e8a762..d044a12d 100644
    --- a/DevDoc/installation/index.html
    +++ b/DevDoc/installation/index.html
    @@ -1314,7 +1314,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/kmers.refs/index.html b/DevDoc/kmers.refs/index.html
    index b12faff2..f789c1f2 100644
    --- a/DevDoc/kmers.refs/index.html
    +++ b/DevDoc/kmers.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/kmers/index.html b/DevDoc/kmers/index.html
    index a707331c..89637ba8 100644
    --- a/DevDoc/kmers/index.html
    +++ b/DevDoc/kmers/index.html
    @@ -1227,7 +1227,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/sitemap.xml.gz b/DevDoc/sitemap.xml.gz
    index 03a78e939757e3344acaf2aa3115fcca5d967fa9..15bde70bddf05ec6e4e702da2414775779e38149 100644
    GIT binary patch
    delta 13
    Ucmb=gXP58h;9#g
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/theory/encoding/index.html b/DevDoc/theory/encoding/index.html
    index e8a2dbb1..abfa12e4 100644
    --- a/DevDoc/theory/encoding/index.html
    +++ b/DevDoc/theory/encoding/index.html
    @@ -1199,7 +1199,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/theory/entropy.refs/index.html b/DevDoc/theory/entropy.refs/index.html
    index 49c1b70d..1a9e9081 100644
    --- a/DevDoc/theory/entropy.refs/index.html
    +++ b/DevDoc/theory/entropy.refs/index.html
    @@ -1124,7 +1124,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/theory/entropy/index.html b/DevDoc/theory/entropy/index.html
    index 5e201572..f913aeca 100644
    --- a/DevDoc/theory/entropy/index.html
    +++ b/DevDoc/theory/entropy/index.html
    @@ -1265,7 +1265,7 @@
       
         
       
    -    Sibling annex (discussion)
    +    Sibling annex
       
     
         
    diff --git a/DevDoc/theory/evolutionary_distances/index.html b/DevDoc/theory/evolutionary_distances/index.html
    index 77c3d6cf..57634792 100644
    --- a/DevDoc/theory/evolutionary_distances/index.html
    +++ b/DevDoc/theory/evolutionary_distances/index.html
    @@ -932,6 +932,17 @@
         
       
       
    +
  • + +
  • + + + + Automatic α estimation (--gamma-shape auto) + + + +
  • @@ -943,6 +954,45 @@ +
  • + +
  • + + + + α-estimation literature survey and design discussion (2026-09-11) + + + + +
  • + +
  • + + + + Sharing one sample across algorithms (--sankoff/--tnt/--phyg/--iqtree + snp-* --distance) — implemented 2026-09-11 + + + + + +
  • @@ -1691,7 +1741,7 @@ - Sibling annex (discussion) + Sibling annex @@ -2268,6 +2318,17 @@ +
  • + +
  • + + + + Automatic α estimation (--gamma-shape auto) + + + +
  • @@ -2279,6 +2340,45 @@ +
  • + +
  • + + + + α-estimation literature survey and design discussion (2026-09-11) + + + + +
  • + +
  • + + + + Sharing one sample across algorithms (--sankoff/--tnt/--phyg/--iqtree + snp-* --distance) — implemented 2026-09-11 + + + + + +
  • @@ -4476,9 +4576,75 @@ F84/TN93 and is included below. snp- prefix on every CLI value.

    +Γ rate-heterogeneity modifier, applicable to snp-jc, snp-k2p, snp-k81, snp-t92, snp-f84, snp-hky85, snp-tn93 (not snp-raw, nothing to correct; not snp-logdet, no standard gamma formulation) — same -formula as the base correction, weighted by a shape parameter α supplied -by the user (--gamma-shape <alpha>), not estimated by ML. A modifier on -existing values, not a separate enum arm per distance.

    +formula as the base correction, weighted by a shape parameter α either +supplied by the user (--gamma-shape <alpha>) or estimated from the data +(--gamma-shape auto/estimate, method-of-moments — not ML; see +"Automatic α estimation" below). A modifier on existing values, not a +separate enum arm per distance.

    +

    Automatic α estimation (--gamma-shape auto)

    +

    Correction (verified against the primary source, 2026-09-11): Jin & +Nei (1990) itself ("Limitations of the Evolutionary Parsimony Method of +Phylogenetic Analysis", Mol. Biol. Evol. 7(2):82–102 — the paper this +whole +Γ correction is cited from, confirmed algebraically to match this +codebase's corrected_log/k2p exactly against their eq. A4, general, and +A8, the a = 1 case) contains no data-driven α-estimation procedure. +Their own recommendation (p. 98) is a fixed default: "we suggest that the +gamma distance with a = 1 [eq. A8] be used. However, one may choose a +different gamma distance, estimating a from data. Wilson et al. (1989) +recently used a distance with a = 1/2 for restriction-site data of +mitochondrial DNA in hominoids." — i.e. Jin & Nei explicitly punt +data-driven estimation to a different paper (Wilson et al. 1989), not +read/verified here. The estimator below is therefore not "Jin & Nei's +method" under any framing — that attribution (present in an earlier +revision of this section) was wrong, not just under-cited.

    +

    Implemented (PartitionDispersion, obikphylo/src/siblings/algorithms/pairwise.rs) +as an independent method-of-moments estimator, unrelated to any specific +published procedure: pools substitution counts by partition rather +than by genome pair, during the same reduce_pairwise pass that builds +PairwiseTally (no second scan).

    +

    For partition i: n_i = substitutions pooled over every genome pair, +L_i = eligible loci pooled over every genome pair, R_i = n_i / L_i. +Modeling among-site rate heterogeneity the same way as the +Γ correction +itself (a Gamma(α, α)-distributed, mean-1, multiplicative rate shared by +every locus in a partition — the classical Poisson-Gamma/negative-binomial +mixture, the general identity behind gamma-rate-heterogeneity corrections, +also behind Uzzell & Corbin's (1971) original observation that substitution +counts across sites/regions are over-dispersed relative to Poisson):

    +
    \[ +\mathbb{E}[R_i] = \mu \qquad \mathrm{Var}[R_i] = \frac{\mu}{L_i} + \frac{\mu^2}{\alpha} +\]
    +

    Weighting each partition's squared deviation by its own L_i cancels the +Poisson term before attributing what's left to α:

    +
    \[ +\hat\mu = \frac{\sum_i n_i}{\sum_i L_i} \qquad +V = \frac{\sum_i L_i (R_i-\hat\mu)^2}{\sum_i L_i} \qquad +\hat\alpha = \frac{\hat\mu^2}{V - \hat\mu/\bar L} +\]
    +

    where \bar L is the mean partition size. Returns "no estimate" (falls +back to the uncorrected formula, warns) when fewer than 2 partitions have +data, \hat\mu \le 0, or V doesn't exceed the Poisson floor +\hat\mu/\bar L — no detectable over-dispersion, α would be unbounded.

    +

    Caveat, stated explicitly rather than left implicit: unlike every +closed-form correction in snp_distance.rs (each verified line-by-line +against ape's dist_dna.c, and now also against Jin & Nei 1990 directly +for the base +Γ formula), this estimator is derived from first +principles (the general Poisson-Gamma/negative-binomial identity) with no +primary-source procedure behind it at all — not Jin & Nei's (confirmed +above), and Wilson et al. (1989), the paper they point to instead, hasn't +been read/verified either. Mathematically self-consistent (re-derived and +checked, not guessed), but a from-scratch method, not a literature +implementation. If --gamma-shape needs a value with a literature +pedigree rather than an estimated one, Jin & Nei's own stated default, +α = 1 (--gamma-shape 1), is the better-supported choice today.

    +

    Deliberately not gated by --sankoff-ratio-ceiling the way +base_pair_tally is (same precedent as cardinality_tally — see its own +doc comment): that filter excludes individual saturated pairs from a +composition estimate computed once at the very end, from the complete +PairwiseTally; the partition axis needed here only exists transiently, +one partition at a time, while PairwiseTally is still being built — long +before any pair's final SNP ratio (and thus its ratio_ceiling eligibility) +is known. --exclude-genome isn't applied either, matching +reduce_pairwise's own raw per-pair fold.

    Implemented now: snp-raw, snp-jc, snp-k2p, snp-k81, snp-f81, snp-t92, snp-tn93, snp-tv, all with +Γ except raw/tv — see "Exact formulas" below. snp-tajima-nei, snp-f84, snp-hky85, @@ -4591,6 +4757,430 @@ special-casing K80). K81/T92/TN93's gamma branches follow the same mechanical substitution but weren't independently checked against an ape-equivalent reference for those three specifically — flagged here, not silently assumed correct.

    +

    α-estimation literature survey and design discussion (2026-09-11)

    +

    Follow-up discussion after implementing PartitionDispersion::estimate_alpha +(previous section), triggered by the user pointing out that the literature +conflates two distinct problems: estimating α from an alignment versus +using a given α in a Jin-Nei-style corrected distance. Jin & Nei (1990) +itself is squarely in the second camp — α is a user-supplied parameter +there, not something their paper estimates (confirmed by reading the paper +directly, see previous section). This section catalogs the actual +α-estimation literature and records which approaches were considered and +why each was or wasn't adopted.

    +

    Bibliography (methodological papers, not exhaustive users of +Γ):

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MethodReferenceNeeds a tree?Estimates "true" α or a task-specific one?Cost
    ML, continuous Γ (foundational)Gu, Fu & Li (1995), MBE 12:546–557 — read in fullyes, topology fixed a prioritrue α (+ θ jointly, invariant+Γ)very high, ≤5–6 taxa in practice
    ML, discrete ΓYang (1993/94), MBE 10:1396; JME 39:306yestrue αhigh
    Parsimony-based ML on inferred changesYang & Kumar (1996), MBE 13:650yes, equal branch lengths assumedtrue α, biased when branches unequallow
    Corrected substitutions/site → NB fitGu & Zhang (1997), MBE 14:1106 — read in fullyes, + ancestral reconstructiontrue α, close to ML accuracylow (given a tree)
    Topology-optimizing grid searchGuindon & Gascuel (2002), MBE 19:534 — read in fullyes, but built from the candidate distances themselves (BIONJ)not true α — deliberately biased upward for topological accuracylow–moderate (grid × tree build)
    Free (non-parametric) rate distributionSusko et al. (2003), Syst. Biol. 52:594yestests the Γ assumption itselfhigh
    Parsimony counts/site → NB fitWakeley (1993), JME 37:613 — read in fullyes, NJ tree + Fitch parsimony ancestral reconstructiontrue α, biased upward by parsimony (conservative)low (given a tree)
    Poisson-Gamma/negative-binomial, foundationalUzzell & Corbin (1971), Science 172:1089 — not read, only seen via citationsunknown — likely also count-per-site/tree-based, unverifiedtrue α (indirect, via NB overdispersion)unknown
    Bayesian posterior over αBPP/MrBayes/BEAST2/RevBayesyesposterior, not point estimatevery high
    +

    Correction (2026-09-11, after reading Wakeley in full): an earlier +revision of this table filed Wakeley (1993) under "no tree needed" — +wrong. Wakeley's method: build a tree (neighbor-joining, Saitou & Nei +1987, on the real mtDNA data; coalescent-simulated for the simulation +study), reconstruct ancestral states at internal nodes by Fitch (1971) +parsimony, count the minimum number of changes per site this implies, +then fit either a two-rate Poisson mixture or a gamma-distributed-rates +model (⟹ negative binomial, parameters by the methods of Cohen 1965 and +Johnson & Kotz 1969) to the resulting per-site count distribution — the +same general "tree + ancestral reconstruction + per-site counts + NB fit" +shape as Gu & Zhang (1997), just using raw parsimony counts directly +rather than Gu & Zhang's multiple-hit-corrected k̂. Confirmed by +simulation (Wakeley's own Fig. 1) that parsimony underestimates both the +mean and (more severely) the variance of per-site change counts, which +biases α upward (toward apparent uniformity) — consistent with Gu & +Zhang's own citation of Wakeley for exactly this bias, and with the +general "parsimony overestimates α" pattern noted throughout this +literature. Real hypervariable-region-1 mtDNA data gave α̂ ≈ 0.44–0.60 +per data set, 0.47 combined — despite the conservative bias, still +solidly in the "strong heterogeneity" range, which is the paper's own +point: the method is biased but usefully conservative, not useless.

    +

    Where PartitionDispersion::estimate_alpha actually sits: nowhere in +this table's tree-dependent rows. It shares only the abstract +mathematical identity (Poisson-Gamma mixture ⟹ negative binomial +over-dispersion) with Wakeley/Uzzell-Corbin/Gu&Zhang, applied via a +genuinely different, tree-free and ancestral-reconstruction-free route: +pairwise genome-to-genome substitution counts pooled by partition +instead of per-site counts pooled across an alignment's columns after +ancestral state reconstruction. As far as this survey has established +(Uzzell & Corbin not read, so not fully ruled out), no tree and no +ancestral reconstruction of any kind appears to be a genuinely +distinguishing property of PartitionDispersion, not something it +inherits from prior art — consistent with it being an independent +derivation rather than a literature implementation (see previous +section's caveat, now on firmer footing).

    +

    Why a tree-free estimate is even valid — the identifiability question. +Raised directly by the user after reading Gu, Fu & Li (1995), which states +explicitly: "one cannot estimate the rate heterogeneity when the number +of sequences is <3." This is a real non-identifiability result, not a +practical inconvenience: with only 2 sequences, a site's history is +summarized by one binary observation (differ / don't differ), collapsed +across all sites into two aggregate numbers (transition and transversion +proportions) — divergence time and among-site rate variance are +confounded in that pair of numbers, with no way to separate them. Every +method surveyed above (Gu,Fu&Li, Yang, Wakeley, Gu&Zhang) needs ≥3 +sequences sharing one evolutionary history (a tree) specifically to get +multiple independent looks at the same site's rate across different +lineages, which is what identifies its variance.

    +

    PartitionDispersion doesn't violate this, because it isn't estimating +the same quantity. The literature above estimates rate variation across +sites, within one shared tree. PartitionDispersion estimates rate +variation across partitions, pooled over every genome pair in the +index — each genome pair stands in for one independent draw of a +partition's relative rate, the same identifying role multiple lineages +play in the tree-based methods, just substituting "genome pair" for +"lineage" and "partition" for "site." With P genome pairs (potentially +in the hundreds here) all contributing to every partition's pooled count, +the ≥3-samples identifiability requirement is met by the pair count, not +by tree depth — closer in spirit to Jin & Nei's own original motivating +scenario (comparing several genomic regions across one fixed panel of +taxa) than to single-tree site-rate estimation.

    +

    This substitution carries its own assumption, which must be named rather +than left implicit: that a partition's relative rate (fast/slow) is +reasonably stable across genome pairs, not just across sites within +one pair — the direct analogue of the "no lineage-specific rate variation, +only site-specific" assumption every tree-based +Γ method already makes, +just moved from lineage→pair and site→partition. If this breaks — e.g. an +index mixing very closely related and very divergent genome pairs, where a +partition's saturation behavior differs qualitatively between the two — +the estimate could be misled in a way a tree-based method would at least +have the topology to detect and a tree-free, pool-everything method +cannot. Not yet tested against real data with strongly heterogeneous +pairwise divergence; worth keeping in mind as the main open validity +question for --gamma-shape auto, not the citation question (now settled) +this whole discussion started from.

    +

    Gu, Fu & Li (1995), read in full — the foundational ML method, not a +candidate to implement. Models rate variation as invariant+Γ (θ = +proportion of invariant sites, α = gamma shape among the variable sites), +and derives the exact joint likelihood over nucleotide configurations +across all n sequences on a fixed, a-priori-known tree topology — +for 3 sequences this is closed-form (Jukes-Cantor), for n sequences it's +a sum over 4^n configurations weighted by coefficients tied to the +specific topology's branching structure (their eq. 32–34), with branch +lengths, α, and θ jointly optimized by Newton-Raphson (Hessian over all +free parameters). The site-rate integral has the same closed form +E = θ + (1-θ)(1+D/α)^{-α} that underlies +Γ distance corrections +generally (the same mechanism as corrected_log, not a coincidence — Jin & +Nei's own formula is the θ=0 special case applied to a pairwise D +rather than a whole-tree one). Explicitly stated by the authors as +infeasible beyond about 5–6 taxa (their own simulations stop at 5); not a +candidate for obikmer's typical genome counts. Notable contribution worth +keeping in mind regardless: ρ = (1+θα)/(1+α) is shown to be a more robust +summary of rate heterogeneity than α alone whenever θ isn't negligible +— α and θ are confounded (nearly-invariant sites can come from either a +small α's left tail or from a nonzero θ), so α alone can be very +unstable while ρ stays well-behaved. PartitionDispersion has no θ +term at all (no invariant-site component), so this confound doesn't arise +for it the same way, but it's worth remembering if an invariant+Γ variant +is ever considered.

    +

    Gu & Zhang (1997), read in full — rejected for now. Their procedure: +(1) a tree with least-squares branch lengths must already exist; (2) +ancestral states are reconstructed at every internal node (they use Zhang & +Nei 1997's likelihood method, but note plain parsimony would work, just +less accurately); (3) per site, branches are split into "changed"/"unchanged" +given the ancestral reconstruction, and the expected substitution count +k̂ (corrected for multiple hits) is obtained by solving their eq. (7) +(or eq. 12 for the generalized model) numerically — a root-find per site, +not a closed form; (4) the site-level k̂ values (real-valued, not +integers) are fit to a negative binomial (Uzzell & Corbin 1971) by ML to +get α — again no closed form, numerical optimization. This is a real +per-site pipeline requiring a tree before distances/α can be computed — +the reverse of obikmer's current dependency order (snp_distance computes +distances, which --nj/--upgma only turn into a tree afterward). Their +own answer to this chicken-and-egg problem is an iterative +distance→tree→α→distance loop, which they themselves flag as expensive. +Adopting this would mean a new ancestral-reconstruction subsystem (even a +parsimony/Fitch-only version) plus a per-family nonlinear solve plus an NB +ML fit — a much bigger addition than PartitionDispersion, not something +to build without a concrete need beyond "closer to Gu&Zhang's own +simulated accuracy than our moment estimator."

    +

    Guindon & Gascuel (2002), read in full — plausible future addition, not implemented. +Their Q criterion: for a candidate α, build a tree (they use BIONJ) from +the α-corrected distance matrix; for every internal branch, group taxa into +the 4 subtrees it separates (A, B, C, D), compute mean inter-subtree +distances \(\bar\delta_{AB}\), \(\bar\delta_{CD}\), etc., and let S ≤ M ≤ L +be the three pairings' sums ({AB,CD}, {AC,BD}, {AD,BC}); the branch's +reliability score is Q_branch = L − M (zero when the four points are +perfectly tree-additive). The whole-tree Q is the mean over internal +branches (negative/zero branches excluded). α* = argmin_α Q(α) over a +grid (their run: ~60–100 points, 0.1 to 5000, finer spacing where +sensitivity is highest). Complexity: O(n²l) once for the base +frequencies, then O(n²r) for the r candidate distance matrices, then +O(n³r) worst case for building r trees and evaluating Q on each +(same order as the tree-building step itself, so "free" relative to it).

    +

    Central finding, the opposite direction from an initial misreading in +this discussion: α_opt (Guindon-Gascuel's topology-optimal value) is +always ≥ the true α, not the reverse — underestimating rate +heterogeneity (i.e., picking a larger α than reality) reduces the variance +of distance estimates and thereby improves NJ/BIONJ topological accuracy, +especially when the molecular clock roughly holds (where α_opt → ∞, i.e. +no correction at all is topologically best). This is an empirical +regularity observed across their tested conditions (η ∈ {0.5, 2.0}, true +α ∈ {0.1, 0.7}, 20-taxon trees) — not a theorem — so treating it as a hard +guarantee would be another unverified-claim mistake of the kind this whole +discussion has been correcting.

    +

    Proposed integration (not yet implemented): since α_true ≤ α_opt +empirically, PartitionDispersion::estimate_alpha()'s output could seed +the lower bound of Guindon-Gascuel's grid (search [α̂, ∞) instead of +[0.1, 5000]), narrowing the grid substantially and reusing the existing +estimator rather than needing an independent "true α" source — this also +sidesteps Gu & Zhang's tree-first dependency problem entirely, since +Guindon-Gascuel's own grid already builds a tree per candidate. Still +requires: (a) a BIONJ implementation (see below), (b) the Q-criterion +subtree-grouping logic, (c) a grid-search driver. Not started.

    +

    BIONJ availability in the Rust ecosystem — checked 2026-09-11, none found. +Searched crates.io directly (API query for "bionj": 0 results) and read the +docs.rs pages of the most plausible candidates: phylotree (builds/reads/ +manipulates trees but has no distance-matrix reconstruction — no NJ, no +BIONJ, no UPGMA despite web-search summaries claiming otherwise, corrected +after checking the actual docs), speedytree (NJ only — Canonical/RapidNJ/ +Hybrid variants, explicitly no BIONJ), nj (plain NJ only), phylo (no +distance-based reconstruction at all). obikphylo::neighbor_joining +(tree.rs) is plain NJ, not BIONJ — BIONJ (Gascuel 1997) differs only in +how branch lengths are weighted during agglomeration (accounts for +distance-estimate variance, not just the sum), a modest delta over an +existing NJ implementation rather than a rewrite, but would need to be +written in-house if the Guindon-Gascuel integration above is ever pursued +(their own paper notes other tree-building methods gave similar results in +their simulations, so plain NJ may be an acceptable substitute if BIONJ +proper is judged not worth the effort).

    +

    Sharing one sample across algorithms (--sankoff/--tnt/--phyg/--iqtree + snp-* --distance) — implemented 2026-09-11

    +

    Correctness bug, not just a performance one. --sankoff/--tnt/--phyg/ +--iqtree (via SiblingExt::sankoff_bundle) and a snp-* --distance +(via SiblingExt::snp_distance) always consume the exact same selection +parameters when requested together in one obikmer phylo invocation +(n/free_loss/no_ambiguity/excluded-set/entropy_bias — the CLI has +no way to give them different values in one run, see cmd/phylo/args.rs). +Before this fix, each independently called sample_index, and +sample_layer's rand::rng() (subsample.rs:235) is a thread-local +generator that advances across calls, not reseeded each time — so the +second call silently drew a different random sample of sites than the +first, even with identical parameters. Verified empirically (throwaway +8-genome index): running snp-jc alone vs. combined with --sankoff +produced visibly different distance matrices from the same index/params. +This defeated the actual point of combining these flags — running several +algorithms (Sankoff/TNT/PhyG/IQ-TREE calibration, a snp-* distance) on +one identical site selection for direct comparison.

    +

    Fix: SankoffBundle (sankoff.rs) now retains its PairwiseTally/ +PartitionDispersion internally (previously consumed into raw/ +base_pair_tally/cardinality_tally and dropped) and exposes +SankoffBundle::snp_distance(kind, gamma_shape), which computes the +matrix from that same already-sampled tally — no second sample_index +call. The shared post-sampling logic (the --gamma-shape support check, +alpha resolution, final matrix build — previously all inline in the +snp_distance() free function) was factored into +pub(crate) fn distance_matrix(tally, dispersion, kind, gamma_shape) in +snp_distance.rs, called by both the standalone snp_distance() (after +its own fresh sampling) and SankoffBundle::snp_distance (reusing the +bundle's). cmd/phylo/mod.rs keeps the Option<SankoffBundle> alive past +the Sankoff-family if block and, when a snp-* --distance is also +requested, calls bundle.snp_distance(...) instead of +cache.snp_distance(...) whenever a bundle was built — since both branches +are driven by the same args.* fields, the parameters trivially always +match when both fire; no runtime "do the params match" check needed. +Verified end-to-end: the reused path logs (reusing the Sankoff bundle's +sample) and the snp_distance stage timer reads 0ms (formula(kind) +is O(n²) post-processing, no I/O), vs. ~150-230ms for a fresh sample on +the same tiny test index.

    +

    Explicit --session (implemented 2026-09-12)

    +

    The in-process fix above only covered reuse within one command. +--session DIR (cmd/phylo/args.rs) names a directory (outside the +index) that persists the snp-* --distance sample — the +PairwiseTally/PartitionDispersion pair sample_index would otherwise +rebuild from scratch — across separate obikmer phylo invocations.

    +

    Crate placement, as planned: a new obiksession crate +(src/obiksession/), domain-agnostic — no PairwiseTally/"site" concept +anywhere in it. Session::open(dir, params: &[u8], force: bool) handles +directory lifecycle and the params-conflict check against an opaque byte +blob; Session::store(name, bytes)/Session::restore(name) -> Option<Mmap> +handle a checksummed artifact cache. obikphylo +(siblings/algorithms/session_cache.rs, shared by snp_distance.rs and +sankoff.rs — see below) is the consumer: it serializes its own +PairwiseTally/PartitionDispersion/SnpAlignment to bytes (via rkyv) +and hands them to obiksession, which never sees their type.

    +

    Serialization: rkyv 0.8.18, added as planned. One deviation from the +original zero-copy pitch, scoped down deliberately: artifacts are restored +via rkyv::from_bytes (full owned deserialize) rather than rkyv::access +(zero-copy over the mmap). True zero-copy would require every +PairwiseTally query method (categories, base_freq, pair, ...) to +work generically over Archived<PairwiseTally> as well as the owned type +— a separate, larger change not needed to get the actual win (skipping +sample_index's expensive re-scan of the sibling annex; deserializing an +already-in-memory-sized count array is comparatively cheap CPU, not I/O). +#[derive(Archive, Serialize, Deserialize)] was added to PairStats, +PairwiseTally, and PartitionDispersion (pairwise.rs) for this.

    +

    Startup behavior, as designed: Session::open creates DIR if +missing (fresh, params recorded); if DIR already holds different saved +params, cmd/phylo reports a hard error and exits rather than silently +preferring one side, unless --session-force is given (which discards the +directory's cached artifacts and starts over under the new params) — +implements exactly the "explicit escape hatch, not silent override" +decision from the design discussion. obiksession itself doesn't decode +either side's bytes to produce a field-by-field diff (true to "opaque +blob," left to whoever needs it); cmd/phylo's conflict message is +currently a single generic sentence listing which flags could be the +cause, not a computed diff — an honest v1 simplification, not a limitation +of obiksession itself.

    +

    Per-artifact caching, as designed: only PairwiseTally/ +PartitionDispersion are cached (snp_distance.rs's restore_tally/ +store_tally) — ratio_ceiling/gamma_shape remain pure post-processing +over whatever tally is in hand, cached or fresh, exactly as planned.

    +

    Locking, reconsidered and kept as the whole-session DirLock. First +written up as a corner cut ("per-entry locking was sketched, not +implemented"). On actually working through the concrete race, whole-session +locking turned out to be the correct choice, not a simplification: +--session-force resets a session by deleting its whole artifacts/ +directory. If the directory lock were only held briefly (during +Session::open, then released so a long-running Session doesn't block +other processes from touching different artifacts), a second process +calling open(..., force: true) could acquire the lock, see mismatched +params, and wipe artifacts/ while a first process's already-open +Session is mid-checkpoint — genuine corruption risk, not just wasted +work. Holding the lock for the Session's entire lifetime makes that +sequence impossible: a force reset must wait for every currently-open +Session on that directory to be dropped first. The concurrency this +gives up (two different processes reading/writing different artifacts +of the same session at once) isn't exercised by anything today — only one +code path (snp_distance/sankoff_bundle, sequentially per invocation) +uses sessions at all.

    +

    Per-layer chunked/resumable dumps — implemented. subsample::sample_index +gained a skip_layers: usize parameter (skip the first N (partition, +layer) pairs, in its own deterministic iteration order, entirely — no +I/O, no on_layer call) and its on_layer callback gained a raw_index +argument (that pair's 0-based position in the same order), so a caller can +tell exactly how far a run has gotten. snp_distance/sankoff_bundle +checkpoint their tally/dispersion (and, for sankoff_bundle, the +in-progress SnpAlignment) to the session every +session_cache::CHECKPOINT_INTERVAL_LAYERS (8) layers or +CHECKPOINT_INTERVAL (30s) of wall time, whichever comes first — bounding +how much completed scanning a crash can lose, without paying a full +tally-rewrite on every one of what can be thousands of layers. A progress +artifact records how many layers are reflected in the checkpoint, with a +PROGRESS_COMPLETE (u64::MAX) sentinel once every layer has been +processed (distinct from any real layer count, so "finished" is never +confused with "stopped after N layers where N happens to equal the total").

    +

    Verified with a real kill: built a 10-genome/64-partition index, ran +--distance snp-jc --session DIR under timeout 0.4, confirmed a +checkpoint at progress=8 (exactly CHECKPOINT_INTERVAL_LAYERS) was on +disk, then re-ran the same command and confirmed it resumed from layer 8 +and completed successfully — 0-effort verification that the mechanism +itself is real, not just plausible-sounding.

    +

    Important correction from that same test — resume is not bit-reproducible, +and shouldn't be. The output matrix from the resumed run differed from +an uninterrupted control run against the same index/params. Root cause: +sample_layer's random draws come from rand::rng(), a per-process, +OS-seeded generator with no continuity across a process boundary — the +layers processed before a kill and the layers processed after a resume +come from two independent random streams. An earlier revision of this doc +(and of the code comments) claimed resume "reproduces the exact same +sample an uninterrupted run would have" — that's wrong, and was corrected +after the user pointed out the actual reasoning error: true randomness has +no notion of continuity in the first place, so there was never a +correctness requirement for a resumed run to match an uninterrupted one +bit-for-bit. The sample produced after a resume is exactly as legitimate a +random draw as one from an uninterrupted run — just a different one, +same as running the command twice without --session already gives two +different samples today. That variance is explicitly wanted (see below), +not a defect to fix. Comments in subsample.rs/session_cache.rs were +corrected to state this plainly instead of the false reproducibility claim.

    +

    Explicitly declined: deterministic/seeded sampling. Raised as a +possible fix for the above (seed sample_layer's RNG from a per-session +value plus (partition, layer), making a resume reproduce bit-identical +results, and, if applied everywhere, making any two runs with identical +parameters produce identical samples). The user explicitly declined this +for now: repeated runs are relied on to measure sampling variance, and a +--seed flag would need its own explicit design if wanted later — this is +not an accidental gap, it's a stated preference to revisit only if asked.

    +

    SankoffBundle integration — implemented. sankoff_bundle accepts +the same session: Option<&Session> and uses the same artifact names +(session_cache::{TALLY_ARTIFACT, DISPERSION_ARTIFACT, PROGRESS_ARTIFACT}) +as snp_distance, plus its own ALIGNMENT_ARTIFACT (sankoff.rs) folded +in lockstep with the tally at every checkpoint (same on_layer call +produces both, so they can never drift out of sync). Consequence verified +directly: a tally checkpointed by --sankoff --session DIR is restored by +a later, separate --distance snp-k2p --session DIR invocation with no +--sankoff at all, and vice versa — cross-consumer reuse, not just +same-command reuse, exactly the synergy this design aimed for.

    +

    Per-artifact caching, as designed: only the tally/dispersion (plus, +for sankoff_bundle, the alignment) are cached — ratio_ceiling/ +gamma_shape remain pure post-processing over whatever tally is in hand, +cached or fresh, exactly as planned.

    +

    Still not done — genuine follow-up, not a justified tradeoff: +exhaustive-sample caching for --subsample-varying exploration. The +design discussion's suggestion — cache the exhaustive (n = None) sample +once and subsample from it on each run instead of one session per exact +n — wasn't implemented; a session today is still scoped to one exact +parameter tuple, --subsample included.

    Output format: PHYLIP-relaxed by default for the distance matrix

    Implemented. The primary distance-matrix output (_dist.csv today) gains multiple formats: PHYLIP-relaxed becomes the diff --git a/DevDoc/theory/indexing.refs/index.html b/DevDoc/theory/indexing.refs/index.html index 890844ab..8dcb615c 100644 --- a/DevDoc/theory/indexing.refs/index.html +++ b/DevDoc/theory/indexing.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/theory/indexing/index.html b/DevDoc/theory/indexing/index.html index afbd21e6..be5ea79a 100644 --- a/DevDoc/theory/indexing/index.html +++ b/DevDoc/theory/indexing/index.html @@ -1199,7 +1199,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/theory/minimizer.refs/index.html b/DevDoc/theory/minimizer.refs/index.html index efa795a8..26fbf103 100644 --- a/DevDoc/theory/minimizer.refs/index.html +++ b/DevDoc/theory/minimizer.refs/index.html @@ -1124,7 +1124,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDoc/theory/minimizer/index.html b/DevDoc/theory/minimizer/index.html index f7d4cdc0..b789b4f2 100644 --- a/DevDoc/theory/minimizer/index.html +++ b/DevDoc/theory/minimizer/index.html @@ -1243,7 +1243,7 @@ - Sibling annex (discussion) + Sibling annex diff --git a/DevDocMD/architecture/siblings.md b/DevDocMD/architecture/siblings.md index 352956f4..a9e32377 100644 --- a/DevDocMD/architecture/siblings.md +++ b/DevDocMD/architecture/siblings.md @@ -1,1024 +1,160 @@ -# Sibling annex — architecture (discussion) - -Status: architecture decided (2026-08-14). Implementation not yet mandated. - -## Two index spaces, uncorrelated - -Every kmer stored in a `Layer` lives in two independent index spaces: - -- **Iteration order**: its position when enumerating `unitigs.bin` (the - superkmer file), deterministic but arbitrary with respect to slot. -- **MPHF slot**: `MphfLayer::index(kmer)`, the number the MPHF assigns. - -The two are not correlated by any formula. Converting from one to the other -requires either recomputing the MPHF (kmer → slot) or scanning the iteration -stream (kmer → order). There is no `slot → kmer` operation: the MPHF is a -one-way function, not an invertible bijection with a stored inverse. Any -method that reconstructs a kmer from a bare slot number is wrong by -construction, regardless of the mechanism used (MPHF re-hash, or evidence -decode + direct unitig read). See `MphfLayer::kmer_at` -(`obikindex/src/layer/mphf_layer.rs`) — flagged for removal, currently called -from `obikphylo/siblings/build.rs` and `family_scan.rs` (since removed — see -"Pending work" status below). - -## Two pipelines, never mixed - -| | origin of the kmer | membership known? | correct mapping | -|---|---|---|---| -| **query pipeline** | external (caller-supplied) | no | `query`/`find`/`find_strict` — MPHF + evidence check | -| **iteration pipeline** | enumerated from this layer's own `unitigs.bin` | yes, by construction | `index`/`index_batch` — MPHF only, no evidence | - -Evidence exists solely to answer "is this external kmer a member of the -layer" for the query pipeline. Using it (or the MPHF) to go the other way — -recover a kmer from a slot, or re-verify a kmer that was just produced by -iterating the layer — is a conceptual error: evidence can be probabilistic -(`Approx` mode), so any slot→kmer attempt is unsound in general, and -pointless even in `Exact`/`Hybrid` mode since the kmer was already known. - -## Sibling annex: an iteration-pipeline artifact only - -The sibling annex (`FamilyMask`/`SiblingAnnex`, `.psib`, -`obicompactvec/src/siblingannex.rs`) records, per kmer, whether it is a -family minorant and which family members are present in the index. Its only -consumers (`obikphylo/siblings/stats.rs`, `family_scan.rs`) enumerate it -exhaustively (`0..annex.len()`); no query-pipeline code path touches it. - -**Decision**: the annex must be persisted in iteration order, not slot -order. This lets readers zip-iterate `Layer::iter_kmers()` and the annex -file directly — one linear, cache-friendly pass, no MPHF/slot indirection, -no `kmer_at`. It also enables specialized iterators building on this zip: -minorants-only iteration, batch-of-kmers → batch-of-family-members, etc. - -Today the annex is built and stored in **slot** order -(`build_layer_sibling_annex`, `siblings/build.rs`): `slot_kmer` is populated -via `(0..n_slots).map(|slot| mphf.kmer_at(slot))`, and the origin `slot` is -threaded through the whole cross-partition reconciliation pipeline (variant -generation, `query_partition_with`, final `mask[slot].fetch_or(...)`). This -must change to iterating `iter_kmers()`/`enumerate_kmers()` and threading -the **iteration index** instead of the slot end to end — eliminating -`kmer_at` from the build path entirely, not just the read path. No -slot-indexed intermediate is needed even during construction; the -iteration-order id is sufficient throughout. - -The cross-partition side of the same pipeline is unaffected: checking -whether a generated family-variant kmer exists in another partition is a -genuine query-pipeline operation (the variant's membership in the *target* -partition is unknown) and must keep going through -`KmerPartition::query_partition_with` (MPHF + evidence), never a raw -`index()`. - -## Pending work — done - -The plan above shipped: `obikphylo` (a new crate — phylo-domain extension -traits over `obikindex::KmerIndex`/`obikindex::layer::Layer`, replacing the -old `obikindex::siblings` module) builds and reads the annex purely in -iteration order (`SiblingLayerExt::iter_siblings`/`iter_minorants`, both with -batch variants, mirroring `Layer`'s own `KmerIter`/`KmerBatchIter` -shape). `MphfLayer::kmer_at` has no remaining callers. - -A separate, unrelated bug surfaced during this work and was fixed -(2026-08-14): `MphfLayer::enumerate_kmers_batch` computed its -`batch_start_index` via the stdlib `.enumerate()` adapter, which counts -*batches* (0, 1, 2…), not the cumulative k-mer offset the annex is actually -keyed on — every batch past the first wrote its mask/annex entries at the -wrong iteration-order position. Fixed by tracking a running offset instead; -regression tests added (`sibling_annex_no_empty_masks_after_build`, -`sibling_histogram_does_not_panic_on_partial_last_batch`). - -## Performance: `build_sibling_annex` parallelism (2026-08-14) - -Investigated on a real multi-genome run (`phyloskims_sal_vac`, k=31/m=11). -Baseline: mostly one active core, with short multi-core bursts — average -~3 cores. - -**Fixes that helped, kept:** - -- `CanonicalKmerOf::minimizer()` (`obikseq/src/kmer.rs`) — a direct O(k) - bit-arithmetic minimiser for a single isolated k-mer, replacing a - `RollingStat` instance fed byte-by-byte through an ASCII round-trip (used - by `helpers::partition_of`, called for every generated family variant). - ~3x wall-clock improvement on its own, confirmed by sampling - (`obiskbuilder::rolling_stat`/`obikentropy` frames disappeared from the - hot path). `CanonicalKmerOf::partition()` added alongside it (wraps - `minimizer().seq_hash() & mask`, the same routing rule - `KmerPartition`/`RoutableSuperKmer` use). -- Cross-partition resolution (`outgoing.par_iter()` in - `build_layer_sibling_annex`) parallelised at the *partition* level — one - Rayon task per non-empty `outgoing[dest]` bucket. For k=31/m=11, a - central-base substitution changes the winning minimiser (and thus the - destination partition) only when that window overlaps the central base: - ~11 of the 21 possible windows do, so ~10/21 (≈48%) of generated variants - route right back to the partition already being built. That self bucket - ends up far larger than any other, so the per-partition split pinned one - thread to it alone while the rest of the pool finished instantly — - confirmed by sampling: one thread solid in `MphfLayer::find`, everyone - else idle. Fixed by splitting each non-empty bucket into - `total_queries / n_workers` (capped 4096) chunks *before* `par_iter()`, - preserving per-partition mmap locality (each chunk stays contiguous - within one partition) while letting Rayon spread an oversized bucket - across several threads. Net effect of both fixes together: ~3 cores - average → ~10-13 cores average on the same run, and a projected total - build time of ~1h15 down to ~30min on the real `phyloskims_sal_vac` run - this was measured against. -- `TracedBar`'s ETA (`obisys/src/progress.rs`) was silently starved: the - custom progress message and the self-computed ETA text used to share one - `pb.set_message()` slot, with the ETA holding off for 2s after any custom - message — fine when custom messages are rare, broken once - `build_sibling_annex`'s per-partition callback fires more often than - that. Fixed by keeping the two texts in separate fields, composed - together on every render instead of one overwriting the other. - -**Tried and reverted — do not repeat blindly:** - -- Parallelising the *outer* partition loop in `build_sibling_annex` with - `obikindex::PartitionRunner` (already used by `merge`/`build_layers`), - splitting a fixed core budget between outer (partition) and inner - (pipeline + resolution) concurrency so their product wouldn't exceed the - budget. Measured *worse*: throughput dropped over time (26 - partitions/5min → 38/11-12min) and peak resolution concurrency fell from - ~11-12 cores to ~7-8. Cause: this capped the resolution burst — which - scales very well on its own — to make room for outer concurrency, and - running several partitions' resolution at once scatters access across - multiple partitions' mmap regions at once, working against the - locality `outgoing`'s per-partition grouping exists for. `PartitionRunner` - stayed exported from `obikindex` (`new_capped` too) since it's - general-purpose, but nothing in `obikphylo` calls it. -- Splitting resolution chunks even finer (`/(n_workers*8)`, cap 1024, - instead of `/n_workers`, cap 4096) to smooth the residual sawtooth. - Measured ~10% *slower*, wider dips, not narrower. Reverted to the - original chunk sizing. - -**Known remaining limitation, not yet worth fixing:** within one layer, the -four stages (sequential `unitigs.bin` read → parallel generation → -parallel resolution → sequential annex write) never overlap — confirmed by -1s-interval sampling: generation alone occupies ~17 threads evenly, but the -next layer's read/generation never starts until the current layer's -resolution and write are both done. This produces a real, periodic (~layer -duration) alternation between "many cores" and "few cores" that neither of -the fixes above touches, since both operate *within* one layer's resolution -step. The only remaining lever is overlapping consecutive layers (e.g. a -depth-2 pipeline: start layer N+1's read/generation while layer N's -resolution/write is still running) — a real restructuring, not a parameter -tweak, and explicitly *not* to be combined with the reverted -budget-capping idea above (let each phase use however many cores it -naturally wants; only the *scheduling* needs to overlap). Deferred, not -started. - -## Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14) - -`family_scan.rs::scan_layer_families` (shared by `snp_pseudo_alignment`, -`sibling_annex_stats`, `cardinality_tally`, `scan_family_pairs`) already -implements most of a dispatch/accumulate/resolve pipeline: generation -(cheap, CPU-only — builds `outgoing[dest_partition]` from `FamilyMask` and -buckets cross-partition queries) runs on an `obipipeline::throttle` + -`make_pipe!` stage, decoupled from resolution (I/O-bound, `rayon::par_iter` -*across partitions*, one generated batch resolved at a time, never several -concurrently — this ordering is deliberate, see the module's own docs on a -reverted concurrent-batch-resolution attempt that scattered mmap access). -The fast/slow mode gate (`PartitionCache::fast_mode`, `cache.rs:162-163`) -already exists: `n_layers <= 7` (checked once from the first non-empty -partition's `PartitionMeta::n_layers`, documented as identical across every -partition of an index — a structural, build-time property, never a -per-partition state) decides whether `FamilyMask`'s recorded `layer_value` -can be trusted to skip straight to the right layer -(`find_presence_batch_fast`) or must fall back to scanning every layer of -the destination partition (`find_presence_batch`). - -**Real gap, confirmed not implemented**: resolution is triggered by the -*source* batch finishing (`FAMILY_BATCH = 65536` minorants read from the -scanned layer), not by an *output* accumulator filling up. Since most -central-base variants of a family route back to the same partition being -scanned (~48% per the k=31/m=11 measurement above), a `FAMILY_BATCH`'s -`outgoing[dest]` is large for the local/self partition and thin for the -other ~255 (or however many) destination partitions — each of those gets -resolved at low query density every batch instead of being accumulated -across several source batches until resolving it is worthwhile. This is -distinct from, and not fixed by, the fast/slow layer gate above. - -Redesign sketched (not built): per-destination accumulators decoupled from -`FAMILY_BATCH`, flushed on reaching a size threshold instead of on source-batch -completion — a "hot" accumulator for the partition being scanned (sharded -one-per-generation-worker, no lock, since all `n_workers` pipeline workers -write to it concurrently — this differs from an earlier, simpler mental -model of "one thread owns one layer's local collector," which doesn't hold -here since `n_workers` threads cooperate on scanning *one* layer at a time, -not one thread per layer) and "cold" mutex-per-partition accumulators for -the rest, low contention expected since traffic to any single cold -destination is a small fraction of total. - -This breaks the current strict-iteration-order delivery of `on_family` -(today: a reorder buffer keyed by batch, since a whole `FAMILY_BATCH` -resolves atomically). With cross-batch accumulation, a family only becomes -complete once *every* accumulator holding one of its outgoing queries has -flushed, at unpredictable, independent times — no longer streamable -strictly in order without a large, unbounded pending buffer. Resolution -sketched: replace order-dependent consumers with coordinate-addressed -writes instead of order-dependent appends (see `PseudoAlignment` idea -below) wherever possible, since `sibling_annex_stats`'s reduction (plain -counts) is already order-independent and needs nothing here. - -**Superseded 2026-08-15** by the `--subsample`/`--shannon` design below, -which sidesteps the accumulator redesign for now: bounding the number of -families actually resolved per layer (via sampling) keeps per-layer -resolution volume small enough that the batch-density problem above stops -mattering in practice for these two consumers. The accumulator redesign -remains relevant for a future *unsampled, full-index* run, but is not -required to ship `--subsample`/`--shannon`. - -## Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15) - -The reference run (`phyloskims_sal_vac`-scale bacterial test set, -`iqtree.fasta`) produced a dense alignment for 13 genomes × 383,965 sites -(4.8 MB) — trivially small. The in-progress plant index is expected to -carry on the order of 9 billion minorant families; a dense byte-per-cell -alignment at that column count is unbuildable regardless of genome count -(hundreds of GB even at a handful of genomes). Long-term ambition is 6,000– -8,000 genomes on a large machine, which makes the per-cell cost dominant in -the other dimension too. Pruning the retained family set before -materializing anything is mandatory, not an optimization. - -**Already free**: `family_size() < 2` (no sibling variant registered at -all) is a zero-cost structural filter, read directly off `FamilyMask` bits, -already applied in `snp_pseudo_alignment`. Insufficient alone — per the -~80-85% mono-family estimate from earlier discussion, this only brings 9 -billion down to roughly 1.3-1.8 billion, still unusable. - -**Entropy definition — settled 2026-08-15, correcting an earlier wrong -turn.** The project does **not** encode families as IUPAC ambiguity codes -interpreted the classical way (Fitch-parsimony subset-compatibility, or -ML's "one true state, uncertain which"); see -`docmd/theory/evolutionary_distances.md` ("Why the IUPAC/DNA encoding used -for the first `--snp` test was wrong") and the Sankoff resolution that -followed it. The real model is a genuine 16-state alphabet (the powerset of -`{A,C,G,T}`, `∅` included as a real state) scored with a *calibrated -pairwise cost matrix* (`obikphylo::cardcomp::pairwise_cost_matrix`, -`cmd/phylo/sankoff.rs`), not a compatibility/subset relation between -states. Under that model, each of the 16 states — including multi-bit ones -like `AC` — is a first-class, independently-costed state, not an -uncertainty encoding of a single true base. So: **entropy over the 15 -non-empty states (`∅` excluded, matching the earlier decision to exclude -genomes where the family is absent) is the correct informativeness -measure** for this project — not a 4-symbol reduction, which would discard -exactly the cardinality/composition information the calibrated cost matrix -is built to exploit. - -## `--subsample` / `--shannon` — sampling strategy (decided 2026-08-15) - -Goal: make both the pseudo-alignment (`--snp`) and a Shannon-entropy -diagnostic usable at any index scale, from the 13-genome bacterial -reference run up to the 9-billion-family plant index, without requiring the -batched-accumulator redesign above. - -**`--subsample N`** (integer, families to retain): bounds the pseudo-alignment -to `N` minorant families, sampled **proportionally per layer** among -non-monomorphic minorants (`family_size >= 2`) — this sidesteps the need -for a true global reservoir merge across layers while still approximating a -uniform sample over the whole index, and directly answers the earlier open -question of global-vs-per-layer selection scope. - -Three passes, in order: - -1. **Global count** (cheap, structural, parallel across layers — same shape - as the existing `sibling_family_size_histogram`, extended to report a - **per-layer** breakdown rather than one index-wide aggregate): for each - layer, `count_layer` = number of non-monomorphic minorants. Gives - `total_count = Σ count_layer`. -2. **Per-layer proportional reservoir sampling** (cheap, structural, one - pass per layer, no cross-partition resolution): `N_layer = round(N × - count_layer / total_count)`. Since `N_layer` is a proportion of - `count_layer`, it can never exceed it as long as `N <= total_count` — the - one edge case is `total_count <= N`, in which case sampling is skipped - entirely and *every* non-monomorphic minorant of every layer is kept - (no reservoir needed, `N` was never a real constraint). Otherwise: - Algorithm-R reservoir sampling over the layer's non-monomorphic minorant - indices, producing `N_layer` iteration-order indices directly, no - intermediate full list ever materialized. -3. **Filtered resolution** (the expensive step, the existing - `scan_layer_families` engine, unchanged): re-scan the layer, generating - and resolving cross-partition queries **only** for the indices selected - in step 2 (cheap membership test against a small per-layer index set) — - this is what keeps `--subsample` cheap even on an unsampled-scale index, - since the cross-partition resolution volume is bounded by `N`, not by - the layer's true size. - -Steps 2 and 3 cannot be merged into one pass: true single-pass reservoir -sampling would waste step-3's expensive resolution work on candidates later -evicted by the reservoir. Step 1 must fully complete (every layer) before -step 2 can start for any layer, since `total_count` is a global quantity. - -**`--shannon`** (no argument): emits a CSV of per-family Shannon entropy -(15 non-empty states, `∅`/absent genomes excluded from the denominator, per -the settled definition above). Independent of `--subsample` — entropy is -computed and written per family as soon as its `genome_mask` resolves, -O(1) memory per family, so it streams fine even unsampled at full index -scale (a time cost, not a memory one). Combined with `--subsample N`, it -delivers the original exploratory diagnostic (e.g. `--subsample 1000000 ---shannon`) directly from this general machinery, rather than a -purpose-built one-off script. - -Validated end-to-end (2026-08-15) against real data: `--sibling-hist` on -`phyloskims_sal_vac` (91 real genomes, k=31/m=11, 256 partitions × 2 -layers) confirms the ~9-billion-family estimate almost exactly (8,925,068,238 -total, 97.9% monomorphic — a sharper mono fraction than the ~80-85% earlier -guess, corrected here). `--subsample`/`--shannon` on the smaller 20-genome -bacterial reference (`benchmark/global_index_presence`) produced a sample -size within rounding of the request (99,742/100,000) and a -[0.8,1.2)-bucket share (40.7%) matching the full unsampled population -(41.6%) — the two histograms only diverged wildly (5‰ vs 41.6%) under a -real bug in `reservoir_sample_layer` (see next section), now fixed. - -**Bug found and fixed (2026-08-15): `family_idx` numbering mismatch.** -`scan_layer_families`'s `family_idx` counts *every minorant* of a layer -(monomorphic ones included, since `iter_minorants_batch` filters only on -`is_minorant()`), not the raw annex slot (`SiblingAnnex::get(slot)` spans -every k-mer, minorant or not) and not a counter over non-monomorphic -minorants alone. `subsample.rs`'s `reservoir_sample_layer` originally -stored raw slot numbers in its `HashSet` selection, which drifts -away from `family_idx` as soon as *any* monomorphic minorant is seen — -i.e. almost immediately, since ~98% of minorants are monomorphic. Fixed by -tracking two separate counters: `family_idx` (every minorant, matching -`scan_layer_families`) and `seen` (non-monomorphic minorants only, what -Algorithm R actually samples over) — only `family_idx` values are ever -stored in the selection set. The existing unit test never exercised this -(its fixture has exactly one non-monomorphic family, always hitting the -"keep everything" shortcut) — a stronger fixture with several interleaved -monomorphic/non-monomorphic families would be needed to catch a regression -here automatically; not yet written. - -## Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15) - -Motivation: on real data (bacterial reference, full unsampled run), only -~5‰ of non-monomorphic minorants fall in the `[0.5, 1.5]` bit band judged -phylogenetically interesting (entropy too low = uninformative near-invariant -site; too high = saturated/noisy, see `family_entropy`'s 15-state -discussion) — roughly 1 in 10,000 minorants overall. Computing exact -entropy for every candidate just to discard 99.99% of them is wasteful at -the full 9-billion scale. - -**The idea**: `PersistentBitMatrix::col_view(c)` gives a genome's whole -presence column as a `BitSliceView` (sequential, no MPHF, no cross-partition -routing — purely local to one layer's own matrix). Accumulating -`TempCompactIntVecBuilder::inc_present(col)` (already exists in -`obicompactvec/src/builder.rs:121`, along with `add`/`min`/`max`/`diff` on -`IntSliceView` — no new low-level API needed) over every column of a layer -produces `coverage[slot]`: how many genomes carry each exact k-mer, in one -sequential per-layer pass, entirely decoupled from family/sibling -structure. Persisted once per layer, a family's members' coverage could -then be looked up via a plain local MPHF `index()` (cheap) instead of a -full cross-partition presence resolution (`find_presence_batch`) — the -expensive part today is specifically the cross-partition/cross-layer -routing to a sibling's own matrix, not the bit-reading itself, and -`coverage[slot]` sidesteps that routing entirely by moving the cost into a -one-time, purely local, embarrassingly-parallel build step. - -**Why not implemented**: `coverage[slot]` is a per-member marginal — -summing members' coverages to approximate a family's entropy silently -assumes no genome carries more than one member at once. It cannot -represent or detect joint co-occurrence (a genome carrying both `A` and -`C` at once, i.e. a combined 15-symbol state) at all, which is exactly the -phenomenon `family_entropy`'s 15-state definition exists to capture (see -`CardinalityTally`/`cardinality_transition_probs`, the project's own -existing machinery for this same co-occurrence structure, built for the -Sankoff matrix calibration). A family that is in truth uniformly `AC` -across every carrying genome would look like a well-balanced 2-state split -under the marginal approximation (entropy ≈ 1) while its true 15-state -entropy is 0 — i.e. the marginal proxy's failure mode lands on exactly the -"saturated, uninformative" tail this pre-filter would need to catch, -undermining the point. It stays plausible as a coarse filter for the -*low* tail only (a dominant single member's marginal share reliably -predicts low true entropy too), but not as a stand-in for the high tail — -not pursued further for now. - -## `--free-loss`/`--tnt` pipeline: four independent scans, three of them unsampled (found 2026-08-15, fixed 2026-08-15 — see "Implemented" below) - -Measured on `phyloskims_sal_vac` (91 genomes): `obikmer phylo --subsample 500000 ---free-loss --tnt` logs four sequential stages — -`raw_snp_distance` (1413s), `base_pair_tally` (1456s), -`cardinality_tally` (2078s), `snp_pseudo_alignment` (143s). Reading the -code (`obikmer/src/cmd/phylo/mod.rs:210-242`, -`obikphylo/src/siblings/distance.rs`, `cardinality.rs`, `alignment.rs`) -surfaced two compounding problems, not one: - -1. **Four separate full scans of the annex**, each opening its own - `KmerPartition`/`PartitionCache` and calling `scan_family_pairs`/ - `scan_layer_families` independently — nothing computed in one stage is - reused by another. `base_pair_tally` is explicitly documented - (`distance.rs:138-143`) as a second full pass over the same - traversal `raw_snp_distance` already did, needed only because - `raw_snp_distance` doesn't keep the resolved bases, only aggregate - counts. `cardinality_tally` and `snp_pseudo_alignment` are each a - third and fourth independent full pass. Per the module's own earlier - profiling note (`family_scan.rs:26-28`, cited already above), this - traversal is page-fault/mmap-bound, not compute-bound — the ~10-12% - CPU efficiency ("contention" status) observed on these three slow - stages is consistent with I/O stalls scaled by repeated full scans, - not lock contention (there are no `Mutex`/`RwLock` anywhere in - `siblings/*.rs`; shared writes use per-slot `AtomicU8::fetch_or`). -2. **Worse: `raw_snp_distance` and `cardinality_tally` don't honor - `--subsample` at all** — both call `scan_layer_families` with - `Selection::All` hardcoded, and `args.subsample` isn't even threaded - into their function signatures (`mod.rs:212`, `mod.rs:226`). Only - `snp_pseudo_alignment(args.subsample)` builds a real reservoir-sampled - `Selection::Some(set)` (via `compute_selections`, `alignment.rs:89`). - So today, `--subsample 500000` only bounds the pseudo-alignment step — - the SNP-distance matrix, the Sankoff base-pair calibration, and the - cardinality histogram are always computed over the **full, unsampled** - index regardless of the flag. This is not merely "different subsamples - per stage" (which would already be a problem worth fixing) — it's that - three of the four stages never subsample, which explains most of the - ~10x runtime gap against `snp_pseudo_alignment` on its own. - -**Decided requirement**: all four stages consume **one shared selection**, -computed once, not each stage either scanning everything or drawing its -own independent sample. Per-genome-pair SNP counts, the Sankoff base-pair -calibration, the cardinality histogram, and the pseudo-alignment all -describe the same set of families — the calibration and the alignment it -calibrates are now guaranteed to agree on which sites exist. - -**Correction to the "single pass" framing above**: `base_pair_tally`/ -`cardinality_tally` both need `raw_snp_distance`'s *complete* aggregate -SNP/shared counts before they can derive `included[i,j]` (the -`ratio_ceiling` filter) — a genuine sequential dependency (`included` -can't be known until every pair's aggregate count is final), not an -artifact of the old code's structure. So the fix is **two** passes over -the shared selection, not one: pass A computes the aggregate counts (and -derives `included`); pass B fuses `base_pair_tally` + `cardinality_tally` -+ the pseudo-alignment (mutually independent once `included` is known) -into a single scan. Still a 4→2 reduction, and — per the clarification -that settled this — pass A itself now runs over the *same shared -selection* pass B uses (not the full unsampled index): "les ratios, on -les fait sur les sites sélectionnés, c'est tout, les autres sites -n'existent pas" — once a selection is chosen, both passes are bounded by -it, so on a real `--subsample`/`--entropy` run pass A is cheap too, not -just pass B. - -**Entropy-biased selection's own resolution to the "forward-looking -complication"** (entropy must be known before selection, but selection -happens during the same scan that would resolve it): see "Entropy-biased -selection" below — resolved via a persisted per-layer entropy annex, not -by restructuring the scan into an inline pre-pass. - -## Entropy-biased selection: soft Gaussian weighting, not a hard cutoff (decided and implemented 2026-08-15) - -Refines the "forward-looking complication" above with a concrete -mechanism. Instead of a hard `[low, high]` entropy band (or any other -exact cutoff) deciding which non-monomorphic minorant families are -eligible, selection is weighted by an **unnormalized Gaussian kernel** -centered on a target entropy: `w(entropy) = exp(-(entropy - μ)² / (2σ²))` -— deliberately not the normalized Gaussian density (which would peak -below 1 and complicate the "probability" reading) — this kernel form -equals 1 exactly at `entropy = μ` and decays smoothly to 0 away from it, -so it reads directly as an acceptance weight: the further a family's -entropy from the target, the less likely it is picked, with no hard -in/out boundary — a few "bad" sites can still get in, by design. `μ` -(default ~1.0) and `σ` (default ~0.5) are meant to be user-tunable. - -**Mechanism: joint probability, not a weighted reservoir**. Not -Efraimidis–Spirakis weighted reservoir sampling (an earlier, more complex -proposal, superseded before implementation) — instead, a single -independent accept/reject draw per qualifying candidate: draw -`u ~ Uniform(0,1)`, accept iff `u < p₀ · w(entropy)`. `p₀` is a single -**index-wide** rate, `N / total_count` (`total_count` = the sum of -`non_monomorphic_counts` across every layer, `N` = `--subsample`'s -target), applied identically at every layer — this alone already gives -each layer its proportional share (the same effect the old uniform -reservoir's explicit per-layer `n_layer = N · count_layer / total_count` -computation achieved, but without needing to compute it: applying one -rate uniformly is mathematically the same as proportioning per layer). -`p₀ = 1.0` when there is no `--subsample` at all — `--entropy` alone is a -pure soft entropy filter over the whole index, no size target. Properties: -(1) **strictly generalizes the existing uniform sampler** — with `σ` large -enough that `w ≈ 1` everywhere, this reduces to the old uniform `N/total_count` -draw; (2) yields "approximately N", not exactly N — expected accepted -count is `N · mean(w)`, always `≤ N` — an intentional relaxation, matching -"sous-échantillonnage à environ n" rather than the old reservoir's exact-N -guarantee; (3) one streaming pass, one random draw per candidate, no -reservoir state. - -**Resolving "entropy must be known before selection, but selection -happens during the same resolving scan"**: solved with a **persisted, -per-layer entropy annex** (`obikphylo/src/siblings/entropy_annex.rs`, -`EntropyAnnex`/`EntropyAnnexBuilder`), not by restructuring the scan. -Mirrors `SiblingAnnex`'s mmap-backed, read-only-after-build convention, -but indexed by `family_idx` (every minorant of the layer, monomorphic -included — the same numbering `Selection`/`scan_layer_families` already -use), one `f32` entropy15 value per entry, `-1.0` sentinel for monomorphic/ -not-yet-computed. First use of `--entropy`/`--entropy-sd` on an index -pays a one-time cost (`ensure_entropy_annexes` in `entropy.rs`) — every -later run (any `μ`/`σ`, any command) reads the file positionally, no -re-scan, restoring the usual `Selection::Some` "skip resolving excluded -families" speedup that a naive "weigh during the resolving scan" design -would have permanently forfeited. - -**Bug found and fixed (2026-08-15): `ensure_entropy_annexes` scanned with -`Selection::All` instead of bounding to non-monomorphic minorants.** -Monomorphism (`family_size() < 2`) is knowable directly from the annex -bits alone, no per-genome resolution needed — but the original -implementation called `scan_layer_families` with `Selection::All` -anyway, so `fill_sub_matrix_carries` (the expensive per-genome -resolution) ran for *every* minorant, ~98% of which are monomorphic -(measured elsewhere in this doc) and had their `genome_mask` immediately -discarded once the callback checked `family_size() < 2`. Fixed by adding -[`subsample::non_monomorphic_selection_layer`] — a cheap, annex-only, -non-sampling pass (same shape as `reservoir_sample_layer`, but keeping -every non-monomorphic minorant's `family_idx` instead of a bounded -reservoir) — and passing `Selection::Some(&eligible)` instead of -`Selection::All`, so the expensive resolution now runs only for the ~2% -of minorants that can actually produce a real entropy value. A -`debug_assert!(mask.family_size() >= 2, ...)` inside the -`scan_layer_families` callback guards the invariant. - -**Resolved**: the existing hard "non-monomorphic minorant" eligibility -filter stays a hard gate upstream of the Gaussian weighting — only -qualifying families ever get a stored entropy value or a weighted draw. - -**CLI, implemented**: two `phylo` options, `--entropy <μ>` and -`--entropy-sd <σ>` (`obikmer/src/cmd/phylo/args.rs`). The entropic filter -activates as soon as *either* is given (`mod.rs`, computed once into an -`Option` threaded through `--snp`/`--family-overlap`/ -`--shannon`/the fused sankoff pipeline below). If active but one or both -are unset, defaults are `μ = 1.0`, `σ = 0.5`. - -## Two entropy definitions kept side by side, for comparison (2026-08-15) - -`--shannon`'s CSV carries both `entropy15` (`family_entropy` — the settled -15-non-empty-state definition, see above) and `entropy4` (`family_entropy_4` -— plain nucleotide reduction), computed from the same already-resolved -`genome_mask`, not from the marginal approximation above. A genome carrying -several bases at once contributes to *each* base's count (counted once per -base present, not fractionally split, not folded into one combined state) -— a genome polymorphic for the family is present at more than one base by -construction, so it is expected to count more than once; the denominator is -the total base-occurrence count, not the genome count (the two coincide -only when no genome carries more than one base). Kept side by side -specifically to measure, on real data, how much the two diverge — not yet -analyzed. - -## `PersistentSparseBitMatrix` — implemented and measured (2026-08-15) - -A row-major (k-mer-major), deduplicated sparse alternative to -`obicompactvec::PersistentBitMatrix`, motivated by the same sparsity that -drove `--subsample`/`--shannon` above, but pursued as a foundational -storage-layer change rather than an index-level workaround. Full design -history, rationale, and rejected alternatives (external Elias-Fano crates, -`cacheline-ef`, a single unsplit `dict_id` array) are in the dedicated -implementation plan (`vivid-mapping-tiger.md` at the time of writing — the -content below is the durable summary, not a pointer to a session-scoped -file). Also directly informed by Alanko, Bille, Gørtz, Navarro, Puglisi, -"Compact Data Structures for Collections of Sets" (2025, -`biblio/Alanko et al. - Compact Data Structures for Collections of -Sets.pdf`) — this design implements only their exact-duplicate special -case (a plain dedup dictionary), not their full subset-containment -hierarchy. - -**Design**: four on-disk components, each mmap-backed, built once per -layer (matching how the rest of the build pipeline already works — never -the whole multi-billion-row index at once): an `is_multi` rank-capable -flag per row (singleton vs. multi-genome), a fixed-bit-width array for -singleton rows (genome index directly, `ceil(log2(n_cols))` bits), a -separate fixed-bit-width array for multi-genome rows (`dict_id`, -`ceil(log2(n_distinct_multi_sets))` bits — kept apart from the singleton -array specifically because `n_distinct_multi_sets` can be large in -absolute terms even when multi-genome rows are a small *fraction* of all -rows, and a single shared array would force every row, singletons -included, to pay the wider width), and a deduplicated dictionary of -distinct multi-genome sets (Elias-Fano-encoded byte offsets + a -varint-encoded values blob). New low-level primitives added to -`obicompactvec` to build this: `PersistentFixedIntVec` (arbitrary, -runtime-parameterized bit width, width 0 included — needed once a real -bug surfaced, see below), `PersistentRankSelectBitVec` (rank1/rank0/select1 -on top of the crate's existing `count_ones`, using -`common_traits::SelectInWord`), `EliasFano` (composes the two). A new -`BinaryMatrix` trait (`n`, `n_cols`, `row`/`fill_row`, `fill_sub_matrix`, -`count_ones`) unifies dense and sparse at the one call site that needs -both interchangeably (`obikphylo::siblings::cache::Mat`) — column-oriented -methods (`col`, `col_view`, the `partial_*_dist_matrix` family) stay -dense-only. - -**Two real bugs caught by tests, not by inspection**: (1) `EliasFano::open` -re-derived its low-bits width from the persisted low-vector file's own -width byte; the zero-width case was built with a dummy 1-bit placeholder -(the builder rejected true width 0), so every reopened value silently -doubled. Fixed by making `PersistentFixedIntVec` genuinely support width 0 -(no storage, `get` always 0) instead of working around the limitation in -`EliasFano`. (2) An empty row (cardinality 0 — not expected on a real -built index, but not guarded against either) was recorded as a singleton -at genome 0, indistinguishable on read-back from a *real* singleton at -genome 0. Fixed by routing cardinality-0 rows through the dictionary path -(a genuine empty entry) instead of the singleton shortcut. Both caught by -`obicompactvec`'s test suite (142 tests, including disk-reopen round-trips -that drop every builder/mmap before reopening fresh), not by manual -review — worth remembering next time a "this edge case can't happen in -practice" shortcut is tempting. - -**Measured on real data** (`layer_1` of `phyloskims_sal_vac`'s -`part_00018`, 30,246,774 rows, 91 genomes — `#[ignore]`d benchmarks in -`obikphylo/src/siblings/tests.rs`): - -| | dense | sparse | ratio | -|---|---|---|---| -| on-disk size | 328.1MB | 43.7MB | **7.5x** smaller | -| build time / peak RSS | — | 4.26s / 628MB | (per-layer, in-memory construction — comfortable) | -| row access, sequential (2M reads) | 43ns/row | 32ns/row | sparse **faster** (smaller structure, better cache fit) | -| row access, random (2M reads) | 409ns/row | 85ns/row | sparse **~4.8x faster** (the real `--shannon`/family-lookup shape) | -| column access, one full column (30.2M rows) | 11.5ms | 993ms | sparse **86x slower** (no native column method — every read decodes a full row to keep one bit) | - -The row-access wins (both directions) weren't the design's stated goal — -compactness was — but turn out real: dense's genome-major layout scatters -a single row read across a much bigger file, which costs more than -sparse's rank/select/varint decode once the file is this much smaller. -The column-access cost is the flip side of the same layout choice, and is -exactly what the next item below exists to fix. - -**Next, not yet planned**: rewrite `partial_jaccard_dist_matrix`/ -`partial_hamming_dist_matrix`/etc. (`obicompactvec/src/bitmatrix/pairwise.rs`) -as a row-major co-occurrence accumulation (`O(Σ_rows k²)`, per-row -increments into an `NxN` genome-pair counter — the known alternative to -today's column-fold, plausibly cheaper on data this sparse, not just a -fallback) so `obikindex`'s `--metric`/distance-matrix path can use the -sparse type without the measured 86x column-access penalty. Needs its own -design pass (in particular how it plugs into the `BitPartials`/ -`ColumnWeights` traits so both matrix types keep serving `--metric`) -before implementation — not just "port the loop", a genuinely different -algorithm. - -Also still deferred, unchanged from the implementation plan: full Alanko -et al. subset-hierarchy compression (only the exact-duplicate special case -is built), a sparse `PersistentCompactIntMatrix` (count matrices), and -BRWT-style column-correlation exploitation. - -## Wired into `pack` and the sibling-annex build path (2026-08-15) - -`PersistentSparseBitMatrix` went from a validated but unused type to a -real, selectable on-disk format: - -- **Generic `Layer`**: `obikindex::layer::Layer`'s presence-only methods - (`n_cols`, `sub_matrix`, `fill_sub_matrix`) are generic over any - `D: LayerData> + BinaryMatrix`, not hardcoded to - `PersistentBitMatrix` — `PersistentSparseBitMatrix` implements - `LayerData` (`open`/`read`) the same way. `find_slot`/`index_batch` were - already generic over any `D: LayerData`, so they needed no change. - Verified by `obikindex::layer`'s - `presence_layer_generic_over_sparse_matches_dense` test: build a dense - presence layer, convert it to sparse via `build_from_dense`, open both - as `Layer`/`Layer` on - the same directory, assert `n_cols`/`sub_matrix`/`find_slot` agree. - (This test must stay at `k=4` with mutually non-colliding canonical - 4-mers across its input sequences — `K`/`M` are process-wide - `AtomicUsize`s in test builds, not thread-local, so a test using a - different `k` races every other test in the same crate binary; a k=11 - version of this test passed alone but failed under the full - `obikindex::layer` suite for exactly that reason before being fixed.) -- **`obikphylo::siblings::cache::Mat`** gained a third variant, - `SparsePresence(Layer)`, alongside `Count` - and `Presence` — every method (`find_slot`, `index_batch`, - `iter_minorants_batch`, `n_cols`, `fill_sub_matrix_carries`) dispatches - to it identically to `Presence`, since both go through the same generic - `Layer` code. `PartitionCache::build` picks the variant per layer by - checking for `presence/is_multi.prsb` (the sparse format's own marker - file, see the design section above) before falling back to the dense - open path. -- **`pack_sparse_bit_matrix`** (new, `obicompactvec::bitmatrix::sparse`): - `pack --sparse`'s entry point. Idempotent (checks `is_multi.prsb` - first); packs to dense `matrix.pbmx` first if that hasn't happened yet - (the dense→sparse transpose needs random row access, which only the - packed/columnar dense forms give), then `build_from_dense`s the sparse - form into the same directory and deletes `matrix.pbmx` — old-format - files are removed only after the new format is fully written, mirroring - `pack_bit_matrix`'s own crash-safety convention. -- **CLI**: `obikmer pack --sparse` threads a `sparse: bool` through - `KmerIndex::pack_matrices` (all other call sites — `select`, `merge`, - `finalize_indexed` — pass `false`, unchanged dense behaviour). Count - matrices are untouched by `--sparse` (no sparse `PersistentCompactIntMatrix` - — see "still deferred" above). -- **End-to-end coverage**: `obikphylo::siblings::tests:: - sibling_annex_works_after_pack_sparse` builds a two-genome index, packs - it `--sparse`, asserts `is_multi.prsb` exists, then runs - `build_sibling_annex` and checks the resulting `FamilyMask`s match the - dense-path test (`sibling_annex_one_sibling_each`) exactly — proves the - sparse format round-trips through the real build pipeline - (`PartitionCache` sparse-detection included), not just the - `obicompactvec`/`obikindex::layer` unit layers below it. - -Full workspace `cargo test` (all crates, unit + doc tests) green after -this change. - -## Sankoff pipeline fusion + entropy-biased selection — implemented (2026-08-15) - -Replaces the "four independent scans" problem above and implements -"Entropy-biased selection" above, end to end: - -- **`SankoffBundleExt::sankoff_bundle`** (new, - `obikphylo/src/siblings/sankoff_bundle.rs`) — the `--sankoff`/`--tnt`/ - `--phyg`/`--iqtree` block in `obikmer/src/cmd/phylo/mod.rs` now calls - this once instead of three separate `raw_snp_distance`/ - `base_pair_tally`/`cardinality_tally` calls. One `PartitionCache`, one - shared (possibly subsampled/entropy-biased) selection computed once via - `compute_selections`, two scans over it: pass A (aggregate SNP/shared - counts, `--exclude-genome` zeroing, then `included[i,j]`), pass B - (`base_pair_tally` + `cardinality_tally` + the pseudo-alignment, fused - into one `scan_layer_families` call per layer, all three read off the - same resolved `genome_mask`). `snp_pseudo_alignment`/ - `shannon_entropy_csv` (still used standalone by `--snp`/ - `--family-overlap`/`--shannon`) both gained an `entropy_bias` parameter - too, so entropy-biased selection isn't sankoff-specific. -- **Regression proof**: `sankoff_bundle_matches_old_separate_calls` - (`obikphylo/src/siblings/tests.rs`) asserts `sankoff_bundle`'s four - outputs are bit-identical to calling the old, separate - `raw_snp_distance`/`base_pair_tally`/`cardinality_tally`/ - `snp_pseudo_alignment` on the same fixture with no subsample — the - fusion is a performance change, not a behavior change. -- **`EntropyAnnex`/`EntropyAnnexBuilder`** (new, - `obikphylo/src/siblings/entropy_annex.rs`) and **`ensure_entropy_annexes`** - (`entropy.rs`) implement the persisted-entropy mechanism from - "Entropy-biased selection" above. `entropy_annex_builds_on_demand_and_biases_selection` - (`tests.rs`) proves, on a fixture with one known-entropy family: the - annex file doesn't exist before any entropy-biased call; `compute_selections` - builds it on first use; `μ` set to the family's exact entropy with - `p₀ = 1.0` selects it deterministically (`u < 1.0` always, for - `u ∈ [0,1)`); `μ` set far away with tiny `σ` deterministically excludes - it (`w` underflows to exactly `0.0`); the persisted value matches - `--shannon`'s own `family_entropy` computation to `1e-6`. -- **`EntropyBias`** (`pub`, `obikphylo::siblings::EntropyBias { mu, sigma }`) - is the one new public type threading `--entropy`/`--entropy-sd` through - every `Option`-accepting method — resolved once in - `obikmer/src/cmd/phylo/mod.rs` from `args.entropy`/`args.entropy_sd` - (activation: either given; defaults `1.0`/`0.5` for whichever is unset). - -Full workspace `cargo test` green after this change (167 unit tests in -`obicompactvec`+`obikindex::layer`+`obikphylo` alone, plus every other -crate's suite, no regressions). - -**Still open, not part of this change** (per "Correction to the 'single -pass' framing" above): `--raw-snp-distance`/`--raw-snp-counts` (the -standalone diagnostic flags, not the `--sankoff` pipeline) still always -scan the full unsampled index — never threaded `--subsample`/`--entropy`, -out of scope here since the reported problem was specifically about the -`--sankoff`/`--tnt` pipeline's redundant/inconsistent scans, not these -two standalone flags. - -## `query` never benefits from sparse row-major access (found 2026-08-19, not implemented) - -Benchmarked `obikmer query` against `global_index_presence` (dense-packed) -vs. `global_index_presence_sparse` (`pack --sparse`), 100k simulated reads -× 2 specimens (`benchmark/`, see -[benchmark_query_testing.md](../implementation/benchmark_query_testing.md)). -Correctness: 0 mismatches — sparse and dense return bit-identical query -results. Performance: sparse consistently *slower* than dense (~30-50%, -reproducible across two runs with warm disk cache), the opposite of -`pack --sparse`'s stated intent ("faster for single-row access... like -query"). - -**Root cause, read from source, not measured in isolation:** -`KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs:155-220`) -is architecturally column-major: stage 2 walks `for g in 0..n_cols { for -slot in hit_slots { layer.col_value(g, slot) } }`, documented (correctly) -as the right locality strategy for the packed/columnar formats, where -`col_value` → `PersistentBitMatrix::get` is a genuine O(1) mmap'd column -read (`persistent.rs:110-113`). - -For `Self::Sparse`, that same `get(c, slot)` (`persistent.rs:114-118`) -allocates a full `n_cols`-wide buffer and calls `fill_row` — materializing -the *entire row* — just to return one cell. Called from inside the -column-major double loop, this reconstructs the same row once per genome -column touched: O(hits × n_cols) full-row rebuilds instead of O(hits). -`PersistentSparseBitMatrix`'s own native row-major decode -(`for_each_genome_in_row`, `sparse.rs:164-177`, used correctly by its own -`row`/`fill_row`/`fill_sub_matrix`) is never reached from the query path -at all. - -**`fill_sub_matrix` (the existing `BinaryMatrix` trait primitive, -`traits.rs:13-37`) is not the right replacement for `query` either**, even -once its own dispatch bug is fixed (see next section) — its output shape -is inherently column-dense: `out[col]` gets an entry for every column, -including columns with zero hits among the requested slots. On real -sparse data (a hit typically touching a handful of genomes out of dozens) -that's still O(n_cols) output regardless of true sparsity. What `query` -actually wants is the sparse triple stream `(slot, col, value)` it already -consumes as `QueryHit::Value` — not a materialized sub-matrix. - -**Proposed primitive** (design only, not implemented — explicit ask: keep -count matrices *not excluded*, even though effort right now is -presence/absence only): - -Not a closure-driven `for_each` — a real `Iterator`, one concrete struct -per matrix format, so the traversal state (current position in the sorted -slot list, current column, permutation, sparse-row decode cursor…) lives -in named struct fields instead of being threaded implicitly through -recursion or a captured closure. RPITIT (stable since 1.75, and this -workspace is edition 2024) means the trait method can return it without -naming or boxing the concrete type: - -```rust -/// Yields every nonzero cell among `slots`, in implementation-defined order. -fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator + 'a; -// item: (idx into `slots`, col, value) -``` - -This is the one foundational primitive per format — both `fill_sub_matrix` -and a `for_each`-style callback become trivial, free consumers of it -(`.for_each(f)` is already `Iterator::for_each` from std; `fill_sub_matrix` -becomes "drain the iterator, scatter into `out[][]`"), instead of two -independently-maintained traversals that can silently diverge (see the bug -below — this is exactly how it happened). - -- **On `PersistentSparseBitMatrix`**: the struct is nearly free to write — - it wraps the existing (currently private) `for_each_genome_in_row` - per-row decode, advancing to the next `slots` entry on exhaustion. O(Σ - row nnz), zero `n_cols`-wide allocation. -- **On `PersistentBitMatrix::{Packed,Columnar}`**: revised — cheaper than - first thought, by reusing the same split already used for - `fill_matrix`'s own implementation instead of hand-writing a resumable - state machine at the matrix level. The base-vector layer - (`BitSliceView`, `views.rs`) already separates the two concerns: - `fill_batch_sorted` (`views.rs:55-60`, sorted-slot batch lookup) and a - genuine per-bit `Iterator` (`BitSliceIter`, `views.rs:94+`) sit side by - side there, one level below the matrix. Adding a - "positions among `sorted_slots` where the bit is set" iterator at that - same vector level is a `filter` over the existing `get()` — no new - state machine, since `std::iter::Filter` already *is* one, generated by - the compiler: - ```rust - fn nonzero_among_sorted<'s>(&'s self, sorted_slots: &'s [usize]) -> impl Iterator + 's { - sorted_slots.iter().copied().filter(move |&slot| self.get(slot)) - } - ``` - The matrix-level `nonzero_iter` then composes these per column with - `flat_map` over `0..n_cols` (each column's hits, tagged with `c`, - slot mapped back through the sort permutation `fill_batch`/ - `fill_batch_sorted` already carry) — again a combinator chain, not a - hand-rolled struct. Same algorithm, same mmap/sort locality as today's - `fill_sub_matrix`; just assembled from `std` iterator adaptors instead - of a loop body writing into a buffer, mirroring the vector/matrix split - the codebase already uses for `fill_batch_sorted` rather than - introducing a new shape. -- **`Implicit`**: trivial (`slots.iter().map(|&i| (i, 0, 1))`, one column, - always present). -- **On `PersistentCompactIntMatrix` (counts)**: same treatment as - `Packed`/`Columnar` — no sparse count format exists yet ("Explicitly - deferred" per `traits.rs:9-12`), so no native low-effort case the way - `Sparse` has one, but not excluded either: the iterator's `Item` is - already `(usize, usize, u32)`, not `bool`, specifically so presence - (`0`/`1`) and counts (arbitrary `u32`) share one primitive instead of a - bool/u32 split forcing counts out of the design. Ready for a native - sparse-count struct later without a signature change. - -Would let `query_partition_with`'s stage 2 collapse to one -`for (i, g, v) in layer.matrix().nonzero_iter(&hit_slots) { on_event(...) }` -per layer, format-agnostic, each backend's struct deciding the actual -traversal. - -**This also closes the existing dispatch bug for free, by construction**: -`PersistentBitMatrix::fill_sub_matrix` (`persistent.rs:190-215`, the enum -wrapper backing `BinaryMatrix`'s trait impl) today does *not* delegate to -`PersistentSparseBitMatrix::fill_sub_matrix` for `Self::Sparse` — it -reimplements the same naive per-(column, slot) `fill_row_bool` loop -instead, bypassing the efficient native method one file over -(`sparse.rs:249-258`). `obikphylo::siblings::cache::Mat` -(`cache.rs:138-145`) independently built its own parallel enum wrapper -that dispatches correctly — evidence this was worked around rather than -fixed at the source: two hand-written traversals for the same format, -free to drift apart, and they did. If `fill_sub_matrix` itself is -rewritten as "drain `nonzero_iter`, scatter into `out[][]`", there is only -one traversal per format left to get right — the bug class doesn't just -get fixed once, it stops being possible to reintroduce. - -## Implemented (2026-08-20) - -Built as designed above, with one deviation from the original sketch: -`nonzero_iter` ended up `Box>`, not a bare `impl -Iterator`, because `Columnar`/`Packed`/`Sparse`/`Implicit` are genuinely -different concrete types and this method isn't on a trait (kept off -`BinaryMatrix` deliberately — that trait is used as `dyn BinaryMatrix` in -`tests/sparse.rs`, and RPITIT methods aren't dyn-compatible). One `Box` -per `nonzero_iter` call, not per cell — negligible next to what it -replaces. - -- `BitSliceView::nonzero_among_sorted` / `IntSliceView::nonzero_among_sorted` - (`obicompactvec/src/views.rs`): the vector-level `filter`/`filter_map` - primitive, exactly as sketched — no new state machine, `std`'s own. -- `PersistentSparseBitMatrix::nonzero_iter` (`bitmatrix/sparse.rs`): native, - `std::iter::from_fn` over one buffered row at a time via the existing - `for_each_genome_in_row` — no `n_cols`-wide allocation, ever. -- `PersistentBitMatrix::nonzero_iter` (`bitmatrix/persistent.rs`): dispatches - to the above for `Sparse`; for `Columnar`/`Packed`, loops columns, - collects each column's `nonzero_among_sorted` hits via `.extend()` (not - `flat_map` — a `flat_map` closure can't lazily return something - borrowing its own captured sort permutation across separate calls - without either boxing per-column or fighting the borrow checker; eager - collection into one `Vec` sidesteps it, at zero cost since - `fill_sub_matrix` already fully materialized anyway). `Implicit` trivial. -- `PersistentBitMatrix::fill_sub_matrix` and `sub_matrix` rewritten to - drain `nonzero_iter` — the dispatch bug is gone because there is now - only one traversal per format, not because the old one was patched. - `PersistentCompactIntMatrix::nonzero_iter` added the same way (counts - not excluded, per the earlier ask) — no native low-effort case, since no - sparse count format exists, but on the same primitive, ready for one. -- `KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs`): - stage 2's column-major `for g { for slot { col_value } }` replaced by one - `layer.nonzero_iter(&slot_list)` call per layer, format-agnostic. -- Tests: `nonzero_iter_matches_dense`, `nonzero_iter_matches_row`, and — - the one that actually targets the dispatch bug rather than each type's - own correctness — `enum_wrapper_dispatches_to_native_sparse` (builds - `PersistentBitMatrix::Sparse(...)` directly, not through `open`, since - `open` only auto-detects `Sparse` from a `presence/` dir layout). - `cargo test --workspace`: green, no regressions. - -**Measured**: re-ran the `benchmark/` query branch (100k reads × 2 -specimens, same setup as the original finding). Correctness still 0 -mismatches. The dense/sparse performance gap is gone — previously sparse -~30-50% slower than dense, reproducibly; now within ~1-3% either way -(7.42s dense vs 7.60s sparse for `Escherichia_coli--K-12_MG1655`; 5.25s vs -5.30s for `Saccharolobus_islandicus--M.16.4`) — noise-level, not a -systematic gap. `pack --sparse`'s claimed query win isn't confirmed -outright by this (sparse should arguably now *beat* dense on truly sparse -real data, not just tie), but the pathological regression is fixed. - -## `PersistentCompactIntMatrix::Sparse` — implemented (2026-08-26) - -Closes the gap flagged throughout this document ("no sparse count format -exists yet", `traits.rs:9-12`'s "Explicitly deferred"): `obicompactvec` -already had `PersistentSparseCompactIntMatrix` (row-major, built on top of -`PersistentSparseBitMatrix` as its "which columns are non-zero" support, -values *not* deduplicated — see that struct's own doc comment), but it was -never wired into `PersistentCompactIntMatrix`, the dense-dispatching enum -every real consumer (`TypedLayer`, -`KmerLayer::Count`) actually holds. Concretely: `kmer_index.rs:: -pack_matrices(sparse=true)` already called `pack_sparse_compact_int_matrix` -on every layer's `counts/` — but `PersistentCompactIntMatrix::open` had no -code path back to what that just wrote, so a `Count` layer became -unreadable ("no count matrix found ... run 'obikmer upgrade'") the moment -anyone ran `pack --sparse` on an index with count layers. Root cause, not a -workaround: add the missing `Sparse` variant. - -- **Enum + dispatch** (`intmatrix.rs`): `PersistentCompactIntMatrix::Sparse - (PersistentSparseCompactIntMatrix)`, detected in `open`/`detect_storage` - via a `singleton_values.pciv` marker (mirrors `PersistentBitMatrix`'s own - `sparse_meta.json` check), reported via `storage_kind()`. `col`/ - `col_view`/`col_persist` panic/`Unsupported` on `Sparse`, same convention - as the bit side. `sub_matrix`/`fill_sub_matrix` and `nonzero_iter` - unified the same way `PersistentBitMatrix`'s already are (drain - `nonzero_iter`, one traversal per format — see "Implemented - (2026-08-20)" above); `nonzero_iter` had to become `Box>` - for the same reason (`Columnar`/`Packed`/`Sparse` are different concrete - types). No change needed in `obikindex` at all — `KmerLayer::Count` - already only ever holds `TypedLayer`, so the - enum absorbing `Sparse` fixes the unreadable-layer bug for free, same as - `PersistentBitMatrix::Sparse` already did on the presence side. - -- **`CountPartials`, non-naive** (`sparse_intmatrix.rs`): unlike - `PersistentSparseBitMatrix`'s dict-driven `col_weights_and_pair_counts`, - values here aren't deduplicated (two rows can share the same non-zero - column set via the same `dict_id` while carrying different counts), so - the "weight by how many rows share a dict entry" shortcut doesn't carry - over. What does: a single row-major pass (`row_major_pairwise`, decodes - each row once via `for_each_cell_in_row`, nests over that row's own - co-present columns) — `O(Σ k̄²)` over populated rows instead of the naive - `O(n_cols² × n)` column-pair rescan, same complexity class as the bit - side minus the dict multiplicity discount. Kernels used: `min(a,b)` - (bray, relfreq-bray — both vanish when either side is absent, so no - correction needed), `a·b` and `√(a·b)` (euclidean/relfreq-euclidean and - hellinger — these *do* need a correction, reconstructed from per-column - marginals via `Σ(a-b)² = Σa²+Σb²-2Σab`, since `(a-0)² = a² ≠ 0` unlike - the `min`-based formulas). `threshold_jaccard(1)` shortcuts straight to - `support`'s own `BitPartials::partial_jaccard` (threshold 1 is exactly - presence); `threshold_jaccard(0)` is closed-form (every `u32` is `≥ 0`). - -- **Two pre-existing bugs found and fixed while wiring the `threshold==1` - shortcut** (`bitmatrix/sparse.rs`, `BitPartials for - PersistentSparseBitMatrix`, present since the 2026-08-15 implementation - above, never caught because no test compared `Sparse`'s raw `partial_*` - output against dense on real data — only the diagonal-blind - `jaccard_dist_matrix`/`hamming_dist_matrix` finalisations were tested): - 1. `partial_jaccard`'s diagonal was `(0, 2×col_weights[i])` instead of a - genuine self-comparison `(col_weights[i], col_weights[i])` — - `col_weights_and_pair_counts`'s `inter` never pairs a column with - itself by construction. - 2. `partial_hamming`'s off-diagonal formula itself was wrong: `total - - union` (count of rows where *neither* column is present) instead of - the actual Hamming distance `col_weights[i] + col_weights[j] - - 2×inter[i,j]` (symmetric-difference size). Only coincides with the - correct value when `col_weights[i] + col_weights[j] == total`, so - small/synthetic test data could easily have hidden it. - - Neither surfaced through `jaccard_dist_matrix`/`hamming_dist_matrix` - (both explicitly zero their own diagonal at finalisation, and the - off-diagonal `partial_hamming` bug had gone untested against dense - entirely) — only visible to a caller of the raw `partial_*` methods - directly, which is exactly what `partial_threshold_jaccard(1)`'s new - shortcut became. Fixed at the source, not patched around at the call - site; regression test added: - `tests::sparse::partial_jaccard_and_hamming_match_dense_including_diagonal`. - -- **Tests**: `tests::intmatrix::sparse_roundtrip_matches_columnar`/ - `sparse_roundtrip_from_packed` (the `open`-dispatch fix, both build - paths); `tests::intmatrix::sparse_count_partials_match_dense` (all six - `CountPartials` formulas, thresholds 0/1/2/3, against `Columnar` on - asymmetric-presence data — this is what caught the diagonal gap in the - int side's own new code before it shipped, the same way it exposed the - two pre-existing bit-side bugs above); `obikindex`'s - `count_layer_transparently_reads_sparse_after_pack` — the actual - end-to-end regression test for the original "layer unreadable after - `pack --sparse`" bug, built → packed sparse → reopened, compared against - the pre-pack dense read. `cargo test -p obicompactvec -p obikindex`: - green, no regressions (180 + 12 tests). +# Sibling annex & SNP-family pipeline — architecture + +## Data model + +A **family** is the up-to-4 k-mers sharing all flanks and differing only at +the central base (well-defined for odd `k`). `FamilyMask` (`siblingannex.rs`) +is a 4-bit presence mask (bit `b` set iff the family member whose own +canonical central base is `b` is observed anywhere in the index) plus a +minorant flag; `family_size()`/`siblings()` derive member count from it. +`SiblingAnnex`/`SiblingAnnexBuilder` persist one `FamilyMask` per k-mer, +**keyed by iteration order** (the physical layout of `unitigs.bin`), never by +MPHF slot — this lets every reader zip-iterate a layer's own k-mer stream +against the annex file directly, no MPHF/slot indirection. + +## Construction + +`algorithms::annex::build_layer_sibling_annex` builds one layer's annex: +seeds each k-mer's own central base, generates its up-to-3 central-base +variants, resolves each cross-partition (`IndexCache::find_in_partition`, +grouped and chunked per destination partition for mmap locality), writes the +final mask (with the minorant flag) in a second streamed pass. +`SiblingExt::build_sibling_annex` drives this for every layer of every +partition, then invalidates and rebuilds the entropy and family-overlap +annexes (their `family_idx` numbering depends on the sibling annex). + +## Shared traversal + +`algorithms::family_scan::scan_layer_families` is the **only** place that +resolves a family's full per-genome presence (`genome_mask: &[u8]`, +cross-partition) — every other consumer in this module is built on it, never +reimplements the resolution. Two decoupled concurrency mechanisms: batch +generation (CPU-only) runs on an `obipipeline` throttle/pipe; per-batch +resolution (I/O-bound, mmap page faults) runs `rayon`-parallel across +destination partitions, one generated batch at a time (never several +concurrently, to keep one partition's pages hot instead of scattering +access). `fast_mode` (`is_fast_mode`, `<= 7` layers per partition) picks +between trusting a family member's recorded layer index outright or probing +every layer of the destination partition. + +`Selection<'a>` bounds which families of a layer are resolved: + +- `All` — every minorant, monomorphic included (`SiblingExt::shannon_entropy_csv`, + `sibling_annex_stats`). +- `Some(&HashSet)` — a caller-picked subset, skipping the expensive + cross-partition resolution entirely for excluded families + (`subsample::sample_layer`'s accepted-candidates batch; + `entropy::ensure_layer_entropy_annex`'s non-monomorphic-only restriction). + +`minorant_selection::eligible_bitset_layer`/`non_monomorphic_reindex_layer` +provide the cheap, annex-bits-only (no cross-partition resolution) filters +these restricted scans are built from: monomorphism (`family_size() < 2`) is +knowable straight from the annex, so nothing pays `scan_layer_families`'s +resolution cost just to discard a monomorphic result. + +## Masking — computed once, shared everywhere + +`algorithms::masking::masked_state(m, free_loss, no_ambiguity)` is the single +function that turns a raw per-genome 4-bit state into the masked view every +consumer needs (`None` for absence under `--free-loss` or ambiguity under +`--no-ambiguity`). It is called **exactly once per (family, genome)**, in +`subsample::sample_layer`, immediately after `scan_layer_families` resolves +the raw state — never recomputed downstream. `survives_masking` (does the +family still carry ≥2 distinct masked states among included genomes?) and +`SurvivingFamily::masked: Vec>` both consume this same +precomputed array, which is also what `AlignmentReducer`/`PairwiseReducer` +read (see below) — this is what guarantees the alignment and the pairwise +tally can never disagree about a genome's state at a given site. + +## Precomputed annexes (entropy, family overlap) + +`algorithms::entropy::ensure_layer_entropy_annex` builds `EntropyAnnex` +(`entropy_annex.rs`, one `f32` Shannon-entropy value per non-monomorphic +minorant, mmap-backed) and/or feeds `FamilyOverlapAccumulator` +(`family_overlap.rs`) from one shared `Selection::Some` scan — both need the +same `genome_mask` resolution, so a missing one never pays for it twice. +`family_entropy` (15 non-empty states of `{A,C,G,T}`'s powerset, `∅` +excluded from the denominator — the project's settled informativeness +definition, matching the calibrated 16-state Sankoff cost matrix) and +`family_entropy_4` (plain 4-symbol reduction, kept only for comparison) are +pure functions of `genome_mask`. `extensions::SiblingBuilder::ensure_entropy_and_overlap_annexes` +drives this for every cached layer, lazily (only what's missing). + +## Sampling and the shared scan trunk + +`algorithms::subsample::sample_index` draws up to `n` non-monomorphic +families index-wide, proportionally per layer: for each layer, a target +quota sized from that layer's share of the index-wide eligible count, filled +by a circular walk with a Bernoulli(`p`) draw per candidate — `p` optionally +weighted by an unnormalized Gaussian kernel over the candidate's entropy +(`EntropyBias { mu, sigma }`, read from the entropy annex). Accepted +candidates are resolved in batches (`scan_layer_families`) and checked +against `survives_masking`; rejected-after-masking candidates don't count +toward quota, triggering further "topup" rounds (bounded, `MAX_TOPUP_ROUNDS += 5`) rather than silently under-filling. Every `SurvivingFamily` reaching a +caller has already survived both the draw and post-masking informativeness — +no second filter pass needed downstream. + +`algorithms::scan::sampled_scan` is the single entry point every sampled +computation goes through: one `sample_index` pass, fanned out to whichever +`LayerReducer` implementations the caller actually needs, each receiving the +exact same per-layer `SurvivingFamily` batch (an `Arc`, cloned not copied). +This is what guarantees several outputs requested together +(`--pseudo-alignment` and `--sankoff` and a `snp-*` `--distance` in one CLI +invocation) come from **one** random sample, not independently-drawn ones — +there is no other driver of `sample_index` anywhere in this crate. + +Two `LayerReducer` implementations exist today: + +- `AlignmentReducer` (`alignment.rs`) — accumulates `SnpAlignment` (one + IUPAC-coded row per included genome), reading `family.masked` directly. +- `PairwiseReducer` (`pairwise.rs`) — accumulates `PairwiseTally`/ + `PartitionDispersion`, also reading `family.masked` directly (substitution/ + agreement counts and cardinality both derive from the same masked state + the alignment uses). + +## From a tally to distances and calibration + +`PairwiseTally` stores per-genome-pair substitution/agreement/cardinality +counts, unfiltered, from which every downstream output is cheap `O(n²)` +post-processing (never a second scan): + +- `algorithms::snp_distance::distance_matrix` — closed-form `snp-*` + corrections (raw p-distance, JC69, K80, K81, F81, T92, TN93, transversions- + only), optionally gamma-corrected (`GammaShape`, `PartitionDispersion::estimate_alpha` + for `Auto`). +- `algorithms::sankoff::SankoffBundle::finish` — assembles the + `--sankoff`/`--tnt`/`--phyg`/`--iqtree` pipeline's inputs + (`RawSnpDistanceOutput`, `BasePairTally`, `CardinalityTally`) from an + already-scanned `AlignmentReducer`/`PairwiseReducer` pair, plus + `SankoffBundle::snp_distance` to reuse that exact sample for a `snp-*` + `--distance` in the same invocation without a second scan. +- `algorithms::cardcomp` — pure functions turning `CardinalityTally`/ + `BasePairTally` into row-stochastic transition probability matrices, then + the calibrated 16-state Sankoff cost matrix (`pairwise_cost_matrix`). + +## Public API surface + +`extensions::SiblingExt` (CLI-facing trait on `IndexCache`) covers only the +operations with no cross-invocation sharing concern: `build_sibling_annex`, +`shannon_entropy_csv`, `sibling_family_size_histogram`/`sibling_annex_stats`, +`family_overlap`. The sampled SNP path (`sampled_scan`, `AlignmentReducer`, +`PairwiseReducer`, `distance_matrix`, `SankoffBundle`) is **not** wrapped in +trait methods — the CLI (`obikmer::cmd::phylo`) composes them directly, +because it alone knows the union of reducers a given invocation needs +(`--pseudo-alignment`/Sankoff group/`snp-*` `--distance` can combine in one +call). A per-output trait method would have to either duplicate that +composition logic or reintroduce the "two independent scans" problem this +design replaced. + +## Known gaps + +- `sample_index`'s `skip_layers` parameter (checkpointed-resume support) is + never called with a non-zero value by any current caller — `--session` + persistence was removed pending a redesign on top of `sampled_scan` + (single shared trunk, not per-driver checkpoint plumbing). The parameter + stays as the seam a future resume mechanism would use. +- No cheap entropy pre-filter exists yet for the very high (saturated) tail — + a row-marginal coverage approximation was considered and rejected (cannot + represent joint multi-base co-occurrence, which is exactly what the + 15-state entropy definition exists to capture). diff --git a/DevDocMD/implementation/benchmark_query_testing.md b/DevDocMD/implementation/benchmark_query_testing.md index e234e984..cb9c358c 100644 --- a/DevDocMD/implementation/benchmark_query_testing.md +++ b/DevDocMD/implementation/benchmark_query_testing.md @@ -12,8 +12,9 @@ this pipeline. by `verify_presence`/`verify_merge_presence`, which only check index *content* against the `.npz` truth, never the query API. - `pack --sparse` produces a presence-matrix format documented (see - [siblings.md](../architecture/siblings.md)) as faster for single-row - access (query) and slower for column-oriented access (phylo `--metric`). + [obicompactvec.md](obicompactvec.md), "Sparse matrix formats") as faster + for single-row access (query) and slower for column-oriented access + (phylo `--metric`). `global_index_presence/` built by `merge_presence.sh` is always packed dense (packing is a stage inside `merge`, not a separate `pack` invocation) — there was no dense/sparse regression check. diff --git a/DevDocMD/implementation/layer_tier.md b/DevDocMD/implementation/layer_tier.md index 57499b99..e61c34a7 100644 --- a/DevDocMD/implementation/layer_tier.md +++ b/DevDocMD/implementation/layer_tier.md @@ -299,7 +299,7 @@ pub fn sub_matrix(&self, slots: &[usize]) -> Vec> pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec]) ``` -Column-first to match the on-disk column-major layout. `fill_sub_matrix` sorts `slots` once, then calls each column's `fill_batch` in turn — no redundant per-column sort. On `PersistentSparseBitMatrix` (k-mer-major, no column method) this degrades to a row-by-row decode; see [siblings.md](../architecture/siblings.md). +Column-first to match the on-disk column-major layout. `fill_sub_matrix` sorts `slots` once, then calls each column's `fill_batch` in turn — no redundant per-column sort. On `PersistentSparseBitMatrix` (k-mer-major, no column method) this goes through `nonzero_iter` instead, decoding one row at a time; see [obicompactvec.md](obicompactvec.md), "Sparse matrix formats". --- diff --git a/DevDocMD/implementation/obicompactvec.md b/DevDocMD/implementation/obicompactvec.md index b999793d..fd29112d 100644 --- a/DevDocMD/implementation/obicompactvec.md +++ b/DevDocMD/implementation/obicompactvec.md @@ -298,13 +298,11 @@ Both matrix types are enums behind a transparent API — the caller never matche |---|---|---| | `Columnar` | one `.pbiv`/`.pciv` file per column + `meta.json` | build-time default (`*Builder::new`) | | `Packed` | single `matrix.pbmx`/`matrix.pcmx` mmap file | query-optimised, produced by `pack_bit_matrix`/`pack_compact_int_matrix` | -| `Sparse` | bit: `sparse_meta.json` + PFIV/Elias-Fano component files, row-major. Int: same support files (built on `PersistentSparseBitMatrix` internally) plus `singleton_values.pciv`/`multi_values.pciv`/`multi_offsets` for the per-row, non-deduplicated values | `pack --sparse`; see [siblings.md](../architecture/siblings.md) for the sparse-vs-dense access-pattern trade-off | +| `Sparse` | bit: `sparse_meta.json` + PFIV/Elias-Fano component files, row-major. Int: same support files (built on `PersistentSparseBitMatrix` internally) plus `singleton_values.pciv`/`multi_values.pciv`/`multi_offsets` for the per-row, non-deduplicated values | `pack --sparse`; see "Sparse matrix formats" below for the sparse-vs-dense access-pattern trade-off | | `Implicit` (bit only) | no file at all | mono-genome presence layers — `n_cols` is always reported as `1`, every value is `true` | `PersistentBitMatrix::open(layer_dir)` auto-detects the variant, in order: `matrix.pbmx` → Packed, `presence/meta.json` → Columnar, `presence/sparse_meta.json` → Sparse, `layer_meta.json` (no presence dir at all) → Implicit. `PersistentCompactIntMatrix::open(layer_dir)` mirrors the same priority order minus `Implicit` (there's no implicit count matrix — counts always have at least one on-disk column): `matrix.pcmx` → Packed, `counts/meta.json` → Columnar, `counts/singleton_values.pciv` → Sparse. `col_view`/`col`/`sub_matrix` panic on `Sparse`/`Implicit` where the operation has no direct-slice equivalent (Sparse is k-mer-major, not column-major; Implicit has no backing storage) — callers needing per-column data on those variants go through `row`/`fill_row`. -Unlike the bit side, `PersistentSparseCompactIntMatrix`'s values are *not* deduplicated across rows — two rows can share the same non-zero column set (same `dict_id` in the shared support) while carrying different counts — so its `CountPartials` impl can't reuse the support's dict-multiplicity shortcut the way `BitPartials for PersistentSparseBitMatrix` does. It still avoids the naive `O(n_cols² × n)` column-pair scan via a single row-major pass (`row_major_pairwise` in `sparse_intmatrix.rs`), reconstructing the squared-difference formulas (`euclidean`/`relfreq_euclidean`/`hellinger`) from per-column marginals via `Σ(a-b)² = Σa²+Σb²-2Σab` — see [siblings.md](../architecture/siblings.md)'s "`PersistentCompactIntMatrix::Sparse` — implemented" entry for the full derivation. - `col_view(c)` returns the appropriate view directly: ```rust @@ -321,6 +319,163 @@ No wrapper enums (`BitColView`, `IntColView`): the caller receives a `Copy` view --- +## Sparse matrix formats + +Motivated by real k-mer-family data being extremely row-sparse (most rows — +k-mers — present in only a handful of genome columns out of dozens to +thousands): a row-major, deduplicated alternative to the dense column-major +formats above, built once per layer like the rest of the pipeline (never the +whole multi-billion-row index at once). Implements the exact-duplicate +special case of Alanko, Bille, Gørtz, Navarro, Puglisi, "Compact Data +Structures for Collections of Sets" (2025, +`biblio/Alanko et al. - Compact Data Structures for Collections of +Sets.pdf`) — a plain dedup dictionary, not their full subset-containment +hierarchy. + +### `PersistentSparseBitMatrix` + +Four mmap-backed on-disk components: an `is_multi` rank-capable flag per row +(singleton vs. multi-genome), a fixed-bit-width array for singleton rows +(genome index directly, `ceil(log2(n_cols))` bits), a separate fixed-width +array for multi-genome rows (`dict_id`, `ceil(log2(n_distinct_multi_sets))` +bits — kept apart from the singleton array because `n_distinct_multi_sets` +can be large in absolute terms even when multi-genome rows are a small +*fraction* of all rows, and a shared array would force every row, singletons +included, to pay the wider width), and a deduplicated dictionary of distinct +multi-genome sets (Elias-Fano-encoded byte offsets + a varint-encoded values +blob). Supporting primitives: `PersistentFixedIntVec` (arbitrary, +runtime-parameterised bit width, width `0` included — needed once an +`EliasFano` reopen bug surfaced, see below), `PersistentRankSelectBitVec` +(rank1/rank0/select1 on top of the crate's existing `count_ones`, via +`common_traits::SelectInWord`), `EliasFano` (composes the two). `BinaryMatrix` +(`n`, `n_cols`, `row`/`fill_row`, `fill_sub_matrix`, `count_ones`) is the +trait unifying dense and sparse at the call sites that need both +interchangeably; column-oriented methods (`col`, `col_view`, the +`partial_*_dist_matrix` family) stay dense-only — `Sparse`/`Implicit` panic +there (see the `PersistentBitMatrix::open` note above). + +Measured on real data (one layer, 30.2M rows, 91 genomes): + +| | dense | sparse | ratio | +|---|---|---|---| +| on-disk size | 328.1MB | 43.7MB | **7.5x** smaller | +| build time / peak RSS | — | 4.26s / 628MB | per-layer, in-memory construction | +| row access, sequential (2M reads) | 43ns/row | 32ns/row | sparse faster (smaller structure, better cache fit) | +| row access, random (2M reads) | 409ns/row | 85ns/row | sparse **~4.8x faster** — the `--shannon`/family-lookup access shape | +| column access, one full column (30.2M rows) | 11.5ms | 993ms | sparse **86x slower** — no native column method, every read decodes a full row to keep one bit | + +The row-access wins (compactness was the stated design goal, not speed) come +from dense's genome-major layout scattering a single row read across a much +bigger file — costlier than sparse's rank/select/varint decode once the file +is this much smaller. Column access is the flip side of the same layout +choice; `nonzero_iter` (below) is what removes it from the hot path that +actually matters (`query`). + +Two reopen-correctness bugs, now fixed at the source: (1) `EliasFano::open` +re-derived its low-bits width from the persisted low-vector's own width +byte; the width-`0` case was built with a dummy 1-bit placeholder (the +builder used to reject true width `0`), silently doubling every reopened +value. Fixed by making `PersistentFixedIntVec` genuinely support width `0` +(no storage, `get` always `0`). (2) An empty row (cardinality 0) used to be +recorded as a singleton at genome `0`, indistinguishable on read-back from a +*real* singleton at genome `0`. Fixed by routing cardinality-0 rows through +the dictionary path (a genuine empty entry). + +Still deferred: full Alanko et al. subset-hierarchy compression (only the +exact-duplicate case is built), a sparse `PersistentCompactIntMatrix` +variant beyond what's described below, BRWT-style column-correlation +exploitation, and a row-major rewrite of +`partial_jaccard_dist_matrix`/`partial_hamming_dist_matrix`/etc. as a +per-row co-occurrence accumulation (`O(Σ_rows k̄²)`) so `obikindex`'s +`--metric` distance-matrix path stops paying the column-access penalty on +`Sparse` — needs its own design pass (how it plugs into +`BitPartials`/`ColumnWeights` so both matrix types keep serving `--metric`), +not just "port the loop". + +### `nonzero_iter` — the one traversal per format + +`fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> Box + 'a>` +(item: index into `slots`, column, value) is the single foundational +per-format traversal primitive every consumer builds on — `fill_sub_matrix` +and a `for_each`-style callback are both trivial, free consumers of it +(`.for_each(f)` is `Iterator::for_each`; `fill_sub_matrix` drains the +iterator and scatters into `out[][]`), instead of two independently +hand-maintained traversals that can silently diverge. `Box`, +not a bare `impl Iterator`: `Columnar`/`Packed`/`Sparse`/`Implicit` are +genuinely different concrete types and the method isn't on a +dyn-compatible trait (`BinaryMatrix` is used as `dyn BinaryMatrix` +elsewhere, and RPITIT methods aren't dyn-compatible) — one `Box` per +`nonzero_iter` call, not per cell. + +Per-format implementation: `PersistentSparseBitMatrix` decodes natively, +buffered one row at a time (`std::iter::from_fn` over the existing +`for_each_genome_in_row`, no `n_cols`-wide allocation ever). `Columnar`/ +`Packed` loop columns, collecting each column's hits via +`BitSliceView::nonzero_among_sorted`/`IntSliceView::nonzero_among_sorted` +(a `filter`/`filter_map` over the vector-level `get()`, no new state +machine — `std`'s own `Filter` iterator already is one) into one `Vec` via +`.extend()` (not `flat_map`: a `flat_map` closure can't lazily return +something borrowing its own captured sort permutation across separate calls +without either boxing per-column or fighting the borrow checker — eager +collection sidesteps it at zero extra cost, since `fill_sub_matrix` already +fully materialises anyway). `Implicit` is trivial (`slots.iter().map(|&i| (i, 0, 1))`). + +This closed a real dispatch bug by construction, not by patching it: +`PersistentBitMatrix::fill_sub_matrix`/`sub_matrix` used to reimplement a +naive per-`(column, slot)` loop instead of delegating to +`PersistentSparseBitMatrix`'s own efficient native method for `Self::Sparse` +— two hand-written traversals for the same format, free to drift apart, and +they had (`obikphylo`'s own cache layer had independently built a parallel +enum wrapper that *did* dispatch correctly, evidence the bug was worked +around rather than fixed at the source). With `fill_sub_matrix`/`sub_matrix` +rewritten to drain `nonzero_iter`, there is only one traversal per format +left to get right. `KmerPartition::query_partition_with` +(`obikindex/src/partition/query_layer.rs`) — previously column-major +(`for g { for slot { col_value } }`, which for `Sparse` rebuilt a full row +per single cell touched, O(hits × n_cols) instead of O(hits) — the original +motivation for this primitive) — now issues one `layer.nonzero_iter(&slot_list)` +call per layer, format-agnostic. Dense/sparse query performance parity +confirmed on real data (previously sparse 30-50% slower than dense, +reproducibly; now within ~1-3% either way — noise level). + +### `PersistentSparseCompactIntMatrix` + +The count-matrix analogue, built on `PersistentSparseBitMatrix` as its +"which columns are non-zero" support, values **not** deduplicated (unlike +the bit side, two rows can share the same non-zero column set — same +`dict_id` — while carrying different counts, so the dict-multiplicity +shortcut `BitPartials for PersistentSparseBitMatrix` uses doesn't carry +over). `CountPartials` here: a single row-major pass (`row_major_pairwise` +in `sparse_intmatrix.rs`, decodes each row once via +`for_each_cell_in_row`, nests over that row's own co-present columns) — +`O(Σ k̄²)` over populated rows, avoiding the naive `O(n_cols² × n)` +column-pair rescan. `min(a,b)`-based kernels (`bray`, `relfreq_bray`) need +no correction — they vanish when either side is absent. `a·b` and `√(a·b)` +kernels (`euclidean`/`relfreq_euclidean`/`hellinger`) do need one, since +`(a-0)² = a² ≠ 0` unlike the `min`-based formulas — reconstructed from +per-column marginals via `Σ(a-b)² = Σa²+Σb²-2Σab`. +`threshold_jaccard(1)` shortcuts to `BitPartials::partial_jaccard` +(threshold 1 is exactly presence); `threshold_jaccard(0)` is closed-form +(every `u32` is `≥ 0`). + +Wiring the `threshold==1` shortcut surfaced two pre-existing formula bugs in +`BitPartials for PersistentSparseBitMatrix` (present since that type's own +introduction, never caught because no test had compared `Sparse`'s raw +`partial_*` output against dense on real data — only the diagonal-blind +`jaccard_dist_matrix`/`hamming_dist_matrix` finalisations were tested), +fixed at the source: + +1. `partial_jaccard`'s diagonal used to be `(0, 2×col_weights[i])` instead + of a genuine self-comparison `(col_weights[i], col_weights[i])` — + `col_weights_and_pair_counts`'s `inter` never pairs a column with itself + by construction, so the diagonal needs its own case. +2. `partial_hamming`'s off-diagonal formula itself was wrong: it computed + `total - union` (rows where *neither* column is present) instead of the + actual Hamming distance `col_weights[i] + col_weights[j] - 2×inter[i,j]` + (symmetric-difference size) — the two only coincide when + `col_weights[i] + col_weights[j] == total`, which small/synthetic test + data can easily satisfy by accident. + ## Aggregation traits (matrix level) ### ColumnWeights diff --git a/mkdocs.yml b/mkdocs.yml index f1aa9f13..6f493d68 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -60,7 +60,7 @@ nav: - Architecture: - Sequences: architecture/sequences/invariant.md - Kmer index: architecture/index_architecture.md - - Sibling annex (discussion): architecture/siblings.md + - Sibling annex: architecture/siblings.md - NUMA-aware worker pools: architecture/numa_worker_pools.md - NUMA-aware partition runner: architecture/numa_partition_runner.md -- 2.54.1 From 0d12494f1a05a075dd448a5ac739df79e81ccbf3 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Mon, 12 Oct 2026 21:00:48 +0200 Subject: [PATCH 09/10] Refactor indexing, sampling, and entropy documentation This commit introduces new theoretical frameworks for k-mer indexing, pair coverage sampling, and sibling partition unification. It also implements new algorithms for calculating family entropy and statistical tests for overlap analysis. Documentation is updated across the board to reflect these new features and theoretical concepts. --- DevDocMD/theory/chance_sharing.md | 69 +++ DevDocMD/theory/pair_coverage_sampling.md | 222 ++++++++++ .../theory/sibling_partition_unification.md | 60 +++ UserDocMD/molecular-ecology-modified.csl | 78 +++- UserDocMD/references.bib | 89 ++++ UserDocMD/theory/kmer_indexing/kmers.md | 240 ++++++++++- UserDocMD/usage/phylo.md | 3 +- mkdocs.yml | 3 + .../pseudo_alignment_nesting.cpython-314.pyc | Bin 0 -> 29161 bytes scripts/pseudo_alignment_nesting.py | 395 ++++++++++++++++++ .../src/siblings/algorithms/entropy.rs | 62 ++- .../src/siblings/extensions/sibling_ext.rs | 17 +- 12 files changed, 1182 insertions(+), 56 deletions(-) create mode 100644 DevDocMD/theory/chance_sharing.md create mode 100644 DevDocMD/theory/pair_coverage_sampling.md create mode 100644 DevDocMD/theory/sibling_partition_unification.md create mode 100644 scripts/__pycache__/pseudo_alignment_nesting.cpython-314.pyc create mode 100644 scripts/pseudo_alignment_nesting.py diff --git a/DevDocMD/theory/chance_sharing.md b/DevDocMD/theory/chance_sharing.md new file mode 100644 index 00000000..181b8d6b --- /dev/null +++ b/DevDocMD/theory/chance_sharing.md @@ -0,0 +1,69 @@ +# Chance sharing of a kmer across genomes (discussion) + +Null model for "a kmer is present in $r$ of $S$ genomes": how likely is this by chance alone? Extends the single-pair estimate $P_{\mathrm{chance}}\simeq G/4^k$ (see user doc, *K-mer specificity*). + +## Homogeneous case: binomial tail + +If every genome has the same per-genome chance probability $p$, the number $R$ of genomes containing a given kmer is $\mathrm{Bin}(S,p)$: + +$$P(R\geq r)=\sum_{j=r}^{S}\binom{S}{j}p^j(1-p)^{S-j}=I_p(r,\,S-r+1)$$ + +where $I_p$ is the regularized incomplete beta function. No elementary closed form; use $I_p$ directly. + +### Numerical evaluation of the sum + +- Term recurrence, one `lgamma` start: $t_{j+1}=t_j\cdot\frac{S-j}{j+1}\cdot\frac{p}{1-p}$. +- Sum the tail **not containing the mode** ($Sp$, not $S/2$). If $r>Sp$, sum $\sum_{j\ge r}$ directly and stop when terms are negligible. The complement $1-\sum_{j=0}^{r-1}$ (note: from $j=0$) suffers catastrophic cancellation when the p-value is tiny. +- Large $S$: normal approximation with continuity correction, or Poisson($Sp$) when $p$ is small. Unreliable in extreme tails. + +## Heterogeneous case: $p_i = f(G_i)$ + +The $G_i$ (number of kmers of genome $i$) are known and differ. With $p_i\simeq G_i/4^k$, the indicators "kmer present in genome $i$ by chance" are independent Bernoulli($p_i$), so $R$ follows a **Poisson-binomial** law, not a binomial: + +$$P(R=j)=\sum_{|A|=j}\prod_{i\in A}p_i\prod_{i\notin A}(1-p_i).$$ + +Everything lives in canonical space: $G_i$ is the number of distinct canonical kmers of genome $i$, and the kmer space has size + +$$N=\frac{4^k}{2}$$ + +exactly, since $k$ is odd (no kmer equals its own reverse complement, so canonicalization pairs the $4^k$ kmers perfectly). Exact per-genome probability: $p_i=1-(1-1/N)^{G_i}\simeq 2G_i/4^k$. This is twice the $G/4^k$ of the user doc, where $G$ is a number of positions on one strand. + +### Exact computation + +Dynamic programming over genomes, $d_j$ = probability of $j$ hits so far: + +$$d'_j=d_j(1-p_i)+d_{j-1}p_i.$$ + +For the upper tail only $d_0\ldots d_{r-1}$ are needed: $P(R\ge r)=1-\sum_{j32$ only ~19 % have $H\in[0.5,1.5)$, and the 129-180 class is 0.04 % of all families. Flattening deep-pair coverage works on little informative material whatever the weights. +- Consequence for a $\mu(n)$ that grows with $n$: it would not recover these families (80 % sit below 0.5 bit); what matters at large $n$ is the share of families in the band around 1 bit, not the position of the centre. Conditional weight $w_H(f)/E[w_H\mid n_f]$ remains an option to make the entropy filter neutral to $n$; $E[w_H\mid n]$ is now available from this table. + +Limit: the table has `n_genomes_present` but not the reach of the family, so the link to specific pairs is still not measured. + +## Site concordance factors on the Primulaceae tree (measured 2026-10-12) + +Command (IQ-TREE 3.1.3): `iqtree3 -t -s -m GTR+G4+ASC --scfl 100 --cf-verbose -T 5 --prefix concord`. With `-t` plus `-m`, IQ-TREE did **not** just annotate the reference tree: it ran a full ML search started from it (102 iterations, ~59 h wall-clock). `concord.treefile` differs from the input treefile by one bipartition (Robinson-Foulds 2), so sCF refer to the re-inferred tree. Without `-m`, ModelFinder is launched and required 157 GB (first attempt failed). The log also warns that all 180 sequences contain more than 50 % gaps/ambiguity. `sN` below = informative sites averaged over the 100 sampled quartets of a branch (alignment: 5 003 414 columns). + +**Deep Primula/Androsace interface: too few informative sites (data scarcity).** + +| Node | Tips (Primula / Androsace) | sCF | sDF1 | sDF2 | `sN` | +|---|---|---|---|---|---| +| 209 | 67 (43 / 24) | 27 | 47 | 26 | 17 | +| 210 | 66 (43 / 23) | 37 | 33 | 30 | 43 | +| 208 | 74 (50 / 24) | 54 | 16 | 29 | 190 | +| 211 | 24 (2 / 22) | 64 | 18 | 18 | 226 | +| 212 | 21 (2 / 19) | 30 | 38 | 32 | 184 | +| 216 | 10 (2 / 8) | 21 | 47 | 32 | 7 | +| 217 | 9 (1 / 8) | 43 | 20 | 36 | 7 | + +Quartets crossing these nodes have tens to a few hundred informative sites out of 5 M columns, the quantitative counterpart of the pair-overlap scarcity. Primula tips are spread over several nodes mixed with Androsace (e.g. 43 + 24 at node 209); no monophyletic Primula in this tree. + +**Within Androsace: conflict, not scarcity.** Nodes 283 (72 Androsace, `sN` 17 127), 292 (49 tips, 10 335), 305 (22, 11 980), 342 (11, 6 566) have thousands of informative sites but sCF 33-37 % with both alternatives nearly as supported: sites split almost evenly among the three quartet topologies. More sites would not fix this; cause not established (a poor fit of a tree model to intra-species divergence is one hypothesis, untested). Node 185 (23 Androsace): sDF1 53.6 % > sCF 30.6 %, a majority of the 893 sites prefer an alternative (real conflict). + +**Well-supported nodes have long branches:** node 296 (45 tips, length 0.23, sCF 90 %), 341 (0.21, 91 %), 330 (0.09, 85.5 %), 181 (0.099, 89.5 %); most branches of length 0.01-0.05 have sCF 33-45 %. Degenerate: node 186 (length 0.001, `sN` 0.37) and 187 (`sN` 2.2) carry no informative site. + +Median sCF shows no gradient with clade size (about 39-48 % in every class), so the clade-size class hides the effect; the scarcity is specific to the interface nodes above. + +**Implication for sampling.** The part a sampling design can address is the interface scarcity: select families carried by genomes from the four groups around each target node (a quartet-targeted selection) rather than flattening the whole pair matrix. Not assessed: how many such families the index holds (the `sN` of nodes 209 and 216 suggests few). For the intra-Androsace conflict no sampling strategy is suggested by these measurements alone. + +## Quartet coverage (measured 2026-10-12, `--quartet-test`) + +`scripts/pseudo_alignment_nesting.py --quartet-test --quartets N` draws random genome quartets and counts exactly the columns where all four carry a call, stratified by the size of the smallest tree clade containing the four. Cost: 20 000 quartets over 5.0 M columns in about 35 s total. Exact per-family quartet weighting is out of reach ($\binom{180}{4}\approx 42.3$ M quartets; one family carried by 100 genomes covers $\binom{100}{4}\approx 3.9$ M), so quartets serve as a *diagnostic* of a sample, not as a weight. + +Result on the current sample: 4-way count quantiles 0 / 1 / 10 / 50 / 90 / 99 / 100 % = 0, 3, 68, 200, 2 329, 66 504, 194 460 (CV 3.5, 0.1 % uncovered). By depth: median 131-209 for quartets inside clades ≥ 65 genomes versus 11 580-57 387 for clades 17-64. The ratio of the 4-way count to the smallest of the six pair overlaps has median 0.20-0.29 for deep quartets (0.53 for clades 33-64; 1 would be perfect nesting): pair overlap is only a loose upper bound of 4-way coverage, so flattening pairs does not by itself flatten quartets. 88 % of random quartets fall in the deepest class (root as written; strata with < 5 quartets are noise). Co-presence is not informativeness (a column must also differ among the four). Next measurement: repeat on a sample drawn with the pair weight and check whether the ratio moves toward 1. + +## Two kinds of family loss: sequencing dropout vs divergence (2026-10-12) + +- **Sequencing dropout (genome skimming, ~1× coverage).** About $e^{-1}\approx 37\%$ of each genome is uncovered, independently per family. Predicted for a family truly present in a clade of $m$ genomes: completeness ≈ 0.63-0.70 and clade-closed fraction $c^m$, independent of clade depth. Consistent with small clades (3-4 tips: completeness 0.71, 20 % closed vs $0.65^3=27\%$, $0.65^4=18\%$). Does not predict the observed decay at larger clades (0.45 at 5-8 tips, 0.21 at 17-32, 0.08-0.12 beyond). +- **Loss by divergence (user observation).** A family is a kmer whose 30 flanking positions (15 each side for $k=31$) are identical, with a variable central base; with per-site divergence $d$ the flanks survive with probability ≈ $(1-d)^{30}\approx e^{-30d}$ (0.74, 0.21, 0.042, 0.0012 for $d$ = 0.01, 0.05, 0.10, 0.20). Phylogeny-dependent and tied to local conservation, unlike sequencing dropout. Explains the loss of completeness with clade size, and sets a floor: no weighting can sample families that no longer exist for pairs whose $e^{-30d}$ is negligible. Surviving deep families are selected for conserved flanks, hence plausibly slow-evolving (fewer variable sites for deep splits; hypothesis, untested). +- The patristic distances of the IQ-TREE tree (1 to 3.5) come from an alignment with variable sites only, hence strongly inflated; the true $d$ is unknown and these numbers cannot be plugged into $e^{-30d}$. +- **Test (user's plan), done 2026-10-12: see the next section.** + +## Overlap vs Mash distance on the full index (measured 2026-10-12) + +Inputs: `phylo --distance mash --csv --shared-kmers --nj` on the 180-genome index (distance matrix, shared-kmer matrix, NJ tree) against the full-index `family_overlap` $O_{ij}$. Mash $D$: range 0.011-0.337, median 0.20. + +| Regression | Slope | $R^2$ | +|---|---|---| +| $\log O_{ij}$ vs $D$, all pairs | -26.95 | 0.979 | +| $\log$(shared kmers) vs $D$ | -31.87 | 0.984 | +| $\log O_{ij}$ vs $D$ + additive genome effect | -25.3 | 0.9999 | + +Spearman($D$, $O$) = -0.95; Spearman($D$, shared kmers) = -0.95; Spearman($D$, IQ-TREE patristic) = 0.81. + +Reading: + +- Overlap decays log-linearly with distance, slope about -27 versus -30 predicted by flank survival (previous section). **Not an independent confirmation of the flank model**: Mash $D$ is derived from kmer sharing and $O$ counts families defined by kmers, so both measure the same shared content. The kmer slope of -32 is $-k$ by construction of Mash (implementation check only). The ratio of the two slopes, 27/32 ≈ 0.85, is below the 30/31 ≈ 0.97 expected from flank survival alone; cause unknown, the gap is modest. +- Local slopes vary: -21 for $D\in[0.011,0.07]$ ($R^2$ 0.45), -28 for [0.07, 0.20] (0.96), -35 for $D>0.23$ (0.83). Half the pairs lie in $D\in[0.20,0.23]$, where the fit explains almost nothing ($R^2$ 0.26 and 0.07): a plateau where Mash has little resolution. +- Genome effects (log scale, sd 0.21): lowest `Androsace_saussurei--PHA010690` (factor ≈ 0.39), `Androsace_triflora--PHA000558` (≈ 0.62), `Androsace_vesulensis--PHA010698` and `Primula_stricta_var_obesior--TROM_V_149128` (≈ 0.65); highest `Androsace_pubescens--PHA010697` (≈ 2.8). Three of the four lowest are also among the genomes with the fewest calls in the pseudo-alignment (saussurei, triflora, stricta), pubescens has the most; consistent with a coverage effect at the extremes, correlation over all genomes not computed. +- NJ on Mash vs the IQ-TREE tree: Robinson-Foulds 182 of 354 (0.51 normalized); the Mash plateau near 0.2 probably limits NJ resolution for most pairs. +- Consequence for normalization: the residual $\log O_{ij}-(a+bD_{ij})$ measures coverage deviation at a given distance, a normalization of $O$ that needs no per-genome call counts. Untested on a weighted sample. + +## Data-poor genomes (measured 2026-10-12) + +Per-genome share of the 5 003 414 sampled columns with a call: median 5.2 %, max 14.3 %; 8 genomes below 1 % and 23 below 2 %. Fewest calls: `Androsace_triflora--PHA000558` 1 936, `Primula_stricta_var_obesior--TROM_V_149128` 2 403, `Primula_carniolica--PHA007185` 4 937, `Androsace_saussurei--PHA010690` 6 986, `Primula_pumila--TROM_V_135924` 17 393, `Androsace_maxima--PHA000534` 17 939, `Primula_auricula_subsp_serratifolia--PHA007184` 30 621, `Androsace_pyrenaica--PHA000543` 37 725. Three of them (saussurei, serratifolia, pyrenaica PHA000543) are among the tips whose placement looked aberrant, previously attributed to possible mislabelling or contamination. A simpler alternative is too little data to place them reliably; testable by removing them (`--exclude-genome`) and watching deep-node support. + +Consequence for flattening: raw $O_{ij}$ mixes evolutionary distance and per-genome coverage; flattening on it chases pairs involving data-poor genomes, for which no family exists. Normalizing, $O_{ij}/(c_ic_j)$ with $c_i$ the number of families carried by genome $i$ (per-genome counts from `--sibling-stats`), targets distance instead of coverage; the best normalization (smoothing of $c_i$, etc.) is not established. + +## Open questions + +- Per-family $S_f$ without materializing the full matrix: a first pass storing only a per-family summary (reach, $|S_f|$), as `--shannon` already does a full unsampled scan for entropy. +- Combination with entropy bias: not independent criteria if entropy correlates with reach (see section above); options: compensate the known entropy weight inside the pair weight, or apply the Gaussian inside reach strata. +- Pairs with near-zero $O_{ij}$ (suspected badly sequenced genomes) set the ceiling for everyone; `--min-shared-family`/`--exclude-genome` should be applied first so the floor is not dictated by anecdotal genomes. +- Effect on the SNP-distance estimators: a non-uniform draw over pairs biases per-pair eligible-family counts, but each pair's $p$-distance is a ratio over its own eligible families, so unbiasedness per pair is expected to hold. To confirm before relying on it. diff --git a/DevDocMD/theory/sibling_partition_unification.md b/DevDocMD/theory/sibling_partition_unification.md new file mode 100644 index 00000000..f06b63b7 --- /dev/null +++ b/DevDocMD/theory/sibling_partition_unification.md @@ -0,0 +1,60 @@ +# Sibling partition unification and masked canonical form (discussion) + +Status: **open, nothing implemented**. Records the state of the reflection on (a) routing the 4 members of a sibling family to one partition and (b) a canonical form that makes siblings directly comparable. Measurements are throwaway (not in the repo); figures are for $k=31$, $m=11$, 256 partitions, 20 bacterial genomes (98.4 M kmer positions), unless stated. + +## Problem + +A sibling family = the 4 kmers differing only at the central base (position $c=(k-1)/2$, $k$ odd). Routing is **not** a hash of all bases: a kmer goes to `minimizer().seq_hash() & mask`, and consecutive kmers of a superkmer share their minimizer. A family splits across partitions whenever the central base falls in (or changes) the selected m-mer. + +Fraction of families whose 4 variants are not all in one partition, standard minimizer: $4m/(k+2m+1)$ (fixed candidates $k-2m+1$ versus $4m$ variable ones). Measured: 81.4 % at $m=11$ (60.2 % at $m=7$, 46.8 % at $m=5$). This is the *theoretical* rate assuming all 4 members present; for two present members it is ≈ 68 %. Real families are overwhelmingly of size 1, so it only becomes cost where something probes variants. + +## Masked canonical form (Can3) + +Notation: `CentreA(K)` = K with its central base set to A; `RC(K)` its reverse complement. + +- Can1 = `min(K, RC(K))` (current). Each sibling canonicalised alone, so some may be flipped relative to the others. +- Can2 = orient by `CentreA(K)` vs `RC(CentreA(K))`. **Not a valid canonical form**: when the flanks are reverse complements of each other ($L=\mathrm{rc}(R)$, frequency ≈ $4^{-c}$, 1/1024 at $k=11$), $K$ and $\mathrm{RC}(K)$ get different canonicals (k=5: `ACAGT` and `ACTGT` both kept). Cause: `CentreA(RC(K)) ≠ RC(CentreA(K))` (the latter has centre T). +- **Can3** (retained): + +``` +Can3(K) = K if CentreA(K) < CentreA(RC(K)) + = RC(K) if CentreA(K) > CentreA(RC(K)) + = Can1(K) otherwise // self-revcomp family +``` + +Orientation is decided on the flanks only (centre ignored), tie broken on the full kmer. Gain: the 4 canonical siblings have identical flanks in the same orientation, so they compare directly with no reverse complement. Self-revcomp families have only 2 distinct canonical members (centres A/C; T/G fold onto them). The A↔T (C↔G) substitution in that context is undetectable by any double-strand canonical set, same as under Can1 — not a loss. + +Gain is simplification only: `central_canonical_neighbors` already canonicalises each variant independently and is correct (see `evolutionary_distances.md`, per-partition sweep). The earlier "masked-flank grouping" there is the Can2 idea; Can3 is its repaired form. **No performance gain by itself**, and it breaks existing indexes (canonical orientation changes for some kmers → MPHF hashes change; rebuild). + +Prerequisite if adopted: `Minimizer = CanonicalKmerOf` shares the canonical type with index kmers; an m-mer has no sibling family and must keep Can1. Decouple with a distinct `Minimizer` type (6 files outside `kmer.rs`: `routable.rs`, `packed_seq.rs`, `obiskbuilder/{rolling_stat,stream_iter,iter}.rs`, `obifastwrite`). + +## Routing that excludes the centre (scheme B) — rejected + +Candidates restricted to m-mers not overlapping the centre ($m\le c$): $S(j)=\min$ hash over m-mers $p\in[j,j+c-m]$, minimizer$(i)=\min(S(i),S(i+c+1))$. Family- and strand-invariant by construction (checked). But unitigs are partition-local (`pipeline.md`), and the hole moves with each kmer, so runs shrink: + +| $m$ | mean run, current | mean run, B | bases/kmer, current → B | +|---|---|---|---| +| 5 | 14.9 | 6.3 | 3.02 → 5.76 | +| 11 | 11.0 | 3.0 | 3.72 → 10.96 | +| 15 | 9.0 | 1.0 | 4.32 → 30.9 | + +Proxy: run = unitig (upper bound; real unitigs may merge). Independent of partition count. In `benchmark/run/global_index_count` (786 MB) `unitigs.bin` + idx ≈ 241 MB (31 %), so tripling them ≈ +60 % total index. Hashing the masked kmer per kmer (no minimizer) would cut every path at every kmer: worse. + +Structural tension: path coherence (long unitigs) needs a key shared by consecutive kmers, family coherence needs a key independent of each kmer's own centre, which is an ordinary base for its neighbour. Decoupling sequence storage from MPHF partitioning fails because a partition can only enumerate its kmers from its own unitigs (MPHF not invertible). Redefining the family on a terminal base would cut the split rate to ≈ 17 % but is not closed under revcomp and changes the SNP semantics. Lower $m$ reduces the split rate for free but deviates from the literature $k/m\approx 2.5$ optimum (balance, bucket size), unmeasured here — $m=11$ kept. + +## What the cross-partition cost actually is + +- Phylo-time: only families of size > 1 trigger remote lookups (a small share). +- Annex construction probes the 3 variants of **every** kmer (size-1 families included), ≈ 2/3 of probes crossing partitions; batch bounded to `#kmers(p)*3` per partition (not the whole index). +- Real remaining benefit of unification: the outer sweep over source partitions could run in parallel (it is sequential today for memory and for the `q >= p` dedup rule); plus code simplification in `family_scan.rs` / `annex.rs`. Not quantified. +- Unification would also allow an MPHF keyed on the family (one lookup, 4-bit member mask, union/intersection as OR/AND); needs the family-size histogram (1–4 members) to judge storage. + +## Open measurements (index not on this machine) + +1. Family-size histogram of the real index: `obikmer phylo --sibling-hist` (needs a built annex). +2. Time and peak RSS of `--sibling-annex` on the Vaccinium/Salix index (`Stage` "sibling_annex" traces time; peak RSS from outside). +3. Only if those show the sequential sweep or the probe volume is a real bottleneck: real-index test of scheme B or of a smaller $m$. + +## Decision state + +No code change decided. Can3 + `Minimizer` decoupling is the only self-contained step and is independent of the measurements above; it forces an index rebuild, so it is worth doing only if its simplification is wanted. diff --git a/UserDocMD/molecular-ecology-modified.csl b/UserDocMD/molecular-ecology-modified.csl index ed2b1ce9..d8551650 100644 --- a/UserDocMD/molecular-ecology-modified.csl +++ b/UserDocMD/molecular-ecology-modified.csl @@ -71,6 +71,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -85,9 +133,7 @@ - - - + + + + + + + + + + + + + @@ -150,6 +208,14 @@ + + + + + + + + diff --git a/UserDocMD/references.bib b/UserDocMD/references.bib index f547a3f4..23ca6fe4 100644 --- a/UserDocMD/references.bib +++ b/UserDocMD/references.bib @@ -768,3 +768,92 @@ pages = {535--563}, year = {1988} } + @ARTICLE{Le-Cam1960-pe, + title = "{An approximation theorem for the Poisson binomial distribution}", + author = "Le Cam, Lucien", + journal = "Pacific Journal of Mathematics", + publisher = "Mathematical Sciences Publishers", + volume = 10, + number = 4, + pages = "1181--1197", + month = dec, + year = 1960, + url = "http://dx.doi.org/10.2140/pjm.1960.10.1181", + doi = "10.2140/pjm.1960.10.1181", + issn = "0030-8730,1945-5844" + } + + @ARTICLE{Hoeffding1956-bx, + title = "{On the distribution of the number of successes in independent + trials}", + author = "Hoeffding, Wassily", + journal = "The Annals of Mathematical Statistics", + publisher = "Institute of Mathematical Statistics", + volume = 27, + number = 3, + pages = "713--721", + abstract = "Let $S$ be the number of successes in $n$ independent trials, and + let $p_j$ denote the probability of success in the $j$th trial, + $j = 1, 2, \cdots, n$ (Poisson trials). We consider the problem + of finding the maximum and the minimum of $Eg(S),$ the expected + value of a given real-valued function of $S,$ when $ES = np$ is + fixed. It is well known that the maximum of the variance of $S$ + is attained when $p_1 = p_2 = \cdots = p_n = p.$ This can be + interpreted as showing that the variability in the number of + successes is highest when the successes are equally probable + (Bernoulli trials). This interpretation is further supported by + the following two theorems, proved in this paper. If $b$ and $c$ + are two integers, $0 \leqq b \leqq np \leqq c \leqq n,$ the + probability $P(b \leqq S \leqq c)$ attains its minimum if and + only if $p_1 = p_2 = \cdots = p_n = p,$ unless $b = 0$ and $c = + n$ (Theorem 5, a corollary of Theorem 4, which gives the maximum + and the minimum of $P(S \leqq c)$). If $g$ is a strictly convex + function, $Eg(S)$ attains its maximum if and only if $p_1 = p_2 = + \cdots = p_n = p$ (Theorem 3). These results are obtained with + the help of two theorems concerning the extrema of the expected + value of an arbitrary function $g(S)$ under the condition $ES = + np.$ Theorem 1 gives necessary conditions for the maximum and the + minimum of $Eg(S).$ Theorem 2 gives a partial characterization of + the set of points at which an extremum is attained. Corollary 2.1 + states that the maximum and the minimum are attained when $p_1, + p_2, \cdots, p_n$ take on, at most, three different values, only + one of which is distinct from 0 and 1. Applications of Theorems 3 + and 5 to problems of estimation and testing are pointed out in + Section 5.", + month = sep, + year = 1956, + url = "http://dx.doi.org/10.1214/aoms/1177728178", + doi = "10.1214/aoms/1177728178", + issn = "0003-4851,2168-8990", + language = "en" + } + @ARTICLE{Gleser1975-yu, + title = "{On the distribution of the number of successes in independent + trials}", + author = "Gleser, Leon Jay", + journal = "Annals of Probability", + publisher = "Institute of Mathematical Statistics", + volume = 3, + number = 1, + pages = "182--188", + abstract = "Let $S$ be the number of successes in $n$ independent Bernoulli + trials, where $p_j$ is the probability of success on the $j$th + trial. Let $\mathbf{p} = (p_1, p_2, \cdots, p_n)$, and for any + integer $c, 0 \leqq c \leqq n$, let $H(c \mid \mathbf{p}) = P\{S + \leqq c\}$. Let $\mathbf{p}^{(1)}$ be one possible choice of + $\mathbf{p}$ for which $E(S) = \lambda$. For any $n \times n$ + doubly stochastic matrix $\Pi$, let $\mathbf{p}^{(2)} = + \mathbf{p}^{(1)}\Pi$. Then in the present paper it is shown that + $H(c \mid \mathbf{p}^{(1)}) \leqq H(c \mid \mathbf{p}^{(2)})$ for + $0 \leqq c \leqq \lbrack\lambda - 2\rbrack$, and $H(c \mid + \mathbf{p}^{(1)}) \geqq H(c \mid \mathbf{p}^{(2)})$ for + $\lbrack\lambda + 2\rbrack \leqq c \leqq n$. These results + provide a refinement of inequalities for $H(c \mid \mathbf{p})$ + obtained by Hoeffding [3]. Their derivation is achieved by + applying consequences of the partial ordering of majorization.", + month = feb, + year = 1975, + url = "http://dx.doi.org/10.2307/2959277", + doi = "10.1214/aop/1176996461", + issn = "0091-1798,2168-894X" + } diff --git a/UserDocMD/theory/kmer_indexing/kmers.md b/UserDocMD/theory/kmer_indexing/kmers.md index 36befb07..5ce36ac2 100644 --- a/UserDocMD/theory/kmer_indexing/kmers.md +++ b/UserDocMD/theory/kmer_indexing/kmers.md @@ -22,24 +22,102 @@ This gives the hard representation limit: $k \leq 32.$ Furthermore, OBIKmer only ### K-mer specificity -A simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are +One simple way to illustrate the effect of k-mer size is to consider a random DNA sequence in which the four nucleotides are independent and equally probable. There are $4^k$ possible k-mers, so a particular k-mer is expected to occur once every $4^k$ positions on average. In a sequence of length $G$, the expected number of occurrences of a particular k-mer is approximately $G \cdot 4^{-k}$. When k is large enough to ensure $4^k >> G$, the chance of a second occurrence of the same k-mer becomes close to zero. Consequently, observing the same k-mer twice can only be explained by a homology relationship, except for low-complexity k-mers, which are overrepresented in genomes. + +### Estimating False discovery rate on homology inference + +OBIKmer uses k-mers long enough that their occurrence in multiple indexed genomes provides evidence of homology. The corresponding false discovery rate (FDR) can be expressed in terms of the probability of observing a k-mer in at least $R$ of the $S$ indexed genomes under a purely random model. + +#### The random model + +The random model, or null hypothesis, assumes that the $S$ genomes are unrelated and that each genome is an independent random sequence in which each nucleotide occurs with equal probability. Rejecting this hypothesis for a given k-mer indicates that its presence in multiple genomes is unlikely to be explained by chance alone, providing evidence for homology. + +For a genome of size $G$, the number of occurrences of a given canonical k-mer $K$ is approximated by a Poisson distribution with parameter + +$$ \lambda = 2\frac{G}{4^k}, $$ + +which is also the expected number of occurrences under the random model. The factor of two accounts for k-mer canonicalisation, which halves the k-mer space when $k$ is odd. + +The probability of observing at least one occurrence of this k-mer is therefore $$ -4^k +\begin{aligned} +P(n_K > 0) &= 1-e^{-\lambda} \\ +&= 1-e^{-2G/4^k}. +\end{aligned} $$ -possible k-mers, and a particular k-mer is therefore expected to occur once every $4^k$ positions on average. In a sequence of length $G$, the expected number of occurrences of a particular k-mer is approximately +Assuming equal genome lengths, the number of genomes containing at least one occurrence of $K$ among the $S$ sequences follows a binomial distribution. The probability of observing $K$ in at least $r$ genomes is therefore + $$ -E[n_K] \simeq \frac{G}{4^k}. +P(X \geq r)=\sum_{j=r}^{S}\binom{S}{j}\left(1-e^{-2G/4^k}\right)^j\left(e^{-2G/4^k}\right){S-j}. $$ -When $4^k\gg G$, this quantity is also a good approximation of the probability that an observed k-mer will occur at least once by chance elsewhere in a sequence of length $G$. It therefore provides a simple measure of the risk of a chance recurrence of an observed k-mer: +This sum is the regularized incomplete beta function $I_p(r,\,S-r+1)$, where $$ -P_{\mathrm{chance}}(k,G) \simeq \frac{G}{4^k}. +p = 1-e^{-2G/4^k}. $$ +Different genome sizes + +When genome sizes differ, the probability of observing a given k-mer also differs between genomes. A simple approximation is to replace the individual genome sizes $G_i$ by their mean + +$$ \bar G = \frac{1}{S}\sum_{i=1}^{S}G_i. $$ + +This gives a common probability + +$$ \bar p = 1-e^{-2\bar G/4^k}, $$ + +and the number of genomes containing the k-mer can then be approximated by a binomial distribution: + +$$ P(X\geq r)=\sum_{j=r}^{S}\binom{S}{j}\bar p^j(1-\bar p)^{S-j}. $$ + +This approximation is conservative for the upper tail. Hoeffding [-@Hoeffding1956-bx, p. 717, Eq. 26] shows that, for a fixed mean, the binomial upper tail is at least as large as the corresponding Poisson-binomial tail for + +$$ r\geq S\bar p+1. $$ + +Using the mean genome size therefore provides a conservative estimate of the FDR. + +When genome sizes differ substantially, however, this approximation can become overly conservative. In that case, the exact Poisson-binomial distribution can be computed directly from the individual genome-specific probabilities $p_i$. + +### Exact tail probability + +With genome-specific probabilities $p_i$, the number of genomes containing the k-mer follows a Poisson-binomial distribution. Writing the probability of exactly $j$ presences as a sum over all subsets of $j$ genomes would be impractical, since there can be up to $2^S$ subsets. The distribution can instead be obtained by adding the genomes one at a time. + +Let $d_j$ be the probability that exactly $j$ of the genomes processed so far contain the k-mer. Adding genome $i$ leaves $j$ unchanged if the genome does not contain the k-mer, and moves $j-1$ to $j$ if it does: + +$$ +\left \{ +\begin{aligned} +d'_j &= d_j(1-p_i) + d_{j-1}p_i, \\ +d_0&=1 +\end{aligned} +\right . +$$ + +before processing the first genome. + +The table $d$ only needs to record how many genomes contain the k-mer, not which ones, reducing the computational cost to $O(S^2)$. The result is independent of the order in which the genomes are processed and of the k-mer itself, so it can be computed once per index. The upper tail is then + +$$ +P(X\geq r)=1-\sum_{j=0}^{r-1}d_j. +$$ + +Only $d_0,\ldots,d_{r-1}$ are needed, reducing the cost to $O(Sr)$. + +When all $p_i$ are small, as is typically the case for large $k$, the Poisson-binomial distribution can also be approximated by a Poisson distribution with mean + +$$ +\lambda=\sum_{i=1}^{S}p_i. +$$ + +The approximation error is bounded in terms of $\sum_i p_i^2$ by Le Cam's inequality [@LeCam1960-pe]. + + +******* + This probability decreases by a factor of four for every additional nucleotide in the k-mer. Conversely, it increases linearly with the amount of sequence searched. This simple model already shows why very short k-mers provide limited specificity for genome-scale analyses. With $k=11$, @@ -189,34 +267,150 @@ Le rapport est exactement $S_1 / P(R=2\mid w) = 4^{k}$ — c'est-à-dire le nomb Une seule réserve sur le texte proposé tel quel : $p_i\simeq1-\exp(-G_i/4^k)$ souffre du même biais que celui identifié plus tôt dans notre discussion — un mot de longueur $k$ n'est pas indépendant à chaque position (chevauchement), donc ce $p_i$ est un peu optimiste (légèrement trop haut) pour les mêmes raisons que $E[C]$ l'était. Négligeable pour $k\geq21$, plus discutable à $k=11$. -## FDR +## Chance sharing across genomes and the FDR of homology -C'est exactement la bonne question à se poser — ni la p-value par hit, ni la probabilité globale « y a-t-il au moins une collision quelque part », mais **la fraction de tes appels d'homologie qui sont du bruit**. C'est un problème de type FDR (*false discovery rate*), et il se construit directement à partir des deux briques qu'on a déjà : +The formulas above ask how likely a match is between two sequences. The question OBIKmer actually has to answer is different: a k-mer is found in $r$ of the $S$ indexed genomes, and one wants to know whether this sharing can be explained by chance or whether it is evidence of homology. This section builds a null model for that question and turns it into a false discovery rate (FDR) for the call "a k-mer shared by at least $r$ genomes is homologous". + +### The null model + +The null hypothesis is that the $S$ genomes are unrelated, that is, that each one is an independent random sequence. Rejecting it for a k-mer means that chance alone cannot explain why the k-mer is shared, which is the signal of homology one is looking for. + +The model only records whether a k-mer is present in a genome, not how many times. Two consequences follow. First, repeats inside a genome are irrelevant, and the size $G_i$ of genome $i$ is its number of **distinct canonical k-mers**. Second, a repeat family shared between genomes is paralogy, which is a form of homology; it is a true positive and not a false one. The only k-mers that could be shared by composition rather than by descent are low-complexity ones, which OBIKmer removes beforehand with its entropy filter. + +Everything takes place in canonical space. Because $k$ is odd, no k-mer equals its own reverse complement, so the $4^k$ k-mers pair up perfectly and the space contains exactly $$ -\widehat{FDR}(r) = \frac{\text{nombre de k-mers *attendus par hasard* avec }R\geq r}{\text{nombre de k-mers *observés* avec }R\geq r} +N=\frac{4^k}{2} $$ -**Le numérateur** (bruit attendu) : c'est la généralisation, au seuil $\geq r$ (pas seulement « partagé par exactement ces $r$ génomes »), de la quantité pont qu'on vient d'identifier — le nombre de k-mers testés $M$ multiplié par la probabilité par-mot sous le null : +canonical k-mers. A given canonical k-mer is present in genome $i$ by chance with probability $$ -E[\text{faux positifs}] \simeq M \cdot P_0(R\geq r) = M\sum_{j=r}^{S}\binom{S}{j}p^{j}(1-p)^{S-j} +p_i = 1-\left(1-\frac1N\right)^{G_i}\simeq \frac{G_i}{N}=\frac{2\,G_i}{4^k}. $$ -avec $p\simeq G/4^k$ (ou la version Poisson-binomiale à $G_i$ inégaux) et $M$ le nombre de k-mers canoniques réellement testés (pas $4^k/2$ théorique, mais le nombre de k-mers **effectivement présents** dans ton index — c'est cette correction, plutôt que le $4^k$ théorique, qui rend l'estimation réaliste, puisque la plupart des $4^k$ mots possibles n'apparaissent jamais dans de vrais génomes). +This is twice the $G/4^k$ used earlier in this page, because there $G$ is a number of positions on one strand, whereas $G_i$ here counts canonical k-mers, for which both strands are already merged. -**Le dénominateur** (bruit + vrai signal, observé) : le nombre de k-mers de ton index dont le masque de présence a une cardinalité $\geq r$ — une donnée que `FamilyMask`/`SiblingAnnex` dans `obikphylo::siblings` te donne déjà directement, sans aucun calcul supplémentaire. +Let $R$ be the number of genomes, out of $S$, that contain a given canonical k-mer by chance. Genomes are independent under the null, so $R$ is a sum of independent presence indicators, each with its own probability $p_i$. -**Ce que ça donne concrètement.** Tu n'as besoin que de trois nombres, tous déjà disponibles ou calculables : -1. $M$ = cardinalité totale de ton set de k-mers indexés (mesure directe de l'index) -2. $N_{obs}(r)$ = nombre de familles/k-mers avec présence $\geq r$ (mesure directe, déjà dans `FamilyMask`) -3. $P_0(R\geq r)$ = calculé depuis le modèle nul (le $p$ de l'approche « naïve » du tour précédent) +### Exact tail probability -et $\widehat{FDR}(r) = M\cdot P_0(R\geq r) / N_{obs}(r)$ te dit directement : « sur les k-mers que j'ai retenus comme preuve d'homologie au seuil $r$, telle fraction (%) est attendue être du pur hasard ». +When genome sizes differ, the $p_i$ differ, and $R$ follows a Poisson-binomial distribution. Writing the probability of exactly $j$ presences as a sum over all subsets of $j$ genomes would be hopeless, since there are up to $2^S$ subsets. The distribution is instead obtained by adding the genomes one at a time. Let $d_j$ be the probability that exactly $j$ of the genomes processed so far contain the k-mer. Adding genome $i$ leaves $j$ unchanged if the genome does not contain the k-mer, and moves $j-1$ to $j$ if it does: -**Pourquoi c'est mieux que ce qu'on avait avant** : -- Le $P(K_{r,s}\geq k)$ de Karlin-Ost (tour précédent) répond à « y a-t-il *au moins un* faux positif quelque part » — binaire, ne dit rien sur combien. -- La p-value par hit répond à « ce hit précis est-il suspect » — mot par mot, ne donne pas de vue d'ensemble. -- Le $\widehat{FDR}$ répond directement à ta question : **quelle proportion** de l'ensemble retenu est due au hasard — c'est la quantité qu'on utiliserait pour dire « à ce seuil $r$, mon jeu de zones d'homologie candidates est fiable à 95 % » ou pour choisir le $r$ minimal qui ramène le $\widehat{FDR}$ sous un seuil acceptable, à $k$ fixé par la contrainte de représentation (64 bits). +$$ +d'_j = d_j\,(1-p_i) + d_{j-1}\,p_i, \qquad d_0=1 \text{ before the first genome.} +$$ -Un point à trancher avec toi avant d'aller plus loin : pour $M$, préfères-tu le nombre théorique de k-mers canoniques possibles ($4^k/2$) ou le nombre réel de k-mers distincts dans ton index (qui donnerait un $\widehat{FDR}$ beaucoup plus conservateur et réaliste, mais nécessite de le lire depuis l'index au moment du calcul plutôt que de le dériver de $k$ seul) ? +The table $d$ only needs to remember how many genomes were hit, not which ones, and this brings the cost down to $O(S^2)$. The result does not depend on the order in which the genomes are processed. It does not depend on the k-mer either, so it is computed once per index. The upper tail is then + +$$ +P_0(R\geq r) = 1-\sum_{j=0}^{r-1} d_j . +$$ + +Only $d_0,\dots,d_{r-1}$ are needed, so the cost is $O(S\,r)$. + +### Approximation with equal genome sizes + +If all genomes are given the same size, taken as the mean + +$$ +\bar G=\frac1S\sum_{i=1}^{S}G_i, \qquad \bar p=\frac{2\bar G}{4^k}, +$$ + +then $R$ follows a plain binomial distribution and the tail is + +$$ +P_0(R\geq r)\simeq\sum_{j=r}^{S}\binom{S}{j}\,\bar p^{\,j}(1-\bar p)^{S-j}. +$$ + +This sum is the regularized incomplete beta function $I_{\bar p}(r,\,S-r+1)$, available in standard statistical libraries. If it is summed by hand, the terms are obtained one from the next by $t_{j+1}=t_j\,\frac{S-j}{j+1}\,\frac{\bar p}{1-\bar p}$. + +A theorem of Hoeffding [@Hoeffding1956-bx] shows that this approximation errs on the safe side. For a fixed mean, the Poisson-binomial distribution is more concentrated than the binomial one, so for $r\geq S\bar p+1$ the binomial tail is never smaller than the exact tail. Using the mean genome size therefore overestimates the chance of sharing and can only make the FDR more pessimistic. Gleser [@Gleser1975-yu] refines this result: replacing the $p_i$ by any partial average of themselves, not only by their full mean, also moves the upper tail towards the binomial one, provided $r-1\geq S\bar p+2$. Grouping genomes of similar size is therefore safe in the same sense. What it loses is power when genome sizes differ by orders of magnitude, since one large genome then weighs as much as several small ones in $\bar p$. + +When all $p_i$ are small, which is the case for large $k$, the number of hits is also well approximated by a Poisson distribution of mean $\lambda=\sum_i p_i$, with an error bounded by $\sum_i p_i^2$ [@Le-Cam1960-pe]. + +For the numerical evaluation, the tail that does not contain the mode $S\bar p$ should be summed directly. When $r>S\bar p$, computing the sum from $j=r$ upwards is accurate even for very small probabilities, whereas computing $1$ minus the sum of the lower terms loses all precision once the tail falls below the machine epsilon. + +### False discovery rate + +The FDR of the call "present in at least $r$ genomes" is the fraction of retained k-mers expected to be pure chance: + +$$ +\widehat{\mathrm{FDR}}(r)=\min\!\left(1,\ \frac{N\,P_0(R\geq r)}{n_{\geq r}}\right). +$$ + +The numerator is the number of k-mers expected to reach $r$ genomes by chance if the genomes were unrelated. Each of the $N$ canonical k-mers has probability $P_0(R\geq r)$ of doing so, and no further correction for multiple testing is needed because the number of tests is already in the product. The denominator $n_{\geq r}$ is the number of distinct k-mers actually observed in at least $r$ genomes, which is read directly from the index. The observed count mixes chance and true homology, so the ratio is an upper bound on the true FDR: it implicitly assumes that all k-mers are null, and it never underestimates the noise. The minimal $r$ for which $\widehat{\mathrm{FDR}}(r)$ falls below the accepted level, for example 5 %, is the sharing threshold to use. + +As an illustration, take $k=21$ (so $N=2.2\times10^{12}$), $S=10$ genomes and $G_i=10^9$ distinct canonical k-mers each, hence $p=4.5\times10^{-4}$. The number of k-mers expected to be shared by chance is: + +| $r$ | $N\,P_0(R\geq r)$ | +| ---: | ---: | +| 2 | $2\times10^{7}$ | +| 3 | $2.5\times10^{4}$ | +| 4 | $\approx 20$ | +| 5 | $\approx 0.01$ | + +Chance alone produces millions of pairwise shared 21-mers at this genome size, a few tens of thousands shared by three genomes, and almost none shared by five or more. Whether a given $r$ is acceptable then depends on how many k-mers the index actually reports at that level. + +### Limits of the model + +- The FDR is computed per k-mer. Neighbouring k-mers of a shared region overlap and are not independent, so a single homologous region contributes many correlated k-mers. A per-region FDR would count regions instead of k-mers and is not covered here. +- The probability $p_i$ assumes equiprobable, independent nucleotides. The composition biases discussed earlier in this page violate this assumption, and the resulting error is not accounted for. + +--- + +Je n'ai rien modifié. Voici la formalisation des deux points, à valider avant de l'écrire dans `kmers.md`. + +## 1. FDR d'un ensemble de k-mers sélectionnés + +**Estimation de l'espérance du nombre de faux positifs.** Soit $M$ le nombre de k-mers canoniques testés, $M = 4^k/2$ pour k impair. Soit $q(r)=P(X\ge r)$ la queue du modèle nul. Le nombre attendu de k-mers présents par hasard dans au moins $r$ génomes vaut + +$$ +E[\mathrm{FP}(r)] = M\,q(r). +$$ + +Cette égalité est exacte par linéarité de l'espérance, même si les k-mers chevauchants sont dépendants. La dépendance n'affecte que la variance, pas l'espérance. Seule l'approximation de Poisson par génome reste une approximation. + +**FDR.** Soit $N(r)=\#\{i : r_i\ge r\}$ le nombre observé de k-mers couvrant au moins $r$ génomes. Alors + +$$ +\mathrm{FDR}(r)=\min\!\Big(1,\ \frac{M\,q(r)}{N(r)}\Big). +$$ + +Cette estimation est légèrement conservatrice. Elle suppose $\pi_0=1$, c'est-à-dire que presque tout l'espace $M$ est nul, ce qui est vrai pour k grand. + +**Ensemble hétérogène en $r_i$.** Si le jeu n'est pas défini par un seuil unique, on utilise le local fdr par classe de couverture : + +$$ +\mathrm{fdr}(r)=\frac{M\,P(X=r)}{N_r},\qquad N_r=\#\{i : r_i=r\}. +$$ + +Le FDR de l'ensemble est alors la moyenne $\frac1N\sum_i \mathrm{fdr}(r_i)$. Pour un ensemble défini par seuil $r_{\min}$, cela redonne exactement la formule précédente. + +**Remarque.** $M\,p\approx G$, d'où $E[\mathrm{FP}(r)]\approx\binom{S}{r}\,G^r/M^{r-1}$ en régime de petits $p$. Cela donne une lecture directe de la quantité. + +## 2. Markov d'ordre 0 avec fréquences inégales + +Soit $\pi_b$ la fréquence du nucléotide $b$. Pour un k-mer $w$ de composition $n=(n_A,n_C,n_G,n_T)$ avec $\sum n_b=k$, on a $P(w)=\prod_b \pi_b^{n_b}$. + +**Attendu par génome.** Le k-mer canonique $K=\{w,\overline w\}$ est vu quand le brin direct porte $w$ ou $\overline w$, donc + +$$ +\lambda_K = G\,\big(P(w)+P(\overline w)\big)=G\Big(\prod_b\pi_b^{n_b}+\pi_A^{n_T}\pi_T^{n_A}\pi_C^{n_G}\pi_G^{n_C}\Big). +$$ + +Cela ne dépend que de la composition. Si $\pi_A=\pi_T$ et $\pi_C=\pi_G$, on retrouve $2P(w)$. + +**Faux positifs.** Le test n'est plus homogène : chaque composition a sa propre $p(n)=1-e^{-\lambda(n)}$. Il faut donc sommer sur les classes de composition (un k-mer et son complément inverse partagent le même $\lambda$, d'où le facteur ½) : + +$$ +E[\mathrm{FP}(r)]=\frac12\sum_{n}\binom{k}{n_A,n_C,n_G,n_T}\,P\big(X\ge r\mid p(n)\big). +$$ + +Cette somme porte sur $\binom{k+3}{3}$ compositions, soit 5 984 pour k = 31. Elle est donc légère avec la binomiale de la moyenne, ou avec une Poisson-binomiale par composition si les $G_i$ diffèrent. Si les $\pi_b$ varient d'un génome à l'autre (GC différent), les $p_i(n)$ se construisent par génome. + +**Effet attendu.** La moyenne de $\lambda_w$ sur tous les k-mers reste $2G/4^k$, car $\sum_w P(w)=1$. Dans le régime de petits $p$, le terme dominant $\binom{S}{r}p^r$ est convexe en $\lambda$. Par Jensen, le modèle équiprobable est donc le minimum de $E[\mathrm{FP}]$, et la déviation de composition ne peut qu'augmenter le FDR. Les k-mers riches en AT ou GC dominent la somme, ce qui rejoint ton remarque sur la faible complexité. C'est un résultat sur le terme dominant, pas une preuve pour la queue exacte. + +**Limites à signaler dans le texte.** Markov 0 ignore les dinucléotides, comme les CpG ou les répétitions, et le $M$ du modèle nul reste celui de l'espace total. + +Veux-tu que j'écrive ces deux sections, avec les corrections de numérique évoquées plus haut (`expm1`, somme directe de la queue) ? diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index 6041c20e..f4b188ed 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -111,6 +111,7 @@ Without `--subsample`, every variable family (family size ≥ 2) is used. With ` | Column | Meaning | |---|---| +| `partition` | the index partition that holds the family | | `layer` | an internal index-layer identifier — stable within one run, not meaningful across indexes | | `family_idx` | the family's position within that layer | | `entropy15` | Shannon entropy (bits) over the 16 possible states (the 15 non-empty subsets of `{A,C,G,T}`), genomes absent from the family excluded from the count | @@ -118,7 +119,7 @@ Without `--subsample`, every variable family (family size ≥ 2) is used. With ` | `family_size` | number of distinct central bases observed anywhere in the index for this family (2-4) | | `n_genomes_present` | how many genomes the entropy was computed over | -Run with `--subsample N --shannon` to get a bounded diagnostic sample instead of a full-index pass — useful for choosing `--entropy`/`--entropy-sd` values before a full run. +`--shannon` always scans every non-monomorphic family of the whole index; `--subsample`, `--entropy` and `--entropy-sd` have no effect on it. The output therefore grows with the index, with one row per family, and can reach hundreds of millions of rows (tens of gigabytes) on a large index. Analyse it with a tool that reads the file as a stream (for example DuckDB or Miller) rather than loading it whole. It is nevertheless the right basis for choosing `--entropy`/`--entropy-sd` values, since it gives the complete entropy distribution, including how it varies with `n_genomes_present`, rather than an estimate from a sample. ### `--entropy MU` / `--entropy-sd SIGMA`: biasing the sample toward informative families diff --git a/mkdocs.yml b/mkdocs.yml index 6f493d68..ee06c0fe 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,7 +36,10 @@ nav: - Entropy filter: theory/entropy.md - Minimizer selection: theory/minimizer.md - Partitioning architecture: theory/indexing.md + - Chance sharing across genomes (discussion): theory/chance_sharing.md - Central-position SNP distance (discussion): theory/evolutionary_distances.md + - Sibling partition unification (discussion): theory/sibling_partition_unification.md + - Pair-coverage-biased family sampling (discussion): theory/pair_coverage_sampling.md - Implementation: - SuperKmer: implementation/superkmer.md - Kmer: implementation/kmer.md diff --git a/scripts/__pycache__/pseudo_alignment_nesting.cpython-314.pyc b/scripts/__pycache__/pseudo_alignment_nesting.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5375ee02267cc7abc1c27b39a93ba4bb6b2c344 GIT binary patch literal 29161 zcmcJ133wYvmR>h5;v@<162+@YN+coi5bw*9Oi`i^QWOPJP1&YMgCt17A_=GgN+Ns= zXEWIWWqSn4)(F}eub>$_g0uDx{beVncXmI^yE8kQ>`WFk2#W(NSx@3G$>jKATTUEL z@+JAJ8x1rl=-Az)B{sUds$RXSdiCC`SFhgobc2?{_3E)-K2y4fqW&#;5ieQn;qD9` z7AXtmrba0XeUNt3{IAR{s7cgYNS)A?QcbaHN&vf@|1)c?XOY}vowd&_Fpe`eZpP|ltaFph#Ef;)Rx>f< zaLsczrmAet=5pKTrpwHXV~VjdgZ%@H(`B2Vbkta9?9+3zwmEmLR=dwSagvFDU?vF%u`P zoOQx&;|QEN2eQmD)^V3@Zoe7 zoSbdSevX;8p8?SE1*XPf?$9#KW5V}cV*+Ypu@0!6aGha9BvDGhcGl|V?B_a|x%t^~ zAcJlYFm*9Nm9qid1Fvz1`(z9h+JuHL#F`E4?bd0>oOOoT$?T|RT-Mo#0-JEmp(Sp+ z16|=_o_37ePt5`+=gEZ`2UAlcG^ZxkoTtswrp5+RXk!dxC&~%~?y`H(t8UKfo}cYt zPCCxwyTfIhWZWlX1Bmf@SW~T_nEAO$041f|oXw`y_6zjG%-XDkqB@u<2Zs(2Fy;Hh zj#@A@q)4pCT{br}Sk27fr@Ly<%!~urNt@O+3t|B{9&(233}C$$=*+nijawizmLPp#RAs}YSP(Hn8G;|5 zBZjmKL~8D2W*ld=JX{xP;v6%{Ih>ApH-lC>t?r4F3j{@%jXPuIN6kLxvQP5O5rBbM zhhlw2P~aR6H!8>2p5^RrG=#Cw@iq6`&e|tVF;n&#(#7Zn7ls@4OmmKTCn?F3xVB0J zfKCF~Ih%d@Cd|Z>_L)i2Iiw%~!%rk9!5S1g zW1m7{()m?2HRPp+yqG(fNry0&WI9g}!XrarWjI0$POF_`s=BK8n86!d4lSQ9OkYpvrkj8GBy2Ed)dIAEIbZHP@Mb1gII z#yEl_PPnQ`DrxB4q+=FM=Q*BhejL*nG)TD25J-@x2_7pj5qt-VQ_PXEu07qLO!6z` z9XpwUs&3{vhV&T9_o2LNewLXV>teRW3z08;LH<*>w0spAm`7t)2Pk93~r*v^5+&n(n3ya<^Ca$3&joUu<>-5^6-jmtJ; zn*b~4sS-q07aytkcusK`03tCQXXiv&hT(BJNFl8NfN)GF=a{ff60wH)f>{VY3h~3Z z`FxB6q6MuWgKi@tM5O%F^Pn4WB9ZuG?+N-x!>4AfWU>;{m(mL%TwoDP)rye#X<(hU z?Zy<2eRDDWBCCmiAv0)_v!YG&b9Qje*_nk-o_iAFujbo!ACZJ=XOI)qUKqj+a)+=w zGA3wiy!S0OrvuHL)Dn>plLI0%h+%{g135vMRq+ECBa(+OWikR%Ou#V&dYHs4;_Tz| zguh^{AnSR~JU_?x4p0;Op_(ThVfc|bhLpCc=B$t|4{9e)#bJ_LzaE}dJgvOgGvIzBw9*bx;`c0?0sE(_Vt}$@QGJ_ zh^Di`(b{lAS~k{kJb{veVn;F!J8+#y${Xk8BqRSK>Nb4l9ehpv zcb=2(sI5;FmvSCboXp)C%v~2M;>$ssn8f60Cbg0bE*plm9|-hMh>9;!Ow;furzpi; z3%**@tZJ*qs$HZ;VoEDT4M=mxzLr9b9Z`r`EV9;E9!DBWXHq>zigL;<^1-wb0-e8D zSraWhRp6&Li=s4CF;$Cy7R9{Wp>}8;8a9npvl>?0s36MqjM^!;D4dEBjZluFSjwVm ziq+IfkJ0n8dHGmey>=>DCF=eb>Q=F;0i9SX@oPyFaj1-aoa`e?I;*gu4*beX@hs*k zp$63hhD2%ciB(zD)tkjP^iM|OS$q;z{!uC^C;DaMCFV@jT1z<uy_ZshS`o)VO{2V=VyNwu*pJf~l|srcE1`-| z2l?06+$k@j?6g_q>9*M1^V}R$HDevO%|J9btrJAN#|7Frksf5W3;Yr+M_k)L37wmE zpETG0mIC#>Nk^5F?ggi9sDC0xOgxH&$VjK$J&CW6DIS4%sR1QlGU>)rVgn^zp-Rfa zzgTFI+@18GVhSWuNJ)s47NIy4hj*p=KB#Cri}hh0Uyfg$60igMduu|c%PmSN3F#?| zJQYlQR}3ojRD)8i9ptM_MVgYfWu~lA#?sOntKz+qfNYVKQ)bm&Qk)}Ppx>kEhz%9v zQ81-Dncw~&TxLyFHs_3LiH0yfKQ(3JqKao6_PMBX-ahAUi^?&&QMqfu6;;^I+1*i< z6+C}#GO8v7LWshwiYkeo;X*Fk=_u4hXh80mvy3UaA9tu4qWk!)jm@P!LhNr*cz_RH30?D?W%i#5Q!?MX69Wd zqh*}UI*GL_rT!lf7UCfIs%DC~nWUB1?)8CSx{(iHLBm6X~^w>BDI)2>Yn2E|A++lTpQt z9cn+}C|nM?<&s+-ZZ5*1D=K(7_BbwX6W*g)DLR*{K}s7b@)>Ub6J2V|y0YogCV%G{F8x(*G^uWyE+#xt__}!6xZJ_Zd)yG3m12+Wp=J_+I&g*NoLL! z{YAZh=-QK4pZwm)N159`&!BShzLHbg&8S>$_%LnLuhj;x?Ik^k;MImJ?U&m9Y&dI6 zU@D}m4k@a?%0o^d_B$7mK_3+F(e2Gu{xny&w?dP|{0JlC<@a}Sw`#3AtA3G}tF;v5 zs?MQzXdz>1u5^?di8IXvtK+&^EvsYojdEh*g3MJ8H??8#_R*7nNEpubhSiAI_15kkX$g4@g}fid>&eq5|bEll%XxUB$#Q3H?hiunal|tVjw23 zP*MD1rR5AX43}steR*W_kpKQFtpz1c&wE?){hHVrVDJMkbVxPY7Q1hUK`>d)QG)}S9KWJD6I>wPM8m1eC3TlyxPRT+zT7cWNrV1q`S$~ z7j&QR;Nda(A~i@4#HV>8qb1&X&U{Be0^jdsn0(DIdi-54zYtOv@sbj1 zi`Jlt(p6EqI!c>8ijHb?XO!;n$obz+p#cgWcR?`9oen2a2^?0p*$`E@ZRgxXIzlQ& z<*@HWX?v8O;~Ifgl%9wx_;x5Kh?y@+KO>02dL(iU1bNaWf+!^NFfI-C(spzEkZG9Y zUd9OjZjt(=us9$K>l~ab2f>X~MVVMf>9EuM`Sp@n?EUr+<0t zx<2#e{azZWKFv$J*7XLT@#UWLUF(K)U-?VN|289gJ+IJjxx8>u@teGoS3Ew&x+ucd zGRxLYx!&$r8E@CGvkPB2ScG)bcc0JSR?OHcwy}H%E_qF}OmbJoqeDWQ=thz+w;YG16 zCZu3^Dw~L}nK6t+4+Fg`?k+=BTg2)% zlRBoDmqBjJdLY%Eidw2)){8W8;FEI1oC&8Ce~ZGT2l(|}q? z^`?VbSSgoJB~^zxRwd2TC}9J`!_t(LrN{4Iy0>4zd?dcqve0=_9YoRwQLc&-*{Npb zwklq}Py^_JjTbAA=hO^8A;mIz#hv`-RM%3h+9Io?>nQH8omy61OS9@7H1}UvLuIUr z$z{G)!vGnb4VQ?eQMt++bnhaYIk?l=3B; z0W8o$g2lYtsc>ct{}rz(8B*CCM&cHZ;lGmBEMg@7LMzgyq7qh>$kF>QWTcE@N%0q@p=CMq6S>*^ zte3GxY;nD;k=JbTjeiqTgm20G0OdBND7Wd}a=rgZpoHFk68rtd zJ(N)>jzr=YBWaS%U##g)pGs~u<|_0Vp0~)H@wxA0SSHz+h4-Z1lJc=p8M~=EE;%SF zsdq~6!Qkl!VSqj>C3^Tg?342{r@}5g$(@{71gyT0Y8Gh$(0!Dd$TN(Soe8JkJ^*neU zMU=+7kKtAZOX?yuMD@g^3TI+vknwnvNEP}VwWutiM5#iYOq%7eyvlw{2Ce^1=-N?P z?T#pYCQ3Vb4I7%KpkZqsgUQu)&dKeHR*;sA@d%BHcqTOMc)qIu=^|9|TObz7XkRU^ zyEy5yF4r%gey8~geTfbi*M03G+Wzn93h7JQLYmuw7v4(Fn?{Y*io5_((_UJY*cZ|KItOHNGol;IBR^8^`Z>eN;i{CaUE}L&&Fz6OgM6@G5X6ZT{Tz60@YG7*Osk z09?YKYYcR@Nc|?SFsv=QIPWWcvEkzUVs}VebSE?4U-409>5@WNE70F*S=VR0+;idZ z(qVsn;OJ|1f9;yye9yO!^ybggC}Y`|YD%BEn$iA!dr-AjTo=x059`_&_pYa9e3(|S zo}P8#)Y2(`RXDvI*3S&n%jf*`%g_4jUw*->AlAAqL3yAcIPR-j+_#>U2kp7ZU*T=~ zTWv;cS@2>nuqvxQ*5J`MPrRz~LFa!(b-l%_Sk37CID6C5UT>H8 zG!{J<4lEt;9et@6%bUyR0;QLq^X_}`z~@RTC;xL5Rk3X?ciZ)$t3&>yubjR<^kMEc zuX;%rTSLBj+<)@5Q{kL#YwGRdvhwJ6GJca^95ysu&GB^wPv1Co{nQ1k*LpQ)u_t6` zxKmUSII~t%yQZsKEvmgZ+V!1`M6s;LV2`!-!J@E^Tg9MLlYV? z74;>xztn%!pBv6(77u=GFad##yqBN#D()C_Uv>GLURn6aSnBP%ld~D!d-}EJ*UJNw zKQw=oQ|+VIx0-`nf~ViEeB;FO-j(bX>!0WS_;4t*?4sId@>#A{`5XM)dR~FQ>+)Iu z>C2wL-k|Y~{lTs`4n^{6v7&tJ@lfXDKBaHmZ}QpvmVnHE%r~=|Sst(j_XN4%zU7AH z@#W@V-)cqs`rd;-I}pn3^J)BLev5xA47qy(IsWR^%*vp7dCxMpyl#l~%t+^|wOl26^^sFy4DN{Bv5x>cXH#fb>2I+wDTARP^O?Gv2Q`p$NcxXLyv%fWh znx9$Di&V5nGTT?WZfgGga3u5K;=x}RZwc$RUY+tAeWe!;cn@5iTGee0%m?iuU290u z`t=vbXi8W7_1!`$^Rch*s;Ru~Uw@&d(i^}2SXcM;7iCmI^Vk1Ix0Sa@{9|2zgW@mC zbp5TWzid$B{?i?X{++6ybz~lqEB-1&MebYEkEm6@+H5?MuKt$=#v@sZqzziaOEFm$ zcRdPWnMl~?#w%G#gmIKgrHCiCYzI^uAy9D(kyAe0I6!LMa8;vz@xo%1nD6;(lpcwj zfQL;`c_daKsR$PpH*pKsskG3Yv{ThdbLU23IFu;g@kMAR4ADLR6pETwf_tV|bKw;d8At7v<)guV@(*xZ{aDMRZHqLvf0u)SQDFVQ6%?Qb$q^HulEsX zCaHyRix`2~UbxuoM=8%Lr7cZ@rhy^12Oq6_|1Tqk*x z!(}fj{Bk1iVH!G;aO2f3cJFYeXE6CzSFl#KVYvIeMY=^m5|Bn`Ne)NK>PCp`+{Y` z^1vbQN$i@wd=?r+dX{hRl>-+KymBa)al2I8^R{{gI z4t+4`Z@oMlm<+bQF&oKkyq(jwn$s4}=?G_ZM%10iWPtX#bR3(=Y9pZRBc|%0B4}C8 z4jz9yJyO_myP$Kmpfg;sGn}(4qTc1ztgAD~Uh|bR7tg$M?v~NB!$#Bxjocs1jTwzEh zcp!eyRgUbisK?pf@VP3g8toFVRF=kj1SfX-QaQ1(nnVp2dU%1@weY*d|H_9K zAi%XW6?f3@;W$4%7&riR*O~OmlzMF`iWCNe9MyL03c08T9UpiO_NSwr(yUGwo{Uv zIg;{X14_JNdG$%@o5fU^_mc8L+ewU@Bt^`Rel>^;U?6VH7gH$;E0RR{RK2bg*9OUY zQST20_8=)8HsZXE{CxA&3r6O8DXUL(2#3ZC&enSjGGu_G$6;~b(pIQ-WIw^$r^;oaz6 z)yPQ{O}Arg);>2zT;i(*mwzaNNSdEDlS3~MNONu+jUxWCF<`Y?c*ZKM=j<-tq{`b` z?G*SC#L^GYDvsFEp&{cmlAGitXP}ToxJyU>ZwX)QkG*mS@FG5wE{Q$=mJp@*Oc;(i zQxI>w^;Hjj8;%NYJCR*cmC)dr85ZhxOOqf44RGl^Ylj;L-(ErLDW>4I4#TZ|4qf#P z0FI`i@%CAweB&XiciF7m#7Uyf+a{5QMk39e&3h1XUQ$3qPP#!ahq)EYYAULo8xuP@ zsu@R5+9q)}iF*xcybO)13~(LYb~Sa=6odM@Zy0xFzX#Zf86)XTe2Ui-&C&| z>KA)pB7|L!sPMk2D>D~o0`-?2f%%UNHITpQ*~9|qx4mZe>wKe;^es?z(~bV7%e9g8 z(m?leHk8{ft&#LrUQhP5`S)Jl70D=zsLN7*NPo5LwyEqxQ(2%fY^uC%+P-SqzHI!n zviGars|uO6hfUoPb@zGI=Q=9A_RDlC({$nN`LjOH>kT*Be$e*To*x}}^FX+~`F45x zYI%D&ql2^uXn%*SJdC-%)BgI)^M31hOJI7;%fI5e5~&^x=M9}dKumt6#O9g%nTj&xM)V~x9P5p~tyjz!%>kp&95$5)j@@|T z`V-4tfhWRSTSBImh`tpa2CrPld57N~%vl~@OW%2uj-+=j?uRet_xJE?pmWQ{aA9L4 zt8sZaqApEN7Oh1m%Fp}e(pwCcbW4WFd+(bs zb0jVwEk=vUlJ(H~vn@H6Tua_VA#ajE&bMrS1jq$|T=)o(izJYXEhUctnHh;Mjx42? zvPURUo~UCq+p^`Mt=*c)Z>g|UK0*yuk`m?(HN+i05~K!Ig1?+J+-Wlt-^Hgu;t!dm9bBg5SD1C_)1x}BY)Ms`C*ldFNk4rkLh}cX-l0$fNjM@Gnr(uVe50Jgv4Hug!)`Xn^z>K*r#J|EN7vb69Fc4(-gArMQU6{Dq&-3%TXoAD%#Ef`Ic!b?uwxMs&nXoona zmO6_WE?@8o)Yu<0ZoXhm;*4%c2dJbmvp7<005R zz5xVY#ga)jhfQoDYfM@#7qLYHgZGw3B$^#@O|Y0P9+kDd1)42k^Ajdkwxo(mRaRu* zDKW@ri$DP-$>Yfuvn5Gb6*~!D#~y+g!}GnkMh6-%;-$;U`lMq-4}&A6YiB8IU>P`Y zC0QbzWfJ;f@5KuAV`9rJ^_B)vbK1aR%ULO3#Ln!-3lPe&ci1S-xuj9cW~H3)ex-}q zvNkNJ0i{XA;{OrVXo=bS zU}Gp@}IGE0kCB9#urst3m61>z5uR!iG}HU1iZNVt(p z`LLy3`XprhTV z2TjT4Knrb{;aih!pHj%l^PSJXp>FK~IYGlK2Jxf@N5w9%;`wD@IO<`7FEG4D`cpR*C!LY(DN?4U&VFlAL?k ziUD!B#NzlBWeZAnCnS-CJI_cm#~y)Og-99VFUs#l4P-NG!zHO`=)eABL^Ut``&&O2Y;VzvWH*oCb;{Vdvg=|lTg9q_vVnSQK}?4&)u7o zz!K?%tzwO2D|cjG%*QI=@Ln~%I^Yq%B!0moWKC(pJ6WkZ&CYGka!FtHp{*&(Vf+sB zbE03`FA^-DOVlAg0ZTKETHBArXPtAqq^2X5fmAZcax{-(w`ay_SZrq!pSQ4P_#Hu) zco2Q_{ArLiKWK)G#OD|CJz%a_hT@z?Lft#Mvy;aZKhgkU3_yI*CKp{Qr|e2_6Y7AHZ|OHiHt|By^XW zo5n3Amg7W5N%E(r(;7>uIKDy&w&oG(OA0$RF58m(QfvoLo$U?j;yWbUze+9HBy;Be zoQ!RV+kvo}e4uR({AzpyzgpZt8h9<0S{{EdReB--f!GFTp#;|_ud_u}oQeIe#J4A&d6HYSvvwZ7vbVden)&u{DLcj z8VO-jEA&&{?x$q!FXoVLB0{WR{ zQ8i~fJ#RjwY5nwdrTWa2JO zLL9awzzJ~$%{aKH;C-HN0T%&edOV^aA7PNhYeH0zF*i4F^R&dG{0ea;<6bE^Er&pf zq=wY4C_q~UFBWvQPz8EP^pLPKu(Ep0i9nYz2x58yV?JP^*`t$$r}Z>(^U^@Ff_Dse zhQ#S4{>`zNo*Uf4J!l~x@-Wt0_Qwr!%F`UHSnR@i5*#@ewpap791bCTVgeoF=l$>h z;3~P^+wEy5ucW=CPl)3RAD5PRkN_wjI=2p?u455t<3o(-i#))1phft@{DO=N85}|= zg0rUGn}yfMHkv&uA!1{{ zrzA#1pcR*!1Y@=)Q_VE0$MxB6-k;u6!Ba4ZN{F-@pF5~{0;d@fMHG&rHcmK@No?KR zb#nV5xxJ1XW}gN=FR^Gepy;zCiZT4qe}zO(R*IP2+!RSkiv{oFTJRR7wH|#u{wCsj zdnWkfk7LAjhPc#JOJA+$s^lJM(j zR|gZN>pU{1%9De*)qG%LX(Jhr>MV!o(~M_RiZ15ejy>tIX~$2T4#tx!PU!e_y9sk?$yo1?F5M)vxMXo|m#E&AYafZrh zp$t^)*fFgAL_bIzJN})#cE?md2dgM?89Rg@{&1}eCQ`(>;oYLR!+4J7%_4SqiZjO3 zG(4oRMJ${#J}|a=Vl;I8iCYu5T7S7Yd}8!#7ukCLy-XABF?1q2Z!BajIz-t?24MQ~ zM-~4;S8Ln_QfQd+7~<|=B>1}L8U9dhjGPE1CcX2dD0U~vj0~^fuU37!en;$yr^k4` zVy=6<_q1FXczO&8{sfNW(Tamp&3G~q)gG~e6D$E?g7CqUK1Ry$E@M2->G2%ZebqyR z&ReE9_#9ClT}M1}@=lLV07qzx(4uQ{CnmFEa?*iG!FYC}SpAeIopDx?N_Oo;$Yt|R zA+Avx@AxOREz1o9Au+zs+UPvUSRu=aB#thV_8PRa1jf-YM$y^EX1!S6Zu0158IMbpN{ za4u+UjN5{8Q9aIlk0CFgL-3R5kMn4SkVr@&u6QI0F9!{k;)u%%ry)0NKERzmCJM&j zIuDOW#J=UD2y#CqIm{%7Cf>)~GI`hY@3EP~m63cuBe#7dQ9;5I@~5XcVp${cDBZ&O z^QTCjA#x*bCfpy9+mpCOwexc%CMAhu$)9WI$WokRNRk3!+TGmq* zgnNg)sNuNCJ10gBPO;VAZRJp`IK~^AsfWyuGGm`o)`G$A|&^V&>=_ozRb&)R! zGjP_$+4)Bj1Pw>iCUK69ZxinN!#_Il=82V-aLt~Z^^ux=YnmP$BvYuqedL=*e7$Q5MtnZBrYI4g z_xiU5-Jy>D#UpEqBjU%tP-PGKuwVReD5wZE4w6qp;-`bJcQ5Cz=l6Xkr^;-y2Wri6+KLlUPZTeAz-UZWA#+^!niPmi7Fn;+apxGaq=paaln!H{(SAg`G<~ zU))V%9q7L8`KHI$_3anF^#V@eT~MD_hl~wt>c&sC`U~bI^KEU(sg!o=Zhv?5-O=UImj-=VzRBCUm8-dx!HP)k z4*34w%7XWAdL|C=kuatgW|wAfYYNvig#p=};%&i(AGN;O`bX{I;^tL#fmi8se}&ki zMW1eIT~V%_x#_+&^fxDdaU#6ssZ~YkVy|CwCr#_^45#INztMl@d!1`(6>-Gd*VG;H zzy-ud`P}()?`#im+V#%%AfgCdE4*45u>N7mJKI+`?Fwh?UOez=BFI314%7$6aY%4{ zxqSKbO8t-LL-{+ynY$Jb-qGm2PrO*{FO6sl7Wdpi$5frK@-=+la_#ul<7?UqbV|(G zdHi+R4fS<6JDV4K#P|iKSM&gZGyYYV5Od&_>cw8-(0lUyNyIxi^Sb56@$1Le(re+* zYcwq$`c!a^zV^)3XKowJa5lzKB^t*R4_!QTIX$2X>nrgM?Rx(F^L{qa6*!H61!sb% zm+Rl2UopNt6RPY8ZRv;1vp%h-GK=q=SPq7rh zzrP6$ZoJWPy(4H{E`8e;E^k^svs%70#O#a|?7CgBZ?$0GPbb3#y=(eDa)PkoO6#Ro zf77+jtDUcNH$2xp;ev+cp?9Bp=c%8Xe!lsG&Ee+0+s*x}&Ha&r{&3cj+ga>t7P~h5 z#O>j+)#0&l*3%L7)84dCNRWt^`^kB$%nS2N^Zrur{EN@yP-I4y@AT3FIUQ+wbpj_Z zo3CxZy8U}K;oPc-y2`6XC_P`7&+XfPc^e|yHof+Yw+S!|=@*VK9rxFJkB1FKI1!2X zfX0`P`Ho(6BWQ#B^6-_>i=#oKZ#0~U!c}4uA41+b?bRPcJK0lp4sc}F7ZBQNScucXqqeYuTNrrlO#f2hdz4g0O&XP7U(>?;Kq3w(x9 z(T-4h{c_`NP3xMbb){ilt-YXLQv34#gM<(RQ{keTh_Pl_5ivHcshjWQB05T-@}t~s zOFf7$;#$|I3-Lw#S(k_5xPR{IxwqtRbt6=>AzacD$!S^1j^yn8NWTj{>UrD0Qc~Ks zFSV35U98?e5=!3|?7pqpv8LIvyeC%sXB2JP3D&6D^(p+~x$ov9lp#V^)?YaDoijK` zdqsUw?LUYJA$|T`L1VCe`RI!A$HyYuJ6Eic?Yl$U9=}Q7EWc^}S;fsww~XtBTS?Af zBcgV+L<;Md%R>b%%hnau%IQDX-0Z$Z|Ewp{cIZ}PsP)LL)9ZzmLHf1+;NY_HZ8lQa zxZE8oXj_qerj%`d?7veo<4#1^Fco~2MrrF2#8_v-_>s{r4IIUJfQYsVlUAYeBHW6v z+qV$N4X8s^$3pDWp<~vNbv&e*s0=8*^?P)g5@L)Q=eurII+Cyg7Iw=Q!l74L6*Z`)6*Z)HU~`fd$| z+K-|p3th5aQg!Y5tIvP$8_QXdlBVUsP;onf+ezR)_9Lod9l-16Dnz4QM|8v zPq+O1&AyQK=$gXvNt*6<+9v#8P1|%QJNL?gO9%W%Umw45^7_fO>^iUJPBC-M{s;CT z{k<8l##aSqe5bT#U2pR3_dgL-{^q^>Ja#qf~?|IMjljnXp5waY=Z8@=OIT5yug{-!4 z|5Rl6RA~B{(5_RV%$fftr!!`0LW^_sr#K}&bmggwPx%f9ibAHk<*eI=<~2j}iYYb? z(3`nkB(q@;{VAfyG<^37gpA29@Sncehli}}E6o?1ue83}y1uCxfn_wlu65F37Y_t9 z!O}PM{uhErg9|Ihm7{;26RF#MvouoI9je`Xv+Ji5w@UwJ>o2zcb>&~q+#C%J3hZI@ z!(*eNv8Tfzu5nvvXeu-{eTON#q5FX@*cIf$Ov7>`h6_@sbmjDW74H|lSM-yTTUnvD z!?%w9lKY#5f3XmHf|QRuVGSP{4^2#k`lmzFb_{&+Df+)tbj}R@IikvxfZEerzBEyq zbf3!SzVhtFXMMAQonRVk>IRJcw_o`73*USJ;^THk`G*G;fz)s zi^V`9UO16GLE~$){xL*wcfb3>J1>M3T!yKkF|JZ8ERZzvzN62<0r7#fP&vWZJP@e%sKpW@uR{ ziw$Qer(#WC`EhaCMXgWav-*u9@7e2b2^b;R2LqMhKlQ=EVCAwYl;8AGW;6I)Njdmj zdMo1M6m-!`hrHds90XMLoxWV@FTY&v?Y@(hbEWN4TX1N(?2nI!3tQei9;6YD$aRhN zvu_?>Eo=#AwIU3=(FCL?E}pubQ?Z&;5zeWK7^{Lu|0HX9=%3}keS$;_=(>ZGKF2VR zIs>j?W56DKcBSe4_V?O<(s|Pr>gWYE_R*WxHx*t}UsWSMt2(@?Dwq|@s{wl}$1K*o zr}>F4($Ia={qu7loC_UeZ=MV9ACA-yhmJoL+A)ISC+M>EvhqN8AP*ccYa2MAVLNC7 z&BE!XBzo8tN^kf`-3WQ3&i%?JgQ5jhETk#?Utj2{{M}#ep~3heVnUkM-+cxCUk`>K z((L%%S1q6|OrnsX`FCGcpwln{Awvr(Rn+PlhSvWJs_0yk@~_i3FR0ZY%l4=j49b7i zSiMlH{MTFd=<)L(OWPKjmH(-D&u;wOt?Kk>l>e>RUE& zp!vF4=TT{Xr_p&fYoh8gM0tbdY|L!nqGX-1PHto+jgZXqv(5!!EnP#tk`=J9_Wdq- zx<+nf-N-K}2qdz6!Mgn*`Ig2f;1*RsCPe$##T8=Ff&Y{X`-c=J7)^h!sHGKm5$1r- z{Fu`ITgvo>s*tYvVpkho`o$T#nr{1Il+LBMebJz&3%)4Mqj!GMv{yzqeb%p_6nd{` TO`iX2h5GkR( literal 0 HcmV?d00001 diff --git a/scripts/pseudo_alignment_nesting.py b/scripts/pseudo_alignment_nesting.py new file mode 100644 index 00000000..7fe59fdc --- /dev/null +++ b/scripts/pseudo_alignment_nesting.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +"""Family overlap and clade-closure ("nesting") of a SNP pseudo-alignment. + +Each alignment column is one family; a genome "carries a call" at a column +when its character is not an absence symbol (default '-' and '?'). + +Outputs (prefix given by -o): + _family_overlap.csv genome x genome matrix: number of columns where + both genomes carry a call (diagonal = 0, same + convention as `obikmer phylo --family-overlap`) + _nesting.csv per clade-size stratum: how closed the families + are under the tree + +Nesting measure: for a family carried by the genome set S, let C(S) be the +smallest tree clade containing S. completeness = |S| / |C(S)|. A value of 1 +means the family is clade-closed (every genome of the clade carries it); low +values mean dropout or patchy conservation inside the clade. + +The tree is rooted as written in the Newick file unless --outgroup is given +(rooted at the neighbour of that tip, which then becomes a child of the root). + +Optional lift test (--lift-test): do families covering a rare pair (A,B) also +cover an abundant pair (A,C) more often than the margins alone imply? For each +genome A, B ranges over its --lift-frac rarest partners and C over its most +abundant ones (ranked on pair overlap). Statistics, on a random column subsample: + T = mean over triples of O_ABC / O_AB = P(C | A,B) + L = sum n_A * O_ABC / sum O_AB * O_AC (lift; 1 if B,C independent given A) +Null: curveball swap randomisation of the genome x family presence matrix, which +preserves every genome's family count and every family's size exactly. Each null +replicate re-selects the rare/abundant partners from its own overlaps, so the +selection procedure is identical to the observed one. + +Optional quartet test (--quartet-test): does flattening pair overlaps also flatten +quartet coverage? --quartets N genome quartets are drawn uniformly; for each, the +number of columns where all four genomes carry a call is counted exactly. Reported +per stratum of the quartet's depth (size of the smallest tree clade containing the +four genomes): count distribution, share of uncovered quartets, and the ratio of the +4-way count to the smallest of the six pair overlaps (1 = perfect nesting; the pair +overlap is an upper bound of the 4-way count). + +Usage +----- + pseudo_alignment_nesting.py ALIGNMENT.fasta TREEFILE -o prefix + [--outgroup LABEL] [--absent '-?'] [--chunk 131072] + [--quartet-test [--quartets 20000]] + [--lift-test [--lift-columns 200000] [--lift-frac 0.1] + [--null-reps 20] [--trades 5000] [--seed N]] +""" + +import argparse +import sys + +import numpy as np + + +def read_fasta_presence(path, absent): + """Return (labels, packed presence bits per genome, alignment length).""" + absent_lut = np.zeros(256, dtype=bool) + for ch in absent: + absent_lut[ord(ch)] = True + labels, packed, length = [], [], None + + def flush(parts): + nonlocal length + seq = np.frombuffer(b"".join(parts), dtype=np.uint8) + if length is None: + length = len(seq) + elif len(seq) != length: + sys.exit(f"{labels[-1]}: length {len(seq)} != {length}") + packed.append(np.packbits(~absent_lut[seq])) + + parts = None + with open(path, "rb") as fh: + for line in fh: + if line.startswith(b">"): + if parts is not None: + flush(parts) + labels.append(line[1:].split()[0].decode()) + parts = [] + else: + parts.append(line.strip()) + if parts is not None: + flush(parts) + return labels, np.vstack(packed), length + + +def parse_newick(text): + """Parse Newick into (names, parent, length) arrays; internal labels ignored.""" + names, parent, length = [], [], [] + stack, cur = [], None + i, n = 0, len(text) + + def new_node(): + names.append(None) + parent.append(-1) + length.append(0.0) + return len(names) - 1 + + while i < n: + c = text[i] + if c in " \n\r\t": + i += 1 + elif c == "(": + if stack: + node = stack[-1] # the placeholder child becomes this internal node + else: + node = new_node() + stack.append(node) + cur = None + i += 1 + child = new_node() + parent[child] = node + stack.append(child) + elif c == ",": + stack.pop() + child = new_node() + parent[child] = stack[-1] + stack.append(child) + i += 1 + elif c == ")": + stack.pop() + cur = stack[-1] + i += 1 + j = i + while j < n and text[j] not in ":,);": + j += 1 + i = j # internal label (support values) discarded + elif c == ":": + j = i + 1 + while j < n and text[j] not in ",);": + j += 1 + length[stack[-1] if cur is None else cur] = float(text[i + 1:j]) + cur = None + i = j + elif c == ";": + break + else: + j = i + while j < n and text[j] not in ":,);": + j += 1 + names[stack[-1]] = text[i:j] + i = j + return names, parent, length + + +def leaf_order_matrices(names, parent, length, outgroup): + """DFS leaf order plus LCA-clade-size and patristic matrices in that order.""" + adj = [[] for _ in names] + for v, p in enumerate(parent): + if p >= 0: + adj[v].append((p, length[v])) + adj[p].append((v, length[v])) + root = next(v for v, p in enumerate(parent) if p < 0) + if outgroup is not None: + tip = names.index(outgroup) + root = adj[tip][0][0] + + order, depth, children = [], {root: 0.0}, {} + stack, seen = [root], {root} + while stack: + v = stack.pop() + kids = [(u, w) for u, w in adj[v] if u not in seen] + for u, w in kids: + seen.add(u) + depth[u] = depth[v] + w + children[v] = [u for u, _ in kids] + stack.extend(reversed(children[v])) + order.append(v) + + leaves = [v for v in order if not children[v]] + rank = {v: r for r, v in enumerate(leaves)} + n = len(leaves) + span = {} + for v in reversed(order): + span[v] = (rank[v], rank[v] + 1) if not children[v] else ( + span[children[v][0]][0], span[children[v][-1]][1]) + + clade = np.ones((n, n), dtype=np.int32) + lca_depth = np.zeros((n, n)) + for v in order: + kids = children[v] + for a in range(len(kids)): + for b in range(a + 1, len(kids)): + (a0, a1), (b0, b1) = span[kids[a]], span[kids[b]] + size = span[v][1] - span[v][0] + clade[a0:a1, b0:b1] = clade[b0:b1, a0:a1] = size + lca_depth[a0:a1, b0:b1] = lca_depth[b0:b1, a0:a1] = depth[v] + leaf_depth = np.array([depth[v] for v in leaves]) + patristic = leaf_depth[:, None] + leaf_depth[None, :] - 2 * lca_depth + return [names[v] for v in leaves], clade, patristic + + +def lift_stats(pres, frac): + """(L, T) for the rare/abundant partner triples of every genome (see module doc).""" + n = pres.shape[0] + pf = pres.astype(np.float32) + overlap = pf @ pf.T + k = max(1, round(frac * (n - 1))) + num = den = ratio_sum = 0.0 + n_ratio = 0 + for a in range(n): + order = [j for j in np.argsort(overlap[a], kind="stable") if j != a] + rare, abundant = order[:k], order[-k:] + cols = pres[a] + n_a = float(cols.sum()) + if n_a == 0: + continue + o_abc = pf[rare][:, cols] @ pf[abundant][:, cols].T + o_ab, o_ac = overlap[a, rare].astype(np.float64), overlap[a, abundant].astype(np.float64) + num += n_a * float(o_abc.sum()) + den += float((o_ab[:, None] * o_ac[None, :]).sum()) + ok = o_ab > 0 + ratio_sum += float((o_abc[ok] / o_ab[ok, None]).sum()) + n_ratio += int(ok.sum()) * k + return num / den, ratio_sum / n_ratio + + +def curveball(pres, trades, rng): + """In-place curveball trades: preserve every row sum and every column sum.""" + n = pres.shape[0] + for _ in range(trades): + i, j = rng.choice(n, size=2, replace=False) + a, b = pres[i], pres[j] + diff = np.flatnonzero(a ^ b) + k = int((a & ~b).sum()) + diff = rng.permutation(diff) + a[diff] = False + b[diff] = False + a[diff[:k]] = True + b[diff[k:]] = True + + +def quartet_test(packed, overlap, tree_rank, clade, n_quartets, bin_edges, rng): + """Exact 4-way co-presence counts for random genome quartets. + + Returns (counts, ratio_to_min_pair, depth_stratum), one entry per quartet. + """ + n = packed.shape[0] + quartets = np.argpartition(rng.random((n_quartets, n)), 3, axis=1)[:, :4] + counts = np.zeros(n_quartets, dtype=np.int64) + block = 2048 # bytes of packed columns per pass: bounds the (Q, block) gathers + for a in range(0, packed.shape[1], block): + blk = packed[:, a:a + block] + acc = blk[quartets[:, 0]] & blk[quartets[:, 1]] + acc &= blk[quartets[:, 2]] + acc &= blk[quartets[:, 3]] + counts += np.bitwise_count(acc).sum(axis=1, dtype=np.int64) + + pair_min = np.full(n_quartets, np.iinfo(np.int64).max) + for x in range(4): + for y in range(x + 1, 4): + pair_min = np.minimum(pair_min, overlap[quartets[:, x], quartets[:, y]]) + ranks = tree_rank[quartets] + csize = clade[ranks.min(axis=1), ranks.max(axis=1)] + stratum = np.searchsorted(bin_edges, csize, side="right") - 1 + return counts, counts / np.maximum(pair_min, 1), stratum + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("alignment") + ap.add_argument("treefile") + ap.add_argument("-o", "--output", required=True, help="output file prefix") + ap.add_argument("--outgroup", help="tip label to root the tree on") + ap.add_argument("--absent", default="-?", help="absence symbols (default '-?')") + ap.add_argument("--chunk", type=int, default=131072, + help="columns per chunk, multiple of 8 (default 131072)") + ap.add_argument("--quartet-test", action="store_true", + help="measure exact 4-way coverage of random genome quartets") + ap.add_argument("--quartets", type=int, default=20000, + help="number of random quartets (default 20000)") + ap.add_argument("--lift-test", action="store_true", + help="run the margin-preserving lift test (see module doc)") + ap.add_argument("--lift-columns", type=int, default=200000, + help="random columns used by the lift test (default 200000)") + ap.add_argument("--lift-frac", type=float, default=0.1, + help="fraction of partners ranked rare / abundant (default 0.1)") + ap.add_argument("--null-reps", type=int, default=20) + ap.add_argument("--trades", type=int, default=5000, + help="curveball trades between replicates, also burn-in (default 5000)") + ap.add_argument("--seed", type=int, default=None) + args = ap.parse_args() + if args.chunk % 8: + sys.exit("--chunk must be a multiple of 8") + + rng = np.random.default_rng(args.seed) + labels, packed, length = read_fasta_presence(args.alignment, args.absent) + keep_p = min(1.0, args.lift_columns / length) if args.lift_test else 0.0 + sampled = [] + with open(args.treefile) as fh: + names, parent, blen = parse_newick(fh.read()) + leaf_labels, clade, patristic = leaf_order_matrices(names, parent, blen, args.outgroup) + if set(leaf_labels) != set(labels): + sys.exit("tree tips and alignment records differ") + n = len(labels) + tree_row = np.array([labels.index(l) for l in leaf_labels]) # tree rank -> alignment row + + bin_edges = np.array([2, 3, 5, 9, 17, 33, 65, 129, n + 1]) # clade-size strata + nb = len(bin_edges) - 1 + cnt = np.zeros(nb, dtype=np.int64) + closed = np.zeros(nb, dtype=np.int64) + sum_ratio = np.zeros(nb) + sum_missing = np.zeros(nb) + overlap = np.zeros((n, n), dtype=np.int64) + n_cols_lt2 = 0 + + for a in range(0, length, args.chunk): + b = min(a + args.chunk, length) + pres = np.unpackbits(packed[:, a // 8:(b + 7) // 8], axis=1)[:, :b - a] + if keep_p: + sampled.append(pres[:, rng.random(b - a) < keep_p].astype(bool)) + overlap += np.rint(pres.astype(np.float32) @ pres.astype(np.float32).T).astype(np.int64) + + t = pres[tree_row] + size = t.sum(axis=0) + keep = size >= 2 + n_cols_lt2 += int((~keep).sum()) + t, size = t[:, keep], size[keep] + lo = t.argmax(axis=0) + hi = n - 1 - t[::-1].argmax(axis=0) + csize = clade[lo, hi] + stratum = np.searchsorted(bin_edges, csize, side="right") - 1 + cnt += np.bincount(stratum, minlength=nb) + closed += np.bincount(stratum, weights=(size == csize), minlength=nb).astype(np.int64) + sum_ratio += np.bincount(stratum, weights=size / csize, minlength=nb) + sum_missing += np.bincount(stratum, weights=csize - size, minlength=nb) + + np.fill_diagonal(overlap, 0) + with open(f"{args.output}_family_overlap.csv", "w") as fh: + fh.write("genome," + ",".join(labels) + "\n") + for l, row in zip(labels, overlap): + fh.write(l + "," + ",".join(map(str, row)) + "\n") + + with open(f"{args.output}_nesting.csv", "w") as fh: + fh.write("clade_size_min,clade_size_max,n_families,mean_completeness," + "frac_clade_closed,mean_missing_in_clade\n") + for k in range(nb): + if cnt[k]: + fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{cnt[k]}," + f"{sum_ratio[k] / cnt[k]:.4f},{closed[k] / cnt[k]:.4f}," + f"{sum_missing[k] / cnt[k]:.3f}\n") + + off = overlap[np.triu_indices(n, 1)] + print(f"genomes={n} columns={length} (columns with <2 calls skipped for nesting: {n_cols_lt2})") + print(f"overlap off-diagonal: min={off.min()} median={int(np.median(off))} " + f"max={off.max()} CV={off.std() / off.mean():.3f}") + tot = cnt.sum() + print(f"clade-closed families: {closed.sum()}/{tot} ({closed.sum() / tot:.3f}); " + f"mean completeness {sum_ratio.sum() / tot:.3f}") + print(f"wrote {args.output}_family_overlap.csv and {args.output}_nesting.csv") + + if args.quartet_test: + tree_rank = np.empty(n, dtype=np.int64) + tree_rank[tree_row] = np.arange(n) + counts, ratio, stratum = quartet_test( + packed, overlap, tree_rank, clade, args.quartets, bin_edges, rng) + print(f"quartet test: {args.quartets} random quartets over {length} columns") + q = np.quantile(counts, [0, .01, .1, .5, .9, .99, 1]) + print("4-way count quantiles 0/1/10/50/90/99/100 %: " + " ".join(f"{v:.0f}" for v in q) + + f"; uncovered: {(counts == 0).mean():.4f}; CV={counts.std() / max(counts.mean(), 1e-9):.2f}") + with open(f"{args.output}_quartets.csv", "w") as fh: + fh.write("depth_clade_min,depth_clade_max,n_quartets,median_count,q10_count," + "frac_uncovered,median_ratio_to_min_pair\n") + for k in range(nb): + m = stratum == k + if m.any(): + c = counts[m] + fh.write(f"{bin_edges[k]},{bin_edges[k + 1] - 1},{int(m.sum())},{np.median(c):.0f}," + f"{np.quantile(c, .1):.0f},{(c == 0).mean():.4f},{np.median(ratio[m]):.4f}\n") + print(f"wrote {args.output}_quartets.csv") + + if args.lift_test: + pres = np.ascontiguousarray(np.hstack(sampled)) + print(f"lift test on {pres.shape[1]} columns, {args.null_reps} null replicates") + rows = [("observed",) + lift_stats(pres, args.lift_frac)] + curveball(pres, args.trades, rng) # burn-in + for r in range(args.null_reps): + rows.append((f"null{r + 1}",) + lift_stats(pres, args.lift_frac)) + curveball(pres, args.trades, rng) + with open(f"{args.output}_lift_test.csv", "w") as fh: + fh.write("replicate,L,T\n") + for name, l, t in rows: + fh.write(f"{name},{l:.6f},{t:.6f}\n") + for idx, label in ((1, "L (lift)"), (2, "T = P(C|A,B)")): + obs = rows[0][idx] + null = np.array([r[idx] for r in rows[1:]]) + p = (1 + (null >= obs).sum()) / (1 + len(null)) + print(f"{label}: observed={obs:.4f} null mean={null.mean():.4f} " + f"sd={null.std(ddof=1):.4f} z={(obs - null.mean()) / null.std(ddof=1):.1f} " + f"p(null>=obs)={p:.3f}") + print(f"wrote {args.output}_lift_test.csv") + + +if __name__ == "__main__": + main() diff --git a/src/obikphylo/src/siblings/algorithms/entropy.rs b/src/obikphylo/src/siblings/algorithms/entropy.rs index 175d46ac..78c27c77 100644 --- a/src/obikphylo/src/siblings/algorithms/entropy.rs +++ b/src/obikphylo/src/siblings/algorithms/entropy.rs @@ -51,28 +51,25 @@ pub(crate) fn family_entropy(genome_mask: &[u8]) -> Option<(f64, usize)> { Some((h, present as usize)) } -/// Shannon entropy (bits) of one family's per-genome states, reduced to the -/// 4 plain nucleotide symbols instead of the 15-state space above — kept -/// alongside it (not instead of it) purely to measure how much the two -/// diverge on real data. A genome carrying several bases at once -/// (`genome_mask[g]` with more than one bit set) contributes to *each* of -/// those bases' counts — counted once per base present, not split -/// fractionally, and not folded into a combined state the way -/// [`family_entropy`] does. The denominator is therefore the total number -/// of base-occurrences summed over every present genome, not the genome -/// count. `None` under the same condition as [`family_entropy`]. +/// Shannon entropy (bits) of one family's per-genome states over the 4 plain +/// nucleotide symbols, restricted to the genomes carrying exactly one member +/// of the family (`genome_mask[g].count_ones() == 1`) — the genomes that stay +/// informative under `--no-ambiguity`, where a genome carrying several +/// members (paralogues, not read ambiguity) becomes missing data. Genomes +/// where the family is absent (`genome_mask[g] == 0`) and genomes carrying +/// several members are both left out of the denominator, which is returned +/// as the second value (`n_genomes_unambiguous`). `None` if no genome carries +/// exactly one member. pub(crate) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> { let mut freq = [0u32; 4]; - let mut total = 0u32; + let mut n_single = 0u32; for &m in genome_mask { - for (base, count) in freq.iter_mut().enumerate() { - if m & (1 << base) != 0 { - *count += 1; - total += 1; - } + if m.count_ones() == 1 { + freq[m.trailing_zeros() as usize] += 1; + n_single += 1; } } - if total == 0 { + if n_single == 0 { return None; } let mut h = 0.0f64; @@ -80,10 +77,37 @@ pub(crate) fn family_entropy_4(genome_mask: &[u8]) -> Option<(f64, usize)> { if c == 0 { continue; } - let p = f64::from(c) / f64::from(total); + let p = f64::from(c) / f64::from(n_single); h -= p * p.log2(); } - Some((h, total as usize)) + Some((h, n_single as usize)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entropy_4_balanced_two_states_is_one_bit() { + // two genomes carry base 0 (A), two carry base 2 (G); one is absent + let (h, n) = family_entropy_4(&[0b0001, 0b0001, 0b0100, 0b0100, 0]).unwrap(); + assert!((h - 1.0).abs() < 1e-12); + assert_eq!(n, 4); + } + + #[test] + fn entropy_4_ignores_genomes_with_several_members() { + // the two-member genomes (0b0011, 0b1100) are left out entirely + let (h, n) = family_entropy_4(&[0b0001, 0b0001, 0b0011, 0b1100]).unwrap(); + assert_eq!(h, 0.0); + assert_eq!(n, 2); + } + + #[test] + fn entropy_4_is_none_without_unambiguous_genome() { + assert!(family_entropy_4(&[0, 0b0011, 0b0110]).is_none()); + assert!(family_entropy_4(&[]).is_none()); + } } /// Builds one layer's entropy annex if it doesn't already exist, and/or diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index 7cc26ba3..b1812c1b 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -45,10 +45,13 @@ pub trait SiblingExt { fn build_sibling_annex(&self) -> OKIResult<()>; /// Write a CSV (`partition,layer,family_idx,entropy15,entropy4, - /// family_size,n_genomes_present`) of per-family Shannon entropy — both - /// [`family_entropy`] (15 non-empty states, the project's settled - /// definition) and [`family_entropy_4`] (plain 4-symbol reduction, kept - /// alongside for comparison) — one row per non-monomorphic minorant of + /// family_size,n_genomes_present,n_genomes_unambiguous`) of per-family + /// Shannon entropy — both [`family_entropy`] (15 non-empty states over + /// every genome carrying the family, the project's settled definition) + /// and [`family_entropy_4`] (4 plain bases over the genomes carrying + /// exactly one member only, i.e. what survives `--no-ambiguity`; + /// `n_genomes_unambiguous` is its denominator, `0` meaning `entropy4` + /// is not defined) — one row per non-monomorphic minorant of /// every cached layer, from an already-built sibling annex (run /// [`build_sibling_annex`](Self::build_sibling_annex) first). /// @@ -166,7 +169,7 @@ impl SiblingExt for IndexCache { let fast_mode = is_fast_mode(self); let mut f = BufWriter::new(std::fs::File::create(path).map_err(OKIError::Io)?); - writeln!(f, "partition,layer,family_idx,entropy15,entropy4,family_size,n_genomes_present") + writeln!(f, "partition,layer,family_idx,entropy15,entropy4,family_size,n_genomes_present,n_genomes_unambiguous") .map_err(OKIError::Io)?; let pb = progress_bar("shannon_entropy", self.n_partition() as u64, "partitions"); @@ -205,10 +208,10 @@ impl SiblingExt for IndexCache { let Some((h15, n_present)) = family_entropy(genome_mask) else { return; }; - let h4 = family_entropy_4(genome_mask).map_or(0.0, |(h, _)| h); + let (h4, n_unambiguous) = family_entropy_4(genome_mask).unwrap_or((0.0, 0)); if let Err(e) = writeln!( f, - "{part},{l},{family_idx},{h15:.6},{h4:.6},{},{n_present}", + "{part},{l},{family_idx},{h15:.6},{h4:.6},{},{n_present},{n_unambiguous}", mask.family_size() ) { write_err = Some(e); -- 2.54.1 From f29dcbcfb8f2eb9b84fcbf3e9727b28d644a5460 Mon Sep 17 00:00:00 2001 From: Eric Coissac Date: Tue, 13 Oct 2026 08:34:09 +0200 Subject: [PATCH 10/10] Implement advanced sampling and family annex logic Introduces complex sampling mechanisms including quartet filtering, pair balance weighting, and entropy bias. Enforces the `--no-ambiguity` convention for family counting and updates annex storage for scalar data. This includes new logic for calculating family weights based on overlap, quota-based layer determination, and filtering families based on quartet informativeness. Updates also include argument parsing for `--n-sites` and improved handling of family overlap files. --- DevDocMD/theory/pair_coverage_sampling.md | 64 +++++ .../phylogeny/snp_based/sampling_design.md | 14 +- UserDocMD/usage/phylo.md | 43 ++- benchmark/simulate_skim.sh | 57 ++++ src/Cargo.lock | 2 +- src/obikmer/Cargo.toml | 2 +- src/obikmer/src/cmd/phylo/args.rs | 75 +++++- src/obikmer/src/cmd/phylo/mod.rs | 30 ++- .../src/siblings/algorithms/entropy.rs | 148 +++-------- .../src/siblings/algorithms/family_annexes.rs | 212 +++++++++++++++ .../src/siblings/algorithms/masking.rs | 93 +++++++ .../siblings/algorithms/minorant_selection.rs | 48 +++- src/obikphylo/src/siblings/algorithms/mod.rs | 12 +- .../src/siblings/algorithms/pair_weight.rs | 121 +++++++++ .../src/siblings/algorithms/quartet.rs | 45 ++++ src/obikphylo/src/siblings/algorithms/scan.rs | 14 +- .../src/siblings/algorithms/subsample.rs | 236 +++++++++++------ src/obikphylo/src/siblings/entropy_annex.rs | 162 ------------ .../siblings/extensions/sibling_builder.rs | 80 ++++-- .../src/siblings/extensions/sibling_ext.rs | 84 +++--- src/obikphylo/src/siblings/family_overlap.rs | 82 +++++- .../src/siblings/family_scalar_annex.rs | 247 ++++++++++++++++++ src/obikphylo/src/siblings/helpers.rs | 12 + src/obikphylo/src/siblings/mod.rs | 15 +- src/obikphylo/src/siblings/quartet_annex.rs | 59 +++++ 25 files changed, 1471 insertions(+), 486 deletions(-) create mode 100755 benchmark/simulate_skim.sh create mode 100644 src/obikphylo/src/siblings/algorithms/family_annexes.rs create mode 100644 src/obikphylo/src/siblings/algorithms/pair_weight.rs create mode 100644 src/obikphylo/src/siblings/algorithms/quartet.rs delete mode 100644 src/obikphylo/src/siblings/entropy_annex.rs create mode 100644 src/obikphylo/src/siblings/family_scalar_annex.rs create mode 100644 src/obikphylo/src/siblings/quartet_annex.rs diff --git a/DevDocMD/theory/pair_coverage_sampling.md b/DevDocMD/theory/pair_coverage_sampling.md index 3e2be6e5..2021518b 100644 --- a/DevDocMD/theory/pair_coverage_sampling.md +++ b/DevDocMD/theory/pair_coverage_sampling.md @@ -214,6 +214,70 @@ Per-genome share of the 5 003 414 sampled columns with a call: median 5.2 %, max Consequence for flattening: raw $O_{ij}$ mixes evolutionary distance and per-genome coverage; flattening on it chases pairs involving data-poor genomes, for which no family exists. Normalizing, $O_{ij}/(c_ic_j)$ with $c_i$ the number of families carried by genome $i$ (per-genome counts from `--sibling-stats`), targets distance instead of coverage; the best normalization (smoothing of $c_i$, etc.) is not established. +## Implemented: entropy annex parametrised by `--no-ambiguity` (2026-10-13) + +Design decision (user): the way genomes are counted depends on `--no-ambiguity`, so that flag, not an "alphabet" option, is the annex parameter. Over single-member genomes the 15-state entropy only has the four single-base states, hence `entropy4` (redefined: 4 bases over single-member genomes) equals `entropy15` under `--no-ambiguity`; a unit test checks it. + +- `entropy_annex.rs`: header `flags` (the former padding), bit 0 = `no_ambiguity`; old files have 0 and stay valid. One file per convention (`entropy.pent`, `entropy.noamb.pent`), so switching the flag builds the missing one without discarding the other. `EntropyAnnex::no_ambiguity()`; `iter_full_entropy` rejects a header that disagrees with the requested convention. +- `ensure_layer_entropy_annex` takes the list of conventions to build; one scan feeds every missing annex and the overlap accumulator. `sample_index` builds and reads the convention matching its `no_ambiguity` argument, removing the mismatch between the draw (previously always raw masks) and the alignment (masked). +- `--sibling-annex` builds both conventions and invalidates both on rebuild. `--shannon` no longer builds any annex as a prerequisite (it recomputes everything from its own scan). +- Checked on the 1x benchmark index: a `--no-ambiguity` run creates `entropy.noamb.pent` beside the existing `entropy.pent` in all 512 layers; workspace tests pass. +- Still to do, in this order: `--quartet` (per-family bit annex, same parameter), `--pair-balance` (mean of 1/shared-families over the carrier pairs; second overlap matrix for the single-member convention, annex header carrying a fingerprint of the matrix it came from). + +## Implemented: `--quartet` (2026-10-13) + +`--quartet` keeps only the families with at least two called states each carried by at least two genomes (`masking::is_quartet_informative` for the raw masks, `survives_quartet` for the exact check on masked states and `--exclude-genome`). Absence never counts as a state; with `--no-ambiguity`, multi-member genomes are left out of the count. The persisted annex is a plain `PersistentBitVec` (one bit per non-monomorphic family, compact ordinal) — no ad hoc format; the `--no-ambiguity` convention is carried by the file name (`quartet.pbiv`, `quartet.noamb.pbiv`) because that header has no room for a flag. It is a safe pre-filter: excluding genomes can only destroy a bipartition, never create one. + +Plumbing: `AnnexRequest` (which annexes, per convention) and `ensure_layer_family_annexes` (one shared scan for entropy, quartet and overlap) replace the entropy-only builder; the sampling parameters travel as one `SamplingSpec { free_loss, no_ambiguity, quartet, entropy_bias }` instead of a growing positional list. + +First test on the 1x benchmark testbed (3 seeds, exhaustive alignment, `GTR+G4+ASC`, species monophyly out of 3 per seed): without filter 1 / 0 / 0 correct monophylies; carriers >= 4 only: 3 / 1 / 2; `--quartet`: 2 / 2 / 3 (370, 148, 369 columns, all informative by construction). *Klebsiella* stays fragile (wrong in 2 seeds out of 3); two seeds still show a branch at the IQ-TREE ceiling of 10 with so few columns. Small testbed, three seeds: a direction, not a measurement. + +## Implemented: `--subsample` renamed `--n-sites N|all` (2026-10-13) + +`--subsample` read as "draw a subset", while `--pseudo-alignment`/`--sankoff` made it mandatory and accepted no way to say "everything" (the workaround was a number larger than the family count, e.g. `--subsample 100000000`). The option is now `--n-sites `: `all` is an explicit value, parsed into its own type (`NSitesArg::{All, N}`, like `GammaShapeArg`) rather than an in-band sentinel (negative number, `0`); `all` gives the exhaustive scan the `snp-*` path already ran when the option was omitted. Still mandatory for `--pseudo-alignment`/`--sankoff`; still optional for `snp-*` `--distance`. No alias is kept for the old name (the software is in active development). The benchmark commands and measurements recorded in the earlier sections of this file were run with the old name. + +## Implemented: `--pair-balance` (2026-10-13) + +Weight of a family = **mean** over the pairs of its carriers of 1/O (O = index-wide shared variable families of the pair, same `--no-ambiguity` convention): the simulation on the *Primulaceae* sample (above) had the mean flatten best (expected-overlap range 23 000× to ~600×, slope 0.37) without chasing high-carrier families, unlike the sum (65 % of the draw at >= 33 carriers). + +- `algorithms/pair_weight.rs`: `inverse_overlap_matrix` (dense n×n of 1/O from the persisted overlap annex), `pair_balance_weight` (mean over carrier pairs; 0 below two carriers). Carriers are defined once, `helpers::is_carrier(mask, no_ambiguity)`, shared with the overlap accumulator and the quartet test. +- Storage: no new format. The entropy annex is generalised to `FamilyScalarAnnex` with a `ScalarKind` (`Entropy` magic `PENT`, `PairWeight` magic `PPWT`, one file per kind and convention, header flag `no_ambiguity`); `entropy_annex.rs` became `family_scalar_annex.rs`. Family overlap is now one file per convention too (`family_overlap.povl`, `family_overlap.noamb.povl`, header flag); `--family-overlap` and `--min-shared-family` follow `--no-ambiguity`. +- Build order: sweep 1 builds entropy, quartet and the missing overlap annexes together; sweep 2 builds the pair weights, which need the finished matrix (`ensure_layer_pair_weight_annexes`). `AnnexRequest` now lists `entropy`, `quartet`, `overlap`, `pair_weight`; a pair weight implies its overlap. A rebuilt sibling annex removes every annex derived from it (entropy, quartet, pair weight, overlap), so a weight can never outlive the matrix it came from; there is no fingerprint, only that single invalidation point. +- Sampler: `LayerPlan` (eligible bits, minorant count, mass) replaces the positional tuple; a layer's quota is `n × mass / Σ mass` (mass = eligible count without the option, Σ weights with it); a family is accepted with probability `quota/mass × weight × entropy kernel`, capped at 1 (the cap, and kernels below 1, undershoot the quota, absorbed by the existing top-up rounds). +- Cost note: weights need O(carriers²) per family at build time; not measured on a large index. + +First test, 1x benchmark testbed (20 genomes, 46 706 variable families, `--n-sites 5000 --free-loss --no-ambiguity`, 8 replicates each; sample overlap computed from the alignment, compared with the index `--family-overlap --no-ambiguity`): + +| | slope of log(sample overlap) on log(index overlap) | pairs (index overlap > 0) covered by >= 1 column | columns with >= 10 carriers | +|---|---|---|---| +| uniform | 1.02 ± 0.14 | 64 % | 0.18 % | +| `--pair-balance` | 0.84 ± 0.001 | 97 % | 1.1 % | + +The weight flattens the representation (slope toward 0) and, above all, covers most of the pairs a uniform draw misses. Not measured: effect on the tree (sCF / monophyly), interaction with `--entropy`, behaviour on the *Primulaceae* index. 20 genomes only. + +## Quota rounding fix, and the combined test on the 1x testbed (2026-10-13) + +**Fix.** `sample_index` rounded each layer's quota down on its own, losing up to one site per layer: with 512 layers a budget of 100 sites gave 0 to 3 columns. Quotas now use cumulative rounding (`floor(n × cumulative mass / total)` minus what earlier layers got), so they sum to exactly `n`; a 100-site budget now yields 50-80 usable columns. Not a result of the options below, a defect the small testbed exposed. + +**Test.** 1x testbed, 3 seeds × 10 replicates per variant, 11 genomes (the 9 genomes with fewest calls excluded, so no all-missing sequence), `--free-loss --no-ambiguity`, IQ-TREE `GTR+G4+ASC` on each sampled alignment; score = number of the three species (*E. coli*, *Salmonella*, *Klebsiella*) recovered monophyletic (0-3), mean over the 30 runs; "informative" = columns with two bases each carried by >= 2 genomes. + +| `--n-sites` | variant | trees obtained | columns | informative columns | mean species monophyletic (of 3) | all 3 | +|---|---|---|---|---|---|---| +| 100 | uniform | 77 % | 49 | 0.5 | 0.04 | 0 % | +| 100 | `--quartet` | 100 % | 82 | 82 | 2.30 | 50 % | +| 100 | `--pair-balance` | 100 % | 52 | 2.1 | 0.20 | 0 % | +| 100 | `--quartet --pair-balance` | 100 % | 67 | 67 | 2.53 | 53 % | +| 100 | `... --entropy 1` | 100 % | 67 | 67 | 2.17 | 17 % | +| 300 | uniform | 93 % | 163 | 2.1 | 0.04 | 0 % | +| 300 | `--quartet` | 100 % | 239 | 239 | 2.33 | 33 % | +| 300 | `--pair-balance` | 100 % | 179 | 6.2 | 0.57 | 0 % | +| 300 | `--quartet --pair-balance` | 100 % | 197 | 197 | 2.67 | 67 % | +| 300 | `... --entropy 1` | 100 % | 196 | 196 | 2.67 | 67 % | + +(Mean over all runs counting a failed tree as 0 is identical to the "mean species" column wherever trees were obtained; the uniform rows lose runs because IQ-TREE refuses alignments where a genome has no data.) + +Reading: `--quartet` is the decisive option here — a uniform draw of 100-300 sites contains almost no informative column (0.5-2), the restricted draw contains only informative ones. `--pair-balance` alone barely helps (it favours rare pairs but does not remove the uninformative columns); added to `--quartet` it raises the share of replicates recovering all three species (50→53 % and 33→67 %). The entropy kernel (centre 1 bit, default width) adds nothing at 300 sites and costs at 100. Limits: 11 genomes, three simulated 1x datasets, small budgets, a single criterion (species monophyly, not deep branches), and the differences between the `--quartet` variants (a few runs out of 30) are small relative to the replicate noise — a direction, not a ranking. + ## Open questions - Per-family $S_f$ without materializing the full matrix: a first pass storing only a per-family summary (reach, $|S_f|$), as `--shannon` already does a full unsampled scan for entropy. diff --git a/UserDocMD/theory/phylogeny/snp_based/sampling_design.md b/UserDocMD/theory/phylogeny/snp_based/sampling_design.md index dbfa64c6..5c406d4d 100644 --- a/UserDocMD/theory/phylogeny/snp_based/sampling_design.md +++ b/UserDocMD/theory/phylogeny/snp_based/sampling_design.md @@ -13,12 +13,22 @@ A family's informativeness is measured directly rather than assumed uniform. For | Quantity | Definition | |---|---| | `entropy15` | Shannon entropy (bits) over the 16 possible states (the 15 non-empty subsets of `{A,C,G,T}`), genomes absent from the family excluded from the count | -| `entropy4` | the same, reduced to the 4 plain bases | +| `entropy4` | the same over the 4 plain bases, restricted to the genomes carrying exactly one member of the family (those that stay informative when multi-member genomes are treated as missing data) | + +With `--no-ambiguity` the entropy is `entropy4`, computed over the genomes carrying exactly one member of the family, so that the weighting sees the same genomes as the final alignment; without it, `entropy15` is used. Each is stored in its own annex. A family where every genome carries the same single state has entropy 0 (uninformative); one where genomes are spread evenly across several states has higher entropy (more informative for distinguishing genomes). +## Quartet restriction + +A family only constrains the shape of a tree if its carrier genomes show a bipartition: two distinct states, each carried by at least two genomes. With fewer carriers, or with a single variant against a uniform background, every topology explains the family equally well. In a skimming dataset many families have only two or three carriers, so a uniform draw spends a large share of its quota on them. `--quartet` removes them before the draw. The test counts called states only, so absence never forms one of the two pairs. + +## Pair balance + +How well a pair of genomes is represented in a sample depends on how many families the pair shares: pairs of distant genomes share few, so a uniform draw gives them few sites, and the deep branches of a tree — which rest on exactly those pairs — are the least supported. `--pair-balance` weights each family by the mean, over the pairs of genomes carrying it, of 1 / (families shared by the pair), computed on the whole index. Averaging rather than summing keeps the weight from growing with the number of carriers, which would favour the families carried by most genomes regardless of which pairs they inform. Layers are allotted their share of the draw in proportion to the total weight of their families. + ## Entropy-biased sampling By default, sampling draws families uniformly. Weighting instead by informativeness biases the draw toward a target entropy $\mu$ with a Gaussian curve of width $\sigma$: a family's probability of being kept scales with $\exp\!\left(-\dfrac{(\text{entropy15} - \mu)^2}{2\sigma^2}\right)$ — a soft preference, not a hard cutoff, so no family is categorically excluded purely for having low or high entropy. -See [`phylo`](../../../usage/phylo.md#sampling-at-scale---subsample---shannon---entropy) for how to bound and bias the sample, and for session/checkpointing behavior. +See [`phylo`](../../../usage/phylo.md#sampling-at-scale---n-sites---shannon---entropy) for how to bound and bias the sample, and for session/checkpointing behavior. diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md index f4b188ed..70c84389 100644 --- a/UserDocMD/usage/phylo.md +++ b/UserDocMD/usage/phylo.md @@ -27,7 +27,7 @@ obikmer phylo INDEX [OPTIONS] Every value routes to one of two independent computations: **whole-index distances** (`jaccard`, `mash`, `hamming`, `bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean` — see [Whole k-mer set distances](../theory/phylogeny/kmer_based/kmer_set_distances.md) for the definitions; `hamming` requires a presence/absence index, the others work on either index type), or **`snp-*` corrections** (`snp-raw`, `snp-jc`, `snp-k2p`, `snp-k81`, `snp-f81`, `snp-t92`, `snp-tn93`, `snp-tv` — see the [central-position SNP model](../theory/phylogeny/snp_based/central_snp_model.md) for the definitions). -`snp-*` values require the sibling annex (`--sibling-annex`, below) and are, by default, computed exhaustively over every non-monomorphic family in the index; add `--subsample N` to bound the computation to approximately `N` families instead (see "Sampling at scale" below — the same flag `--pseudo-alignment`/`--sankoff` use, but optional here). +`snp-*` values require the sibling annex (`--sibling-annex`, below) and are, by default, computed exhaustively over every non-monomorphic family in the index; add `--n-sites N` to bound the computation to approximately `N` families instead (see "Sampling at scale" below — the same option `--pseudo-alignment`/`--sankoff` use, but optional here). ### Output @@ -53,7 +53,7 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model | `--sibling-hist` | Print the global family-size histogram (1-4 members) only | | `--family-overlap` | Write `_family_overlap.csv`: for every genome pair, how many variable families both genomes carry a call for | | `--shannon` | Write `_entropy.csv`: per-family Shannon entropy, one row per family, full unsampled scan | -| `--pseudo-alignment` | Write `_alignment.fasta`: a SNP-only pseudo-alignment. Requires `--subsample N` | +| `--pseudo-alignment` | Write `_alignment.fasta`: a SNP-only pseudo-alignment. Requires `--n-sites N\|all` | ### `--sibling-stats` @@ -66,7 +66,7 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model ### Family Overlap -`--family-overlap` writes `_family_overlap.csv`: header `genome,,,...`, one row per genome, cell `[i][j]` = number of variable families (family size ≥ 2) where both genome `i` and genome `j` carry a call. The diagonal is always `0`. Every genome is written, unfiltered by `--exclude-genome`/`--min-shared-family`. +`--family-overlap` writes `_family_overlap.csv`: header `genome,,,...`, one row per genome, cell `[i][j]` = number of variable families (family size ≥ 2) where both genome `i` and genome `j` carry a call. The diagonal is always `0`. Every genome is written, unfiltered by `--exclude-genome`/`--min-shared-family`. With `--no-ambiguity`, a genome carrying several members of a family is not counted as carrying it, as in the alignment; without it, any genome with a call counts. The matrix is stored once per convention, so switching `--no-ambiguity` builds the other one without discarding the first (`--min-shared-family` follows the same convention). `--min-shared-family N` uses the mean of each genome's own row (excluding the diagonal) against this same matrix as its exclusion statistic. There is no universal value for `N` — inspect `--family-overlap`'s own output to find where the real gap sits in a given genome collection before choosing a threshold. @@ -76,15 +76,17 @@ See [Central-position SNP model](../theory/phylogeny/snp_based/central_snp_model | Option | Default | Description | |---|---|---| -| `--subsample N` | none (mandatory here) | Target number of families to sample | +| `--n-sites N\|all` | none (mandatory here) | Number of families to draw, or `all` to use every variable family that passes the other filters, with no sampling. A number is a target, not a guarantee. | | `--free-loss` | off | Treat a genome carrying none of a family's observed members as missing data (`?`) instead of `-` | | `--no-ambiguity` | off | Treat a genome carrying more than one member of a family as missing data (`?`) instead of an IUPAC ambiguity code | +| `--quartet` | off | Keep only the families that can separate two pairs of genomes from each other: at least two distinct called states, each carried by at least two genomes. Which genomes count as carriers follows `--no-ambiguity` | +| `--pair-balance` | off | Favour the families covering the pairs of genomes that share the fewest families across the index, so that every pair is better represented in the sample. Which genomes count as carriers follows `--no-ambiguity` | | `--entropy MU` | off (`1.0` if only `--entropy-sd` is given) | Center of the entropy band to favor when sampling | | `--entropy-sd SIGMA` | off (`0.5` if only `--entropy` is given) | Width of that band | -## Sampling at scale: `--subsample`, `--shannon`, `--entropy` +## Sampling at scale: `--n-sites`, `--shannon`, `--entropy` -`--subsample`, `--free-loss`, `--no-ambiguity`, `--entropy`/`--entropy-sd` are shared by `--pseudo-alignment`, `--sankoff` (and everything it implies: `--tnt`/`--phyg`/`--iqtree`), and a `snp-*` `--distance` value — one draw feeds all of them in a single invocation. `--subsample` is mandatory for `--pseudo-alignment`/`--sankoff`; for a `snp-*` `--distance` value it is optional (omitted means every non-monomorphic family in the index, not an approximation). +`--n-sites`, `--free-loss`, `--no-ambiguity`, `--quartet`, `--pair-balance`, `--entropy`/`--entropy-sd` are shared by `--pseudo-alignment`, `--sankoff` (and everything it implies: `--tnt`/`--phyg`/`--iqtree`), and a `snp-*` `--distance` value — one draw feeds all of them in a single invocation. `--n-sites` is mandatory for `--pseudo-alignment`/`--sankoff`, where it takes either a number of sites or `all`; for a `snp-*` `--distance` value it is optional, and omitting it is the same as `all` (every non-monomorphic family in the index, not an approximation). Combining `--sankoff` (or `--tnt`/`--phyg`/`--iqtree`) with a `snp-*` `--distance` value in the same command reuses that one draw for both — the distance and the Sankoff calibration/alignment are guaranteed to be computed from the *identical* set of sampled sites, never two independent samples, so the two outputs are directly comparable. This only holds within a single command; running them as two separate `obikmer phylo` invocations draws two independent samples even with the same flags — unless `--session` is used (below). @@ -97,13 +99,13 @@ Every invocation, with or without `--session`, draws its own fresh random sample | `--session DIR` | none | Persist the sample (and, for `--sankoff`, its calibration/alignment) in `DIR` so a later, separate `obikmer phylo` invocation with the exact same selection parameters restores it instead of resampling | | `--session-force` | off | With `--session DIR`: overwrite its saved parameters and cached sample instead of erroring out when this run's parameters don't match. No effect without `--session` | -`DIR` is created if it doesn't exist. If it already holds a sample built with different `--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/`--min-shared-family`/`--entropy`/`--entropy-sd` values than this run, the command exits with an error rather than silently using either the old or the new values — pass `--session-force` to discard the old sample and rebuild under the new parameters, or point `--session` at a different directory to keep both. +`DIR` is created if it doesn't exist. If it already holds a sample built with different `--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/`--min-shared-family`/`--entropy`/`--entropy-sd` values than this run, the command exits with an error rather than silently using either the old or the new values — pass `--session-force` to discard the old sample and rebuild under the new parameters, or point `--session` at a different directory to keep both. A `--sankoff`-family run and a `snp-*` `--distance` run share the same cached sample when pointed at the same `--session DIR` — build it once with either, reuse it from the other, in either order, across separate commands. If a run using `--session` is interrupted (crash, kill, `Ctrl-C`), the next run against the same `--session DIR` resumes from the last automatic checkpoint (roughly every 8 partitions'-worth of sampling progress) instead of starting over. The resumed run's sample is **not** guaranteed to be identical to what an uninterrupted run would have produced past that checkpoint — each process draws its own independent random sequence, same as any two separate invocations do — but nothing already checkpointed is lost, and no work needs redoing beyond that point. -Without `--subsample`, every variable family (family size ≥ 2) is used. With `--subsample N`, roughly `N` families are kept instead (see [Sampling design](../theory/phylogeny/snp_based/sampling_design.md) for how the draw stays representative of the whole index). If the index has fewer than `N` candidate families, `--subsample` has no effect. +With `--n-sites all` (or, for a `snp-*` `--distance` value, without `--n-sites`), every variable family (family size ≥ 2) that passes the other filters is used. With `--n-sites N`, roughly `N` families are kept instead (see [Sampling design](../theory/phylogeny/snp_based/sampling_design.md) for how the draw stays representative of the whole index). If fewer than `N` candidate families remain, `--n-sites` has no effect. ### `--shannon`: measuring how informative a family is @@ -115,23 +117,38 @@ Without `--subsample`, every variable family (family size ≥ 2) is used. With ` | `layer` | an internal index-layer identifier — stable within one run, not meaningful across indexes | | `family_idx` | the family's position within that layer | | `entropy15` | Shannon entropy (bits) over the 16 possible states (the 15 non-empty subsets of `{A,C,G,T}`), genomes absent from the family excluded from the count | -| `entropy4` | Shannon entropy (bits) reduced to the 4 plain bases, kept alongside `entropy15` for comparison | +| `entropy4` | Shannon entropy (bits) over the 4 plain bases, computed only over the genomes that carry exactly one member of the family — the genomes that remain informative under `--no-ambiguity`, since a genome carrying several members of a family (paralogous copies) is treated as missing data there. `0` when `n_genomes_unambiguous` is `0` | | `family_size` | number of distinct central bases observed anywhere in the index for this family (2-4) | -| `n_genomes_present` | how many genomes the entropy was computed over | +| `n_genomes_present` | how many genomes the family is present in; `entropy15` is computed over these | +| `n_genomes_unambiguous` | how many of those genomes carry exactly one member of the family; `entropy4` is computed over these | -`--shannon` always scans every non-monomorphic family of the whole index; `--subsample`, `--entropy` and `--entropy-sd` have no effect on it. The output therefore grows with the index, with one row per family, and can reach hundreds of millions of rows (tens of gigabytes) on a large index. Analyse it with a tool that reads the file as a stream (for example DuckDB or Miller) rather than loading it whole. It is nevertheless the right basis for choosing `--entropy`/`--entropy-sd` values, since it gives the complete entropy distribution, including how it varies with `n_genomes_present`, rather than an estimate from a sample. +`--shannon` always scans every non-monomorphic family of the whole index; `--n-sites`, `--entropy` and `--entropy-sd` have no effect on it. The output therefore grows with the index, with one row per family, and can reach hundreds of millions of rows (tens of gigabytes) on a large index. Analyse it with a tool that reads the file as a stream (for example DuckDB or Miller) rather than loading it whole. It is nevertheless the right basis for choosing `--entropy`/`--entropy-sd` values, since it gives the complete entropy distribution, including how it varies with `n_genomes_present`, rather than an estimate from a sample. + +### `--quartet`: keeping only families that can resolve a tree + +A family carried by fewer than four genomes, or whose genomes all share one state except a single variant, fits every tree topology equally well and so tells a tree nothing: to separate two pairs of genomes from each other (`AB|CD`) it takes at least two distinct states, each carried by at least two genomes. `--quartet` restricts the draw to the families that show this pattern. Absence is never a state here: a genome that does not carry the family does not count, with or without `--free-loss`. A genome carrying several members of the family counts as a carrier of its own combination of bases, unless `--no-ambiguity` is given, in which case it is left out, as in the alignment. + +The restriction is applied before the draw, from a persisted annex (one bit per variable family, built the first time `--quartet` is used with a given `--no-ambiguity` convention, in one pass shared with the other annexes, and built for both conventions by `--sibling-annex`), then checked exactly on each family drawn, taking `--exclude-genome` into account. The quota given to `--n-sites` is therefore spent only on usable families. + +### `--pair-balance`: representing every pair of genomes + +Two genomes that are distantly related share few families, so a uniform draw covers such a pair with few sites, far fewer than a close pair: the pairs that matter for the deep branches of a tree are the worst represented. `--pair-balance` compensates. Each family is weighted by the mean, over the pairs of genomes carrying it, of 1 divided by the number of families that pair shares across the whole index (the matrix `--family-overlap` writes): a family carried by a rarely-sharing pair weighs more. It is the mean, not the sum, so a family carried by most genomes — in skimming data often a high-copy region — is not favoured merely for covering many pairs. + +Each layer's share of the draw is proportional to the total weight of its families, and a family's chance of being drawn is proportional to its own weight. The weights are persisted (one value per variable family, built the first time `--pair-balance` is used with a given `--no-ambiguity` convention, after the family-overlap matrix it depends on, and by `--sibling-annex` for both conventions). They combine with `--entropy` and `--quartet`: the weight multiplies the entropy kernel, and `--quartet` restricts the families first. ### `--entropy MU` / `--entropy-sd SIGMA`: biasing the sample toward informative families -By default, `--subsample` draws families uniformly. With `--entropy`/`--entropy-sd`, each family's chance of being kept is instead weighted by how close its own entropy (`entropy15`) is to `MU`, using a Gaussian curve of width `SIGMA` — no hard cutoff (see [Sampling design](../theory/phylogeny/snp_based/sampling_design.md) for the weighting formula). The filter activates as soon as either flag is given; the other defaults to `1.0`/`0.5`. Combine with `--subsample N` (expect somewhat fewer than `N` families kept in practice) or use alone (a soft filter over the whole index, no size target). +By default, `--n-sites N` draws families uniformly. With `--entropy`/`--entropy-sd`, each family's chance of being kept is instead weighted by how close its own entropy (`entropy15`) is to `MU`, using a Gaussian curve of width `SIGMA` — no hard cutoff (see [Sampling design](../theory/phylogeny/snp_based/sampling_design.md) for the weighting formula). The filter activates as soon as either flag is given; the other defaults to `1.0`/`0.5`. Combine with `--n-sites N` (expect somewhat fewer than `N` families kept in practice) or use alone (a soft filter over the whole index, no size target). The first `phylo` run on a given index that uses `--entropy`/`--entropy-sd` pays a one-time extra cost (every candidate family's entropy is computed once and saved alongside the index); later runs, even with different `MU`/`SIGMA`, reuse that saved data. +The entropy that weights the draw follows `--no-ambiguity`, so that it describes the same genomes as the alignment. Without the flag, every genome carrying the family counts, and a genome carrying several members of it is a state of its own (`entropy15`). With the flag, only the genomes carrying exactly one member count, since the others become missing data (`entropy4`). The two entropies are saved separately: switching `--no-ambiguity` builds the other one once, without discarding the first, and `--sibling-annex` builds both. + ## Sankoff calibration and phylogenetic exports | Option | Default | Description | |---|---|---| -| `--sankoff` | off | Calibrate a 16-state parsimony cost matrix and matching pseudo-alignment. Requires `--subsample N` | +| `--sankoff` | off | Calibrate a 16-state parsimony cost matrix and matching pseudo-alignment. Requires `--n-sites N\|all` | | `--sankoff-ratio-ceiling` | `0.5` | Exclude genome pairs whose raw SNP ratio exceeds this value from the base-composition part of the calibration | | `--free-loss` | off | Recode a family's non-detection as the `?` missing-data symbol instead of an ordinary, costed state, throughout `--sankoff` and every export built from it | | `--tnt` | off | Also write a TNT script (implies `--sankoff`) | diff --git a/benchmark/simulate_skim.sh b/benchmark/simulate_skim.sh new file mode 100755 index 00000000..9ae11990 --- /dev/null +++ b/benchmark/simulate_skim.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Genome-skimming test bed: re-simulate the benchmark genomes at a low +# coverage, index each specimen, and merge them into one presence index. +# +# Usage: simulate_skim.sh (env: COVERAGE=1 SEED=1 CPUS=...) +# +# The genome → specimen mapping is read from deps.mk (same source of truth +# as the Makefile pipeline); the pipeline's own run/ tree is left untouched. +# Output: run/skim_${COVERAGE}x_s${SEED}/ +# simulated_data///reads_R{1,2}.fastq.gz +# specimen_index_presence/--/ +# global_index_presence/ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ISS="${SCRIPT_DIR}/../.venv/bin/iss" +BINARY="${SCRIPT_DIR}/../src/target/release/obikmer" +COVERAGE="${COVERAGE:-1}" +SEED="${SEED:-1}" +READ_LENGTH=150 +CPUS="${CPUS:-$(sysctl -n hw.logicalcpu 2>/dev/null || nproc 2>/dev/null || echo 2)}" + +OUT="${SCRIPT_DIR}/run/skim_${COVERAGE}x_s${SEED}" +mkdir -p "${OUT}/simulated_data" "${OUT}/specimen_index_presence" + +tmp_fasta=$(mktemp "${TMPDIR:-/tmp}/obikmer_skim_XXXXXX.fna") +trap 'rm -f "${tmp_fasta}"' EXIT + +# lines of the form run/simulated_data///reads_R1.fastq.gz: run/genomes/ +sed -nE 's#^run/simulated_data/([^/]+)/([^/]+)/reads_R1\.fastq\.gz: (run/genomes/[^ ]+)$#\1 \2 \3#p' \ + "${SCRIPT_DIR}/deps.mk" | sort -u | while read -r species strain genome; do + specimen="${species}--${strain}" + reads_dir="${OUT}/simulated_data/${species}/${strain}" + index_dir="${OUT}/specimen_index_presence/${specimen}" + mkdir -p "${reads_dir}" + + gzip -dc "${SCRIPT_DIR}/${genome}" > "${tmp_fasta}" + genome_size=$(grep -v "^>" "${tmp_fasta}" | tr -d '[:space:]' | wc -c | tr -d ' ') + n_reads=$(python3 -c "import math; print(math.ceil(${COVERAGE} * ${genome_size} / (2 * ${READ_LENGTH})))") + echo "[${specimen}] genome=${genome_size} bp → ${n_reads} read pairs (${COVERAGE}x, seed ${SEED})" + + "${ISS}" generate \ + --genomes "${tmp_fasta}" --model HiSeq --n_reads "${n_reads}" \ + --cpus "${CPUS}" --seed "${SEED}" --compress \ + --output "${reads_dir}/reads" 2>/dev/null + rm -f "${reads_dir}"/reads.iss.tmp.*.vcf + + "${BINARY}" index --output "${index_dir}" --force --theta 0 \ + --label "${specimen}" --meta "species=${species}" \ + "${reads_dir}/reads_R1.fastq.gz" "${reads_dir}/reads_R2.fastq.gz" 2>/dev/null +done + +mapfile -t sources < <(find "${OUT}/specimen_index_presence" -mindepth 1 -maxdepth 1 -type d | sort) +echo "Merging ${#sources[@]} presence indexes → ${OUT}/global_index_presence" +"${BINARY}" merge --output "${OUT}/global_index_presence" --force --force-presence \ + "${sources[@]}" 2>/dev/null +echo "Done: ${OUT}/global_index_presence" diff --git a/src/Cargo.lock b/src/Cargo.lock index beb2b3ff..c2816d1a 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1532,7 +1532,7 @@ dependencies = [ [[package]] name = "obikmer" -version = "1.3.5" +version = "1.3.6" dependencies = [ "clap", "csv", diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index e274c0ea..c736bf61 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "obikmer" -version = "1.3.5" +version = "1.3.6" edition = "2024" [[bin]] diff --git a/src/obikmer/src/cmd/phylo/args.rs b/src/obikmer/src/cmd/phylo/args.rs index 8226533f..a5cf659c 100644 --- a/src/obikmer/src/cmd/phylo/args.rs +++ b/src/obikmer/src/cmd/phylo/args.rs @@ -32,6 +32,36 @@ fn parse_gamma_shape(s: &str) -> Result { .map_err(|_| format!("invalid --gamma-shape value {s:?}: expected a positive number or \"auto\"/\"estimate\"")) } +/// `--n-sites` CLI value: a number of sites to draw, or `all` for every site +/// — the draw then keeps every variable family that passes the other filters, +/// with no sampling involved. Its own tiny type (like [`GammaShapeArg`]) so +/// the two cases stay distinct up to the call that needs the target, instead +/// of an in-band sentinel (a negative number, `0`, ...) that a typo could hit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NSitesArg { + All, + N(usize), +} + +impl NSitesArg { + /// The target to hand to the sampler: `None` = exhaustive. + pub fn target(self) -> Option { + match self { + NSitesArg::All => None, + NSitesArg::N(n) => Some(n), + } + } +} + +fn parse_n_sites(s: &str) -> Result { + if s.eq_ignore_ascii_case("all") { + return Ok(NSitesArg::All); + } + s.parse::() + .map(NSitesArg::N) + .map_err(|_| format!("invalid --n-sites value {s:?}: expected a non-negative integer or \"all\"")) +} + /// `--distance` value — either one of `obikphylo::DistanceMetric`'s /// whole-index metrics (routed to `IndexCache::distance`) or one of /// `obikphylo::SnpDistanceKind`'s `snp-*` corrections (routed to @@ -108,7 +138,7 @@ impl DistanceArg { /// (classic metrics + `snp-*` corrections/NJ/UPGMA), annex construction /// (`--sibling-annex`), annex diagnostics (`--sibling-stats`, /// `--sibling-hist`), entropy reporting (`--shannon`), SNP pseudo-alignment -/// sampling (`--pseudo-alignment`, `--subsample`, `--free-loss`, +/// sampling (`--pseudo-alignment`, `--n-sites`, `--free-loss`, /// `--no-ambiguity`, `--entropy`/`--entropy-sd`), Sankoff cost-matrix /// calibration (`--sankoff`, `--sankoff-ratio-ceiling`) and its TNT/PhyG/ /// IQ-TREE exports (`--tnt`, `--phyg`, `--iqtree`/`--iqtree-min-freq`, @@ -180,26 +210,25 @@ pub struct PhyloArgs { /// Write a per-family Shannon entropy report (CSV) — requires an /// already-built sibling annex (`--sibling-annex` first, in this /// invocation or an earlier one). Always a full, unsampled scan of - /// every family (`--subsample`/`--entropy`/`--entropy-sd` below only + /// every family (`--n-sites`/`--entropy`/`--entropy-sd` below only /// apply to `--pseudo-alignment`, not this). #[arg(long)] pub shannon: bool, /// Write a SNP-only pseudo-alignment (FASTA) — requires - /// `--subsample ` and an already-built sibling annex + /// `--n-sites ` and an already-built sibling annex /// (`--sibling-annex` first, in this invocation or an earlier one). #[arg(long)] pub pseudo_alignment: bool, - /// Target number of variable sites to sample index-wide for - /// `--pseudo-alignment`/`--sankoff` (mandatory for both) and for a - /// `snp-*` `--distance` value (optional there: omitted means exhaustive - /// — every non-monomorphic minorant of the whole index, not an - /// approximation) — a target, not a guarantee when given (proportional - /// per-layer sampling; see `obikphylo::siblings::sampled_scan`'s own - /// docs). - #[arg(long)] - pub subsample: Option, + /// Number of variable sites to draw index-wide, or `all` for every site + /// that passes the other filters (no sampling then). Mandatory for + /// `--pseudo-alignment`/`--sankoff`; optional for a `snp-*` + /// `--distance` value, where omitting it also means every site. A + /// number is a target, not a guarantee (proportional per-layer + /// sampling; see `obikphylo::siblings::sampled_scan`'s own docs). + #[arg(long = "n-sites", value_name = "N|all", value_parser = parse_n_sites)] + pub n_sites: Option, /// In `--pseudo-alignment`, treat a genome carrying none of a family's /// observed members (`∅`) as missing data (`?`) rather than a real @@ -213,6 +242,24 @@ pub struct PhyloArgs { #[arg(long)] pub no_ambiguity: bool, + /// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, keep only + /// the families that can separate two pairs of genomes from each other + /// (a bipartition): at least two distinct called states, each carried + /// by at least two genomes. A singleton variant, or a family carried by + /// fewer than four genomes, fits every topology and tells a tree + /// nothing. Which genomes count as carriers follows `--no-ambiguity`. + #[arg(long)] + pub quartet: bool, + + /// In `--pseudo-alignment`/`--sankoff`/`snp-*` `--distance`, favour the + /// families that cover the pairs of genomes sharing the fewest families + /// across the index: each family is weighted by the mean, over the pairs + /// of genomes carrying it, of 1 / (number of families the pair shares). + /// Evens out how well each pair of genomes is represented in the sample. + /// Which genomes count as carriers follows `--no-ambiguity`. + #[arg(long)] + pub pair_balance: bool, + /// Entropy-biased sampling target (Gaussian kernel mean) for /// `--pseudo-alignment` — activates biasing as soon as this or /// `--entropy-sd` is given; the other defaults to 1.0/0.5. @@ -227,7 +274,7 @@ pub struct PhyloArgs { /// Persist the `snp-*` `--distance` sample (the pairwise substitution /// tally) in `DIR` across separate `obikmer phylo` invocations: a later /// run against the same index with the exact same - /// `--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/ + /// `--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/ /// `--min-shared-family`/`--entropy`/`--entropy-sd` restores it instead /// of resampling. `DIR` is created if it doesn't exist. If it already /// holds a sample built under *different* selection parameters, this @@ -247,7 +294,7 @@ pub struct PhyloArgs { /// Calibrate a 16-state Sankoff cost matrix (and its matching /// pseudo-alignment) from an already-built sibling annex — requires - /// `--subsample `, and shares `--free-loss`/`--no-ambiguity`/ + /// `--n-sites `, and shares `--free-loss`/`--no-ambiguity`/`--quartet`/ /// `--entropy`/`--entropy-sd` with `--pseudo-alignment` (one draw, same /// selection feeds both the alignment and every calibration tally). #[arg(long)] diff --git a/src/obikmer/src/cmd/phylo/mod.rs b/src/obikmer/src/cmd/phylo/mod.rs index 43e451bf..eb95fa06 100644 --- a/src/obikmer/src/cmd/phylo/mod.rs +++ b/src/obikmer/src/cmd/phylo/mod.rs @@ -11,13 +11,14 @@ use std::sync::Arc; use obikidxcache::index_cache::IndexCache; use obikindex::KmerIndex; use obikphylo::siblings::{ - EntropyBias, GammaShape, SiblingExt, cardinality_transition_probs, + EntropyBias, GammaShape, SamplingSpec, SiblingExt, cardinality_transition_probs, composition_transition_probs, pairwise_cost_matrix, }; use obikphylo::{Metrics, neighbor_joining, upgma}; use obisys::{Reporter, Stage}; use tracing::info; +use args::NSitesArg; use iqtree::write_iqtree; use phyg::write_sankoff_phyg; use phylip::write_phylip_relaxed; @@ -154,7 +155,7 @@ pub fn run(args: PhyloArgs) { // ── Family Overlap matrix (`--family-overlap`) ────────────────────────────── if args.family_overlap { let t = Stage::start("family_overlap"); - let overlap = cache.family_overlap().unwrap_or_else(|e| { + let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| { eprintln!("error computing family overlap: {e}"); std::process::exit(1); }); @@ -201,7 +202,7 @@ pub fn run(args: PhyloArgs) { // `args::PhyloArgs::min_shared_family`'s own docs on why). let snp_exclude_mask: Vec = match args.min_shared_family { Some(threshold) => { - let overlap = cache.family_overlap().unwrap_or_else(|e| { + let overlap = cache.family_overlap(args.no_ambiguity).unwrap_or_else(|e| { eprintln!("error computing family overlap: {e}"); std::process::exit(1); }); @@ -235,6 +236,13 @@ pub fn run(args: PhyloArgs) { } else { None }; + let sampling = SamplingSpec { + free_loss: args.free_loss, + no_ambiguity: args.no_ambiguity, + quartet: args.quartet, + pair_balance: args.pair_balance, + entropy_bias, + }; // ── `--session` ────────────────────────────────────────────────────────── // Temporarily unavailable: the per-driver checkpoint/resume plumbing @@ -253,7 +261,7 @@ pub fn run(args: PhyloArgs) { // ── Sampled SNP scan (`--pseudo-alignment`, Sankoff group, `snp-*` `--distance`) ── // All three consume exactly the same selection parameters - // (`--subsample`/`--free-loss`/`--no-ambiguity`/`--exclude-genome`/ + // (`--n-sites`/`--free-loss`/`--no-ambiguity`/`--quartet`/`--exclude-genome`/ // `--min-shared-family`/`--entropy`/`--entropy-sd`), and must draw from // the *same* random sample when combined in one invocation — so // whichever of them are requested here share exactly one @@ -265,13 +273,15 @@ pub fn run(args: PhyloArgs) { let snp_kind = args.distance.as_snp(); let need_alignment = args.pseudo_alignment || sankoff_requested; let need_pairwise = sankoff_requested || snp_kind.is_some(); + // `None` = every site (`--n-sites all`, or `--n-sites` omitted where it is optional) + let target = args.n_sites.and_then(NSitesArg::target); - if args.pseudo_alignment && args.subsample.is_none() { - eprintln!("error: --pseudo-alignment requires --subsample "); + if args.pseudo_alignment && args.n_sites.is_none() { + eprintln!("error: --pseudo-alignment requires --n-sites "); std::process::exit(1); } - if sankoff_requested && args.subsample.is_none() { - eprintln!("error: --sankoff requires --subsample "); + if sankoff_requested && args.n_sites.is_none() { + eprintln!("error: --sankoff requires --n-sites "); std::process::exit(1); } @@ -290,13 +300,13 @@ pub fn run(args: PhyloArgs) { info!( "sampling SNP sites{}", - match args.subsample { + match target { Some(n) => format!(" (target {n} site(s))"), None => " (exhaustive)".into(), } ); let t = Stage::start("sampled_scan"); - obikphylo::siblings::sampled_scan(&cache, args.subsample, args.free_loss, args.no_ambiguity, &snp_exclude_mask, entropy_bias, &mut reducers) + obikphylo::siblings::sampled_scan(&cache, target, sampling, &snp_exclude_mask, &mut reducers) .unwrap_or_else(|e| { eprintln!("error sampling SNP sites: {e}"); std::process::exit(1); diff --git a/src/obikphylo/src/siblings/algorithms/entropy.rs b/src/obikphylo/src/siblings/algorithms/entropy.rs index 78c27c77..0e88bc6f 100644 --- a/src/obikphylo/src/siblings/algorithms/entropy.rs +++ b/src/obikphylo/src/siblings/algorithms/entropy.rs @@ -8,19 +8,6 @@ //! it is actually observed) is the consistent informativeness proxy for //! this project, not a 4-symbol reduction. -use std::collections::HashSet; -use std::path::Path; - -use obikidxcache::index_cache::IndexCache; -use obikindex::{OKIError, OKIResult}; - -use super::family_scan::{Selection, scan_layer_families}; -use super::minorant_selection::{is_non_monomorphic_minorant, non_monomorphic_reindex_layer}; -use crate::siblings::{ - ANNEX_FILE_NAME, ENTROPY_ANNEX_FILE_NAME, EntropyAnnex, EntropyAnnexBuilder, - FamilyOverlapAccumulator, SiblingAnnex, -}; - /// Shannon entropy (bits) of one family's per-genome states, over the 15 /// non-empty subsets of `{A,C,G,T}` actually observed among the genomes /// carrying it (`genome_mask[g] != 0`) — genomes where the family is absent @@ -103,6 +90,26 @@ mod tests { assert_eq!(n, 2); } + #[test] + fn entropy_15_over_single_member_genomes_equals_entropy_4() { + // `--no-ambiguity` leaves only the single-base states, over which the + // 15-state entropy and the 4-base one are the same quantity + let masks = [0b0001u8, 0b0001, 0b0100, 0b1000, 0b0100, 0b0100, 0b0010]; + let (h15, n15) = family_entropy(&masks).unwrap(); + let (h4, n4) = family_entropy_4(&masks).unwrap(); + assert!((h15 - h4).abs() < 1e-12); + assert_eq!(n15, n4); + } + + #[test] + fn entropy_under_selects_the_counting_convention() { + let masks = [0b0001u8, 0b0001, 0b0011]; // the last genome carries two members + assert_eq!(family_entropy_under(&masks, true).unwrap().1, 2); + assert_eq!(family_entropy_under(&masks, false).unwrap().1, 3); + assert_eq!(family_entropy_under(&masks, true).unwrap().0, 0.0); + assert!(family_entropy_under(&masks, false).unwrap().0 > 0.0); + } + #[test] fn entropy_4_is_none_without_unambiguous_genome() { assert!(family_entropy_4(&[0, 0b0011, 0b0110]).is_none()); @@ -110,104 +117,21 @@ mod tests { } } -/// Builds one layer's entropy annex if it doesn't already exist, and/or -/// feeds `overlap_acc` (`Some` iff the index-wide family-overlap annex is -/// itself missing — see -/// [`crate::siblings::FamilyOverlap`]'s own module docs) — a single shared -/// `scan_layer_families` pass covers both, since both need the exact same -/// per-family `genome_mask` resolution and there is no reason to pay for it -/// twice just because only one of the two happens to be missing. A no-op -/// (no scan at all) when entropy already exists for this layer *and* -/// `overlap_acc` is `None`. Bounded to non-monomorphic minorants only -/// ([`non_monomorphic_reindex_layer`], a cheap annex-only pass — no -/// cross-partition resolution) whenever a scan does happen: monomorphism is -/// already known from the annex bits alone, so there is no reason to pay -/// `scan_layer_families`'s expensive per-genome resolution for the ~98% of -/// minorants that are monomorphic only to discard the result — true for -/// family-overlap's own "variable families only" scope just as much as for -/// entropy's. -pub(crate) fn ensure_layer_entropy_annex( - cache: &IndexCache, - partition: usize, - layer_idx: usize, - n_genomes: usize, - fast_mode: bool, - mut overlap_acc: Option<&mut FamilyOverlapAccumulator>, -) -> OKIResult<()> { - let layer = cache - .get_layer(partition, layer_idx) - .expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)"); - let path = layer.dir().join(ENTROPY_ANNEX_FILE_NAME); - let need_entropy = !path.exists(); - if !need_entropy && overlap_acc.is_none() { - return Ok(()); - } - - let reindex = non_monomorphic_reindex_layer(layer.dir())?; - let eligible: HashSet = reindex.keys().copied().collect(); - let mut builder = need_entropy - .then(|| EntropyAnnexBuilder::new(reindex.len(), &path)) - .transpose() - .map_err(OKIError::Io)?; - - scan_layer_families( - cache, - partition, - layer_idx, - n_genomes, - fast_mode, - &Selection::Some(&eligible), - |family_idx, mask, genome_mask| { - debug_assert!( - mask.family_size() >= 2, - "Selection::Some(eligible) must only visit non-monomorphic minorants" - ); - if let Some(builder) = builder.as_mut() { - if let Some((h15, _)) = family_entropy(genome_mask) { - builder.set(reindex[&family_idx], h15 as f32); - } - } - if let Some(acc) = overlap_acc.as_deref_mut() { - acc.add_family(genome_mask); - } - }, - )?; - if let Some(builder) = builder { - builder.close().map_err(OKIError::Io)?; - } - - Ok(()) -} - -/// Streams one entropy value per minorant of the layer, **in `family_idx` -/// order** — monomorphic minorants included, valued `0.0` by convention -/// (trivially uninformative, never computed or stored in the compact -/// entropy annex). Exists so a caller can `.zip()` this directly against -/// any other `family_idx`-ordered stream (e.g. -/// `SiblingLayerExt::iter_minorants`) instead of separately consulting -/// [`non_monomorphic_reindex_layer`]'s map at each site. +/// Shannon entropy (bits) of one family under the counting convention +/// `--no-ambiguity` selects: +/// - `false`: [`family_entropy`] — every genome carrying the family counts, +/// a genome with several members being a state of its own; +/// - `true`: [`family_entropy_4`] — only the genomes carrying exactly one +/// member count, the others being missing data, exactly as the alignment +/// itself will see them. Over those genomes the 15-state entropy can only +/// ever take the four single-base states, so the two definitions coincide. /// -/// Requires this layer's entropy annex to already exist -/// ([`ensure_layer_entropy_annex`] first) — reads it positionally -/// (`compact` advances only on non-monomorphic minorants, mirroring -/// exactly how the annex was written), never recomputes anything. -pub(crate) fn iter_full_entropy(layer_dir: &Path) -> OKIResult + use<>> { - let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?; - let entropy_annex = - EntropyAnnex::open(&layer_dir.join(ENTROPY_ANNEX_FILE_NAME)).map_err(OKIError::Io)?; - - let mut compact = 0usize; - Ok((0..annex.len()).filter_map(move |slot| { - let mask = annex.get(slot)?; - if !mask.is_minorant() { - return None; // not a minorant at all — no entry in family_idx numbering either - } - if is_non_monomorphic_minorant(mask) { - let value = entropy_annex.get(compact).unwrap_or(0.0); - compact += 1; - Some(value) - } else { - Some(0.0) // monomorphic — trivially zero by convention, never stored - } - })) +/// Second value: the number of genomes the entropy was computed over. +pub(crate) fn family_entropy_under(genome_mask: &[u8], no_ambiguity: bool) -> Option<(f64, usize)> { + if no_ambiguity { + family_entropy_4(genome_mask) + } else { + family_entropy(genome_mask) + } } + diff --git a/src/obikphylo/src/siblings/algorithms/family_annexes.rs b/src/obikphylo/src/siblings/algorithms/family_annexes.rs new file mode 100644 index 00000000..ffcec3c5 --- /dev/null +++ b/src/obikphylo/src/siblings/algorithms/family_annexes.rs @@ -0,0 +1,212 @@ +//! Per-layer family annexes built from a `scan_layer_families` sweep: the +//! entropy annex and the pair-weight annex (both a +//! [`crate::siblings::FamilyScalarAnnex`]), the quartet annex (a +//! [`obicompactvec::PersistentBitVec`], see `crate::siblings::quartet_annex`) +//! and the index-level family-overlap accumulators. The entropy, quartet and +//! overlap builders all need the identical per-family `genome_mask` +//! resolution, so one sweep feeds every one that has something missing; the +//! pair weights need the *finished* overlap matrix, hence a second sweep +//! (`ensure_layer_pair_weight_annexes`). + +use std::collections::HashSet; +use std::io; +use std::path::PathBuf; + +use obicompactvec::PersistentBitVecBuilder; +use obikidxcache::index_cache::IndexCache; +use obikindex::{OKIError, OKIResult}; + +use super::entropy::family_entropy_under; +use super::family_scan::{Selection, scan_layer_families}; +use super::masking::is_quartet_informative; +use super::minorant_selection::non_monomorphic_reindex_layer; +use super::pair_weight::pair_balance_weight; +use crate::siblings::quartet_annex::quartet_annex_file_name; +use crate::siblings::{FamilyOverlapAccumulator, FamilyScalarAnnexBuilder, ScalarKind}; + +/// Which annexes a caller needs, each listed once per `--no-ambiguity` +/// counting convention (`false` = genomes carrying several members count as a +/// state of their own, `true` = they are left out) — an empty list asks for +/// none. A pair weight is computed from the family-overlap annex of its own +/// convention, so asking for one asks for the other. +#[derive(Clone, Debug, Default)] +pub(crate) struct AnnexRequest { + pub entropy: Vec, + pub quartet: Vec, + pub overlap: Vec, + pub pair_weight: Vec, +} + +impl AnnexRequest { + /// Everything under both conventions — what `--sibling-annex` precomputes. + pub(crate) fn everything() -> Self { + let both = vec![false, true]; + Self { entropy: both.clone(), quartet: both.clone(), overlap: both.clone(), pair_weight: both } + } + + /// The overlap conventions to ensure: the requested ones plus every one a + /// requested pair weight is computed from, each once. + pub(crate) fn overlap_conventions(&self) -> Vec { + dedup(self.overlap.iter().chain(&self.pair_weight).copied()) + } + + /// The pair-weight conventions requested, each once. + pub(crate) fn pair_weight_conventions(&self) -> Vec { + dedup(self.pair_weight.iter().copied()) + } +} + +fn dedup(variants: impl IntoIterator) -> Vec { + let mut v: Vec = variants.into_iter().collect(); + v.sort_unstable(); + v.dedup(); + v +} + +/// Builds the entropy and quartet annexes of one layer that `request` asks +/// for and that don't exist yet, and feeds `overlap_accs` (one accumulator +/// per family-overlap annex still missing, see +/// [`crate::siblings::FamilyOverlap`]'s own module docs), in one +/// `scan_layer_families` pass. A no-op (no scan at all) when everything +/// requested already exists for this layer *and* `overlap_accs` is empty. +/// Bounded to non-monomorphic minorants only +/// ([`non_monomorphic_reindex_layer`], a cheap annex-only pass — no +/// cross-partition resolution) whenever a scan does happen: monomorphism is +/// already known from the annex bits alone, so there is no reason to pay +/// `scan_layer_families`'s expensive per-genome resolution for the ~98% of +/// minorants that are monomorphic only to discard the result. +pub(crate) fn ensure_layer_family_annexes( + cache: &IndexCache, + partition: usize, + layer_idx: usize, + n_genomes: usize, + fast_mode: bool, + request: &AnnexRequest, + overlap_accs: &mut [(bool, FamilyOverlapAccumulator)], +) -> OKIResult<()> { + let layer = cache + .get_layer(partition, layer_idx) + .expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)"); + let missing = |name: &dyn Fn(bool) -> &'static str, variants: &[bool]| -> Vec<(bool, PathBuf)> { + dedup(variants.iter().copied()) + .into_iter() + .map(|no_ambiguity| (no_ambiguity, layer.dir().join(name(no_ambiguity)))) + .filter(|(_, path)| !path.exists()) + .collect() + }; + let missing_entropy = missing(&|nb| ScalarKind::Entropy.file_name(nb), &request.entropy); + let missing_quartet = missing(&quartet_annex_file_name, &request.quartet); + if missing_entropy.is_empty() && missing_quartet.is_empty() && overlap_accs.is_empty() { + return Ok(()); + } + + let reindex = non_monomorphic_reindex_layer(layer.dir())?; + let eligible: HashSet = reindex.keys().copied().collect(); + let mut entropy_builders = missing_entropy + .iter() + .map(|(no_ambiguity, path)| { + FamilyScalarAnnexBuilder::new(reindex.len(), path, ScalarKind::Entropy, *no_ambiguity) + .map(|b| (*no_ambiguity, b)) + }) + .collect::>>() + .map_err(OKIError::Io)?; + let mut quartet_builders = missing_quartet + .iter() + .map(|(no_ambiguity, path)| { + PersistentBitVecBuilder::new(reindex.len(), path).map(|b| (*no_ambiguity, b)) + }) + .collect::>>() + .map_err(OKIError::Io)?; + + scan_layer_families( + cache, + partition, + layer_idx, + n_genomes, + fast_mode, + &Selection::Some(&eligible), + |family_idx, mask, genome_mask| { + debug_assert!( + mask.family_size() >= 2, + "Selection::Some(eligible) must only visit non-monomorphic minorants" + ); + let compact = reindex[&family_idx]; + for (no_ambiguity, builder) in entropy_builders.iter_mut() { + if let Some((h, _)) = family_entropy_under(genome_mask, *no_ambiguity) { + builder.set(compact, h as f32); + } + } + for (no_ambiguity, builder) in quartet_builders.iter_mut() { + builder.set(compact, is_quartet_informative(genome_mask, *no_ambiguity)); + } + for (_, acc) in overlap_accs.iter_mut() { + acc.add_family(genome_mask); + } + }, + )?; + for (_, builder) in entropy_builders { + builder.close().map_err(OKIError::Io)?; + } + for (_, builder) in quartet_builders { + builder.close().map_err(OKIError::Io)?; + } + + Ok(()) +} + +/// Builds the pair-weight annexes of one layer that don't exist yet, one per +/// `(no_ambiguity, inverse overlap matrix)` in `matrices` (see +/// [`super::pair_weight::inverse_overlap_matrix`]), in one scan. A no-op (no +/// scan) when they all exist. Needs the family-overlap annex of each +/// convention to be complete: run after the sweep that builds it. +pub(crate) fn ensure_layer_pair_weight_annexes( + cache: &IndexCache, + partition: usize, + layer_idx: usize, + n_genomes: usize, + fast_mode: bool, + matrices: &[(bool, Vec)], +) -> OKIResult<()> { + let layer = cache + .get_layer(partition, layer_idx) + .expect("cache-consistent: caller already bounds layer_idx by n_layer(partition)"); + let pending: Vec<&(bool, Vec)> = matrices + .iter() + .filter(|(no_ambiguity, _)| !layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)).exists()) + .collect(); + if pending.is_empty() { + return Ok(()); + } + + let reindex = non_monomorphic_reindex_layer(layer.dir())?; + let eligible: HashSet = reindex.keys().copied().collect(); + let mut builders = pending + .iter() + .map(|(no_ambiguity, inverse)| { + let path = layer.dir().join(ScalarKind::PairWeight.file_name(*no_ambiguity)); + FamilyScalarAnnexBuilder::new(reindex.len(), &path, ScalarKind::PairWeight, *no_ambiguity) + .map(|b| (*no_ambiguity, inverse, b)) + }) + .collect::>>() + .map_err(OKIError::Io)?; + + scan_layer_families( + cache, + partition, + layer_idx, + n_genomes, + fast_mode, + &Selection::Some(&eligible), + |family_idx, _mask, genome_mask| { + let compact = reindex[&family_idx]; + for (no_ambiguity, inverse, builder) in builders.iter_mut() { + builder.set(compact, pair_balance_weight(genome_mask, *no_ambiguity, inverse)); + } + }, + )?; + for (_, _, builder) in builders { + builder.close().map_err(OKIError::Io)?; + } + + Ok(()) +} diff --git a/src/obikphylo/src/siblings/algorithms/masking.rs b/src/obikphylo/src/siblings/algorithms/masking.rs index afc788d6..b547f55b 100644 --- a/src/obikphylo/src/siblings/algorithms/masking.rs +++ b/src/obikphylo/src/siblings/algorithms/masking.rs @@ -17,6 +17,8 @@ //! waste sample quota on sites that die anyway once the excluded genome's //! data is dropped. +use crate::siblings::helpers::is_carrier; + /// This genome's masked character state, or `None` for "?" (missing). `m` /// is one `genome_mask` entry (bit `b` set iff this genome carries the /// family member whose own canonical central base is `b`; `0` = no member @@ -64,6 +66,97 @@ pub(crate) fn survives_masking(masked: &[Option], excluded: &[bool]) -> bool seen.count_ones() >= 2 } +/// Whether `counts` — the number of genomes per *called* state, indexed by +/// the `0..=15` state value, index `0` (absence) never counted — shows a +/// bipartition: at least two distinct states each carried by at least two +/// genomes. That is the pattern that can separate two pairs of genomes from +/// each other (`AB|CD`); a singleton variant, or a family carried by fewer +/// than four genomes, fits every topology and informs none. +fn has_bipartition(counts: &[u32; 16]) -> bool { + counts[1..].iter().filter(|&&c| c >= 2).count() >= 2 +} + +/// Quartet-informativeness of a family from its raw per-genome masks alone, +/// under the `--no-ambiguity` counting convention: absent genomes (`0`) +/// never count, and with `no_ambiguity` a genome carrying several members +/// doesn't either — otherwise its combination of bases is a state of its +/// own. Ignores `--exclude-genome` and `--free-loss` on purpose: absence is +/// never a called state here, and removing genomes can only destroy a +/// bipartition, never create one — so `false` is final, and `true` is only +/// a necessary condition, re-checked exactly by [`survives_quartet`] once +/// the family is resolved. This is what the persisted quartet annex stores. +pub(crate) fn is_quartet_informative(genome_mask: &[u8], no_ambiguity: bool) -> bool { + let mut counts = [0u32; 16]; + for &m in genome_mask { + if !is_carrier(m, no_ambiguity) { + continue; + } + counts[(m & 0b1111) as usize] += 1; + } + has_bipartition(&counts) +} + +/// Exact quartet check on an already-[`masked_state`]d family, over the +/// *included* genomes (`excluded` as in [`survives_masking`]). Only called +/// states count: a genome whose masked state is `0` (absence kept as a real +/// state, i.e. no `--free-loss`) is not a carrier of anything. +pub(crate) fn survives_quartet(masked: &[Option], excluded: &[bool]) -> bool { + let mut counts = [0u32; 16]; + for (g, &state) in masked.iter().enumerate() { + if excluded.get(g).copied().unwrap_or(false) { + continue; + } + if let Some(state) = state { + counts[(state & 0b1111) as usize] += 1; + } + } + has_bipartition(&counts) +} + +#[cfg(test)] +mod quartet_tests { + use super::*; + + #[test] + fn two_pairs_of_distinct_states_is_informative() { + assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], false)); + assert!(is_quartet_informative(&[0b0001, 0b0001, 0b0100, 0b0100], true)); + } + + #[test] + fn singleton_variant_is_not_informative() { + // three genomes in one state plus one lone variant: fits any topology + assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0001, 0b0100], false)); + // fewer than four carriers can never be informative + assert!(!is_quartet_informative(&[0b0001, 0b0100, 0, 0], false)); + assert!(!is_quartet_informative(&[0b0001, 0b0001, 0b0100], false)); + } + + #[test] + fn absent_genomes_are_not_a_state() { + // two carriers of one base, two absences: absence must not form the 2nd pair + assert!(!is_quartet_informative(&[0b0001, 0b0001, 0, 0], false)); + } + + #[test] + fn no_ambiguity_drops_multi_member_genomes_from_the_count() { + // the two-member genomes form a pair of their own state without the flag + let masks = [0b0001u8, 0b0001, 0b0011, 0b0011]; + assert!(is_quartet_informative(&masks, false)); + assert!(!is_quartet_informative(&masks, true)); + } + + #[test] + fn exact_check_honours_exclusions_and_missing() { + let masked = [Some(1u8), Some(1), Some(4), Some(4), None, Some(0)]; + assert!(survives_quartet(&masked, &[])); + // excluding one genome of the second pair breaks the bipartition + assert!(!survives_quartet(&masked, &[false, false, true])); + // an absence kept as state 0 (no --free-loss) is not a called state + assert!(!survives_quartet(&[Some(1), Some(1), Some(0), Some(0)], &[])); + } +} + /// IUPAC ambiguity code for a [`masked_state`] value (`0..=15`): single bit /// -> the plain base; 2 or 3 bits -> the matching IUPAC ambiguity code /// (preserves partial information instead of collapsing to `N`, the same diff --git a/src/obikphylo/src/siblings/algorithms/minorant_selection.rs b/src/obikphylo/src/siblings/algorithms/minorant_selection.rs index 8da4a33e..fa67698f 100644 --- a/src/obikphylo/src/siblings/algorithms/minorant_selection.rs +++ b/src/obikphylo/src/siblings/algorithms/minorant_selection.rs @@ -11,7 +11,7 @@ use std::path::Path; use obicompactvec::TempBitVecBuilder; use obikindex::{OKIError, OKIResult}; -use crate::siblings::{ANNEX_FILE_NAME, FamilyMask, SiblingAnnex}; +use crate::siblings::{ANNEX_FILE_NAME, FamilyMask, FamilyScalarAnnex, ScalarKind, SiblingAnnex}; pub(crate) fn is_non_monomorphic_minorant(mask: FamilyMask) -> bool { mask.is_minorant() && mask.siblings() >= 1 @@ -82,3 +82,49 @@ pub(crate) fn eligible_bitset_layer(layer_dir: &Path) -> OKIResult OKIResult + use<>> { + let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?; + let scalars = + FamilyScalarAnnex::open(&layer_dir.join(kind.file_name(no_ambiguity)), kind).map_err(OKIError::Io)?; + if scalars.no_ambiguity() != no_ambiguity { + return Err(OKIError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{kind:?} annex was built under the other --no-ambiguity convention"), + ))); + } + + let mut compact = 0usize; + Ok((0..annex.len()).filter_map(move |slot| { + let mask = annex.get(slot)?; + if !mask.is_minorant() { + return None; // not a minorant at all — no entry in family_idx numbering either + } + if is_non_monomorphic_minorant(mask) { + let value = scalars.get(compact).unwrap_or(0.0); + compact += 1; + Some(value) + } else { + Some(0.0) // monomorphic — trivially zero by convention, never stored + } + })) +} diff --git a/src/obikphylo/src/siblings/algorithms/mod.rs b/src/obikphylo/src/siblings/algorithms/mod.rs index 24b62f4c..1446c76b 100644 --- a/src/obikphylo/src/siblings/algorithms/mod.rs +++ b/src/obikphylo/src/siblings/algorithms/mod.rs @@ -17,10 +17,13 @@ mod alignment; mod annex; mod cardcomp; mod entropy; +mod family_annexes; mod family_scan; mod masking; mod minorant_selection; +mod pair_weight; mod pairwise; +mod quartet; mod sankoff; mod scan; mod snp_distance; @@ -38,10 +41,15 @@ pub use sankoff::SankoffBundle; pub use scan::{LayerReducer, sampled_scan}; pub use snp_distance::{GammaShape, SnpDistanceKind, distance_matrix}; pub use stats::SiblingAnnexStats; -pub use subsample::{EntropyBias, SurvivingFamily}; +pub use subsample::{EntropyBias, SamplingSpec, SurvivingFamily}; pub(crate) use annex::build_layer_sibling_annex; -pub(crate) use entropy::{ensure_layer_entropy_annex, family_entropy, family_entropy_4}; +pub(crate) use entropy::{family_entropy, family_entropy_4}; +pub(crate) use family_annexes::{ + AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes, +}; +pub(crate) use pair_weight::inverse_overlap_matrix; +pub(crate) use quartet::restrict_to_quartet; pub(crate) use family_scan::{Selection, scan_layer_families}; pub(crate) use minorant_selection::non_monomorphic_reindex_layer; pub(crate) use stats::{sibling_annex_stats, sibling_family_size_histogram}; diff --git a/src/obikphylo/src/siblings/algorithms/pair_weight.rs b/src/obikphylo/src/siblings/algorithms/pair_weight.rs new file mode 100644 index 00000000..3ba10eba --- /dev/null +++ b/src/obikphylo/src/siblings/algorithms/pair_weight.rs @@ -0,0 +1,121 @@ +//! `--pair-balance`: a per-family sampling weight that favours the families +//! covering the pairs of genomes the index shares the fewest families between. +//! +//! The weight of a family is the **mean**, over the pairs of genomes carrying +//! it, of `1 / O(pair)`, where `O` is the index-wide number of variable +//! families the pair shares (the family-overlap annex, under the same +//! `--no-ambiguity` counting convention). It is the mean, not the sum: a sum +//! grows with the square of the number of carriers, so it would chase the +//! families carried by most genomes — in a skimming dataset typically +//! high-copy regions, sampled deeply for reasons unrelated to phylogeny — +//! instead of the ones covering rare pairs. +//! +//! Values are only meaningful relative to each other; the sampler rescales +//! them per layer (`SamplingSpec::pair_balance`). + +use crate::siblings::FamilyOverlap; +use crate::siblings::helpers::is_carrier; + +/// `1 / O` for every pair of genomes, as a dense symmetric `n × n` matrix +/// (diagonal and pairs sharing nothing hold `0.0`) — what +/// [`pair_balance_weight`] reads, built once per run from the persisted +/// family-overlap annex. +pub(crate) fn inverse_overlap_matrix(overlap: &FamilyOverlap) -> Vec { + let n = overlap.n_genomes(); + let mut inv = vec![0.0f32; n * n]; + for i in 0..n { + for j in (i + 1)..n { + let shared = overlap.get(i, j); + if shared > 0 { + let r = 1.0 / shared as f32; + inv[i * n + j] = r; + inv[j * n + i] = r; + } + } + } + inv +} + +/// The pair-balance weight of one family from its raw per-genome masks: +/// mean of `inv_overlap` over the pairs of carriers (carriers per +/// [`is_carrier`], under `no_ambiguity`). `0.0` for a family with fewer than +/// two carriers — it covers no pair, so it can balance none. +pub(crate) fn pair_balance_weight( + genome_mask: &[u8], + no_ambiguity: bool, + inv_overlap: &[f32], +) -> f32 { + let n = genome_mask.len(); + let carriers: Vec = + (0..n).filter(|&g| is_carrier(genome_mask[g], no_ambiguity)).collect(); + let k = carriers.len(); + if k < 2 { + return 0.0; + } + let mut sum = 0.0f64; + for (a, &i) in carriers.iter().enumerate() { + for &j in &carriers[a + 1..] { + sum += f64::from(inv_overlap[i * n + j]); + } + } + (sum / (k * (k - 1) / 2) as f64) as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 4 genomes; pair (0,1) shares 2 families, (0,2) shares 4, the others 1. + fn matrix() -> Vec { + let n = 4; + let mut m = vec![0.0f32; n * n]; + let mut set = |i: usize, j: usize, shared: f32| { + m[i * n + j] = 1.0 / shared; + m[j * n + i] = 1.0 / shared; + }; + set(0, 1, 2.0); + set(0, 2, 4.0); + for (i, j) in [(0, 3), (1, 2), (1, 3), (2, 3)] { + set(i, j, 1.0); + } + m + } + + #[test] + fn single_pair_weight_is_the_inverse_overlap() { + let w = pair_balance_weight(&[1, 1, 0, 0], false, &matrix()); + assert!((w - 0.5).abs() < 1e-6); + } + + #[test] + fn weight_is_the_mean_not_the_sum_over_pairs() { + // carriers 0, 1, 2: pairs (0,1)=1/2, (0,2)=1/4, (1,2)=1 -> mean 7/12 + let w = pair_balance_weight(&[1, 1, 1, 0], false, &matrix()); + assert!((w - 7.0 / 12.0).abs() < 1e-6); + } + + #[test] + fn fewer_than_two_carriers_weigh_nothing() { + assert_eq!(pair_balance_weight(&[1, 0, 0, 0], false, &matrix()), 0.0); + assert_eq!(pair_balance_weight(&[0, 0, 0, 0], false, &matrix()), 0.0); + } + + #[test] + fn no_ambiguity_leaves_multi_member_genomes_out_of_the_pairs() { + let masks = [1u8, 1, 0b0011, 0]; + // without the flag genome 2 carries the family: pairs (0,1), (0,2), (1,2) + let with = pair_balance_weight(&masks, false, &matrix()); + // with it, only the pair (0,1) remains + let without = pair_balance_weight(&masks, true, &matrix()); + assert!((with - 7.0 / 12.0).abs() < 1e-6); + assert!((without - 0.5).abs() < 1e-6); + } + + #[test] + fn a_rare_pair_outweighs_a_common_one() { + // genomes (0,2) share many families, (1,3) few + let common = pair_balance_weight(&[1, 0, 1, 0], false, &matrix()); + let rare = pair_balance_weight(&[0, 1, 0, 1], false, &matrix()); + assert!(rare > common); + } +} diff --git a/src/obikphylo/src/siblings/algorithms/quartet.rs b/src/obikphylo/src/siblings/algorithms/quartet.rs new file mode 100644 index 00000000..667beaa1 --- /dev/null +++ b/src/obikphylo/src/siblings/algorithms/quartet.rs @@ -0,0 +1,45 @@ +//! `--quartet`: restricting a layer's eligible families to the +//! quartet-informative ones, from the persisted quartet annex. + +use std::path::Path; + +use obicompactvec::{PersistentBitVec, TempBitVecBuilder}; +use obikindex::{OKIError, OKIResult}; + +use super::minorant_selection::is_non_monomorphic_minorant; +use crate::siblings::quartet_annex::quartet_annex_file_name; +use crate::siblings::{ANNEX_FILE_NAME, SiblingAnnex}; + +/// Clears, in `eligible` (indexed by `family_idx`, monomorphic minorants +/// included), every family the quartet annex flags as not quartet-informative +/// under the `no_ambiguity` counting convention. The annex is indexed by the +/// compact non-monomorphic ordinal, so the sibling annex is walked in the same +/// order to translate one numbering into the other — annex-only, no +/// cross-partition resolution. The annex must already exist +/// ([`super::family_annexes::ensure_layer_family_annexes`] first). +pub(crate) fn restrict_to_quartet( + eligible: &mut TempBitVecBuilder, + layer_dir: &Path, + no_ambiguity: bool, +) -> OKIResult<()> { + let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).map_err(OKIError::Io)?; + let flags = PersistentBitVec::open(&layer_dir.join(quartet_annex_file_name(no_ambiguity))) + .map_err(OKIError::Io)?; + + let mut family_idx = 0usize; + let mut compact = 0usize; + for slot in 0..annex.len() { + let Some(mask) = annex.get(slot) else { continue }; + if !mask.is_minorant() { + continue; + } + if is_non_monomorphic_minorant(mask) { + if !flags.get(compact) { + eligible.set(family_idx, false); + } + compact += 1; + } + family_idx += 1; + } + Ok(()) +} diff --git a/src/obikphylo/src/siblings/algorithms/scan.rs b/src/obikphylo/src/siblings/algorithms/scan.rs index 99112fcc..f6cf217a 100644 --- a/src/obikphylo/src/siblings/algorithms/scan.rs +++ b/src/obikphylo/src/siblings/algorithms/scan.rs @@ -14,7 +14,7 @@ use obikidxcache::index_cache::IndexCache; use obikindex::OKIResult; use super::stats::sibling_family_size_histogram; -use super::subsample::{EntropyBias, SurvivingFamily, sample_index}; +use super::subsample::{SamplingSpec, SurvivingFamily, sample_index}; /// One accumulator fed every layer's [`SurvivingFamily`] batch during a /// [`sampled_scan`] pass — self-contained (no sampling/masking logic of @@ -42,26 +42,24 @@ fn resolve_target(cache: &IndexCache, n: Option) -> OKIResult { } } -/// Run one shared, possibly-subsampled/entropy-biased selection over -/// `cache` (`n`/`free_loss`/`no_ambiguity`/`excluded`/`entropy_bias` — see -/// [`sample_index`]'s own docs for what each controls), feeding every +/// Run one shared, possibly-subsampled/entropy-biased/quartet-restricted +/// selection over `cache` (`n`/`spec`/`excluded` — see [`sample_index`]'s +/// own docs and [`SamplingSpec`] for what each controls), feeding every /// surviving family of every visited layer to every reducer in /// `reducers`, in the same order, before moving to the next layer. A /// no-op (not an error) if the resolved target is `0`. pub fn sampled_scan( cache: &IndexCache, n: Option, - free_loss: bool, - no_ambiguity: bool, + spec: SamplingSpec, excluded: &[bool], - entropy_bias: Option, reducers: &mut [&mut dyn LayerReducer], ) -> OKIResult<()> { let target = resolve_target(cache, n)?; if target == 0 { return Ok(()); } - sample_index(cache, target, free_loss, no_ambiguity, excluded, entropy_bias, 0, |partition, layer, _raw_index, survivors| { + sample_index(cache, target, spec, excluded, 0, |partition, layer, _raw_index, survivors| { for reducer in reducers.iter_mut() { reducer.reduce(partition, layer, &survivors); } diff --git a/src/obikphylo/src/siblings/algorithms/subsample.rs b/src/obikphylo/src/siblings/algorithms/subsample.rs index c33cde1e..a2ced5bf 100644 --- a/src/obikphylo/src/siblings/algorithms/subsample.rs +++ b/src/obikphylo/src/siblings/algorithms/subsample.rs @@ -51,12 +51,11 @@ use obikidxcache::index_cache::IndexCache; use obikindex::OKIResult; use rand::RngExt; -use super::entropy::iter_full_entropy; use super::family_scan::{Selection, scan_layer_families}; -use super::is_fast_mode; -use super::masking::{masked_state, survives_masking}; -use super::minorant_selection::eligible_bitset_layer; -use crate::siblings::FamilyMask; +use super::masking::{masked_state, survives_masking, survives_quartet}; +use super::minorant_selection::{eligible_bitset_layer, iter_full_scalar}; +use super::{AnnexRequest, is_fast_mode, restrict_to_quartet}; +use crate::siblings::{FamilyMask, ScalarKind}; use crate::siblings::extensions::SiblingBuilder; /// Resolution rounds a single layer's sampling may take before giving up on @@ -96,21 +95,61 @@ pub struct EntropyBias { pub sigma: f64, } -/// [`iter_full_entropy`], rotated to start at `start` and wrap once back to -/// it — the same `family_idx` order [`sample_layer`]'s own circular index -/// sequence walks, so the two can be `.zip()`-ed directly. Two more streamed -/// passes over the layer's (small) annex/entropy files, not a materialised -/// `Vec`/`HashMap` — `iter_full_entropy` itself never allocates more than -/// its own iterator state. -fn circular_entropy_values( +/// How a draw is conditioned: the character masking (`--free-loss`, +/// `--no-ambiguity`), the quartet restriction (`--quartet`: only families +/// that can separate two pairs of genomes) and the optional entropy bias. +/// `--no-ambiguity` also selects the counting convention of the persisted +/// entropy and quartet annexes the draw reads, so the draw and the alignment +/// it feeds always describe the same genomes. `pub`, not `pub(crate)`: part +/// of the public signature of [`super::scan::sampled_scan`]. +#[derive(Clone, Copy)] +pub struct SamplingSpec { + pub free_loss: bool, + pub no_ambiguity: bool, + pub quartet: bool, + /// `--pair-balance`: weight each family by the mean, over the pairs of + /// genomes carrying it, of 1 / (number of families the pair shares), so + /// the draw favours the families covering the rarest pairs (see + /// `algorithms::pair_weight`). + pub pair_balance: bool, + pub entropy_bias: Option, +} + +/// [`iter_full_scalar`] of `kind` (entropy, pair weight), rotated to start at +/// `start` and wrap once back to it — the same `family_idx` order +/// [`sample_layer`]'s own circular index sequence walks, so the streams can be +/// `.zip()`-ed directly. Two more streamed passes over the layer's (small) +/// annex files, not a materialised `Vec`/`HashMap` — `iter_full_scalar` +/// itself never allocates more than its own iterator state. `no_ambiguity` +/// selects the annex built under the same counting convention as the masking +/// the survivors go through afterwards. +fn circular_scalar_values( layer_dir: &Path, + kind: ScalarKind, start: usize, + no_ambiguity: bool, ) -> OKIResult + use<>> { - let head = iter_full_entropy(layer_dir)?.skip(start); - let tail = iter_full_entropy(layer_dir)?.take(start); + let head = iter_full_scalar(layer_dir, kind, no_ambiguity)?.skip(start); + let tail = iter_full_scalar(layer_dir, kind, no_ambiguity)?.take(start); Ok(head.chain(tail)) } +/// One layer's place in a draw, planned before any layer is sampled: which of +/// its families may be drawn, and how much of the index-wide quota it +/// deserves. +struct LayerPlan { + partition: usize, + layer: usize, + /// One bit per minorant (`family_idx` order): still eligible to be drawn. + eligible: TempBitVecBuilder, + n_minorants: usize, + /// Total sampling weight of the eligible families: their number, or, with + /// `--pair-balance`, the sum of their pair weights. A layer's quota is its + /// share of this mass over every layer, so a layer rich in families that + /// cover rare pairs gets more of the draw. + mass: f64, +} + /// Samples up to `n` non-monomorphic families index-wide, proportionally /// per layer, calling `on_layer(partition, layer, survivors)` once per /// cached layer that contributed anything, with every family from that @@ -159,10 +198,8 @@ fn circular_entropy_values( pub(crate) fn sample_index( cache: &IndexCache, n: usize, - free_loss: bool, - no_ambiguity: bool, + spec: SamplingSpec, excluded: &[bool], - entropy_bias: Option, skip_layers: usize, mut on_layer: impl FnMut(usize, usize, usize, Arc>), ) -> OKIResult { @@ -170,64 +207,79 @@ pub(crate) fn sample_index( let fast_mode = is_fast_mode(cache); // Entropy-biased mode reads each candidate's entropy from the entropy - // annex (`EntropyWeights::open`, per layer, below) — build any missing - // one now, once, rather than mid-walk. - if entropy_bias.is_some() { - cache.ensure_entropy_and_overlap_annexes()?; + // annex, `--quartet` restricts the candidates through the quartet annex, + // and `--pair-balance` weights them by the pair-weight annex (all per + // layer, below, each under this run's `--no-ambiguity` counting + // convention) — build any missing one now, once, rather than mid-walk. + let request = AnnexRequest { + entropy: spec.entropy_bias.map(|_| vec![spec.no_ambiguity]).unwrap_or_default(), + quartet: if spec.quartet { vec![spec.no_ambiguity] } else { Vec::new() }, + overlap: Vec::new(), + pair_weight: if spec.pair_balance { vec![spec.no_ambiguity] } else { Vec::new() }, + }; + if !request.entropy.is_empty() || !request.quartet.is_empty() || !request.pair_weight.is_empty() { + cache.ensure_family_annexes(&request)?; } - // Every cached layer's eligibility bitset, built up front — annex-only, - // no cross-partition resolution, so cheap even summed over the whole - // index — needed before any layer's quota can be computed (each - // layer's share depends on every other layer's own count). - let mut layers: Vec<(usize, usize, TempBitVecBuilder, usize)> = Vec::new(); - for part in cache.partitions() { - let n_layer = cache.n_layer(part).unwrap_or(0); - for l in 0..n_layer { + // Every cached layer's plan, built up front — annex-only, no + // cross-partition resolution, so cheap even summed over the whole index + // — needed before any layer's quota can be computed (each layer's share + // depends on every other layer's own mass). + let mut plans: Vec = Vec::new(); + for partition in cache.partitions() { + let n_layer = cache.n_layer(partition).unwrap_or(0); + for layer_idx in 0..n_layer { let layer = cache - .get_layer(part, l) - .expect("cache-consistent: n_layer(part) already bounds l"); - let eligible = eligible_bitset_layer(layer.dir())?; + .get_layer(partition, layer_idx) + .expect("cache-consistent: n_layer(partition) already bounds layer_idx"); + let mut eligible = eligible_bitset_layer(layer.dir())?; + if spec.quartet { + restrict_to_quartet(&mut eligible, layer.dir(), spec.no_ambiguity)?; + } let n_minorants = eligible.view().len(); - layers.push((part, l, eligible, n_minorants)); + let mass = if spec.pair_balance { + iter_full_scalar(layer.dir(), ScalarKind::PairWeight, spec.no_ambiguity)? + .enumerate() + .filter(|&(family_idx, _)| eligible.view().get(family_idx)) + .map(|(_, weight)| f64::from(weight)) + .sum() + } else { + eligible.view().count_ones() as f64 + }; + plans.push(LayerPlan { partition, layer: layer_idx, eligible, n_minorants, mass }); } } - let total_eligible: u64 = layers - .iter() - .map(|(_, _, e, _)| e.view().count_ones()) - .sum(); - if total_eligible == 0 { + let total_mass: f64 = plans.iter().map(|plan| plan.mass).sum(); + if total_mass <= 0.0 { return Ok(0); } + // Quotas by cumulative rounding: layer `i` is allotted + // `floor(n × cumulative mass up to i / total) − what the previous layers + // were allotted`, so the quotas sum to exactly `n`. Rounding each layer's + // share down on its own would lose up to one site per layer — all of + // them, when `n` is smaller than the number of layers. let mut total_kept = 0usize; - for (raw_index, (part, l, mut eligible, n_minorants)) in layers.into_iter().enumerate() { - if raw_index < skip_layers { - continue; - } - let count_layer = eligible.view().count_ones(); - if count_layer == 0 { - continue; - } - let quota = ((n as u128 * count_layer as u128) / total_eligible as u128) as usize; - if quota == 0 { + let mut cumulative_mass = 0.0f64; + let mut allotted = 0usize; + for (raw_index, mut plan) in plans.into_iter().enumerate() { + cumulative_mass += plan.mass; + let cumulative_quota = (n as f64 * cumulative_mass / total_mass) as usize; + let quota = cumulative_quota.saturating_sub(allotted); + allotted = allotted.max(cumulative_quota); + if raw_index < skip_layers || plan.mass <= 0.0 || quota == 0 { continue; } total_kept += sample_layer( cache, - part, - l, raw_index, - &mut eligible, - n_minorants, + &mut plan, quota, n_genomes, fast_mode, - free_loss, - no_ambiguity, + spec, excluded, - entropy_bias, &mut on_layer, )?; } @@ -236,32 +288,28 @@ pub(crate) fn sample_index( } /// One layer's share of [`sample_index`] — see the module docs for the -/// per-family accept/reject/mark rules and the round-batching scheme. -#[allow(clippy::too_many_arguments)] +/// per-family accept/reject/mark rules and the round-batching scheme. Each +/// eligible family is accepted with probability +/// `quota / mass × pair weight × entropy kernel` (capped at 1): the pair +/// weight is `1` and `mass` the number of eligible families without +/// `--pair-balance`, and the kernel `1` without an entropy bias. fn sample_layer( cache: &IndexCache, - partition: usize, - layer_idx: usize, raw_index: usize, - eligible: &mut TempBitVecBuilder, - n_minorants: usize, + plan: &mut LayerPlan, quota: usize, n_genomes: usize, fast_mode: bool, - free_loss: bool, - no_ambiguity: bool, + spec: SamplingSpec, excluded: &[bool], - entropy_bias: Option, on_layer: &mut impl FnMut(usize, usize, usize, Arc>), ) -> OKIResult { - if n_minorants == 0 { + let LayerPlan { partition, layer: layer_idx, n_minorants, mass, .. } = *plan; + let eligible = &mut plan.eligible; + if n_minorants == 0 || mass <= 0.0 { return Ok(0); } - let count_layer = eligible.view().count_ones() as usize; - if count_layer == 0 { - return Ok(0); - } - let p0 = (quota as f64 / count_layer as f64).min(1.0); + let p0 = quota as f64 / mass; let layer_dir = cache .get_layer(partition, layer_idx) .expect("cache-consistent") @@ -301,34 +349,50 @@ fn sample_layer( // This round's own circular walk, resuming exactly where the // previous round left off (`start + visited`, wrapped) — zipped - // against that same rotation of the entropy stream when biased, so - // each candidate's own kernel weight is read in lockstep, never via - // a separate lookup structure. Unbiased draws skip the entropy - // stream entirely (a flat `1.0` kernel), never touching the entropy - // annex. + // against that same rotation of the entropy and pair-weight streams + // when they are used, so each candidate's own kernel and weight are + // read in lockstep, never via a separate lookup structure. A draw + // without the corresponding option skips its stream entirely (flat + // `1.0`), never touching the annex. let round_start = (start + visited) % n_minorants; - let entropies: Box> = match entropy_bias { - Some(_) => Box::new(circular_entropy_values(&layer_dir, round_start)?), + let entropies: Box> = match spec.entropy_bias { + Some(_) => Box::new(circular_scalar_values( + &layer_dir, + ScalarKind::Entropy, + round_start, + spec.no_ambiguity, + )?), None => Box::new(std::iter::repeat(0.0f32)), // never read: kernel forced to 1.0 below }; + let weights: Box> = if spec.pair_balance { + Box::new(circular_scalar_values( + &layer_dir, + ScalarKind::PairWeight, + round_start, + spec.no_ambiguity, + )?) + } else { + Box::new(std::iter::repeat(1.0f32)) + }; let candidates = (round_start..n_minorants) .chain(0..round_start) - .zip(entropies); + .zip(entropies) + .zip(weights); let mut accepted: HashSet = HashSet::new(); - for (family_idx, entropy) in candidates { + for ((family_idx, entropy), weight) in candidates { visited += 1; if !eligible.view().get(family_idx) { continue; } - let kernel = match entropy_bias { + let kernel = match spec.entropy_bias { Some(bias) => { let d = (entropy as f64 - bias.mu) / bias.sigma; (-0.5 * d * d).exp() } None => 1.0, }; - if rng.random::() < (p0 * kernel).min(1.0) { + if rng.random::() < (p0 * f64::from(weight) * kernel).min(1.0) { eligible.set(family_idx, false); accepted.insert(family_idx); if accepted.len() >= target_this_round { @@ -353,9 +417,13 @@ fn sample_layer( mask.family_size() >= 2, "eligible bitset must only accept non-monomorphic minorants" ); - let masked: Vec> = - genome_mask.iter().map(|&m| masked_state(m, free_loss, no_ambiguity)).collect(); - if survives_masking(&masked, excluded) { + let masked: Vec> = genome_mask + .iter() + .map(|&m| masked_state(m, spec.free_loss, spec.no_ambiguity)) + .collect(); + if survives_masking(&masked, excluded) + && (!spec.quartet || survives_quartet(&masked, excluded)) + { survivors.push(SurvivingFamily { family_idx, mask, masked }); kept += 1; } diff --git a/src/obikphylo/src/siblings/entropy_annex.rs b/src/obikphylo/src/siblings/entropy_annex.rs deleted file mode 100644 index fce5dfd9..00000000 --- a/src/obikphylo/src/siblings/entropy_annex.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! Persisted, per-layer Shannon entropy (`entropy15`, see -//! `algorithms::entropy`), one value per **non-monomorphic minorant -//! family** — compact: indexed by the dedicated ordinal -//! `algorithms::non_monomorphic_reindex_layer` computes (skips monomorphic -//! minorants entirely, ~98% of minorants on real data), never by -//! `scan_layer_families`'s own `family_idx` (which counts every minorant, -//! monomorphic included). -//! -//! Exists so repeated reads of the same layer's entropy values (e.g. -//! entropy-biased family selection, once reconnected) are a plain -//! positional mmap lookup instead of re-paying `scan_layer_families`'s -//! cross-partition resolution — building this file costs exactly that same -//! full scan the first time, it does not make any single computation of -//! entropy itself faster. -//! -//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat -//! per-entry data, read-only after build), just with `f32` payloads and a -//! different (compact) numbering — see the module docs above. - -use std::fs::{File, OpenOptions}; -use std::io; -use std::path::Path; - -use memmap2::{Mmap, MmapMut}; - -const MAGIC: [u8; 4] = *b"PENT"; - -// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows. -const HEADER_SIZE: usize = 16; - -/// Sentinel: not yet computed. Entropy is always `>= 0.0`, so any negative -/// value is unambiguous — chosen over `NaN` to avoid `NaN`-comparison -/// footguns at every read site. -const SENTINEL: f32 = -1.0; - -pub(crate) const ENTROPY_ANNEX_FILE_NAME: &str = "entropy.pent"; - -pub struct EntropyAnnex { - mmap: Mmap, - n: usize, -} - -impl EntropyAnnex { - pub fn open(path: &Path) -> io::Result { - let mmap = unsafe { Mmap::map(&File::open(path)?)? }; - if mmap.len() < HEADER_SIZE { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "PENT file too short", - )); - } - if mmap[0..4] != MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PENT magic")); - } - let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; - if mmap.len() < HEADER_SIZE + n * 4 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "PENT file truncated", - )); - } - Ok(Self { mmap, n }) - } - - pub fn len(&self) -> usize { - self.n - } - - /// `None` for the sentinel (not yet computed — shouldn't happen against - /// a fully-built annex). - pub fn get(&self, idx: usize) -> Option { - let off = HEADER_SIZE + idx * 4; - let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap()); - (v >= 0.0).then_some(v) - } -} - -pub struct EntropyAnnexBuilder { - mmap: MmapMut, -} - -impl EntropyAnnexBuilder { - /// Create a new annex of `n` entries (a layer's non-monomorphic - /// minorant count) at `path`, pre-initialised to the sentinel. - pub(crate) fn new(n: usize, path: &Path) -> io::Result { - let file_size = HEADER_SIZE + n * 4; - let file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(path)?; - file.set_len(file_size as u64)?; - let mut mmap = unsafe { MmapMut::map_mut(&file)? }; - mmap[0..4].copy_from_slice(&MAGIC); - mmap[4..8].copy_from_slice(&[0u8; 4]); - mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); - for i in 0..n { - let off = HEADER_SIZE + i * 4; - mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes()); - } - Ok(Self { mmap }) - } - - pub(crate) fn set(&mut self, idx: usize, value: f32) { - debug_assert!(value >= 0.0, "entropy must be non-negative, got {value}"); - let off = HEADER_SIZE + idx * 4; - self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes()); - } - - pub(crate) fn close(self) -> io::Result<()> { - self.mmap.flush() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn sentinel_is_negative_and_unset_entries_read_as_none() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let builder = EntropyAnnexBuilder::new(4, &path).unwrap(); - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - for i in 0..4 { - assert_eq!(annex.get(i), None); - } - } - - #[test] - fn roundtrip_including_zero_entropy() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let mut builder = EntropyAnnexBuilder::new(4, &path).unwrap(); - let values = [0.0f32, 1.5, 3.9, 0.0001]; - for (i, &v) in values.iter().enumerate() { - builder.set(i, v); - } - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - for (i, &v) in values.iter().enumerate() { - assert_eq!(annex.get(i), Some(v), "entry {i}"); - } - } - - #[test] - fn mixed_sentinel_and_real_values() { - let dir = tempdir().unwrap(); - let path = dir.path().join("test.pent"); - let mut builder = EntropyAnnexBuilder::new(3, &path).unwrap(); - builder.set(1, 2.0); - builder.close().unwrap(); - let annex = EntropyAnnex::open(&path).unwrap(); - assert_eq!(annex.get(0), None); - assert_eq!(annex.get(1), Some(2.0)); - assert_eq!(annex.get(2), None); - assert_eq!(annex.len(), 3); - } -} diff --git a/src/obikphylo/src/siblings/extensions/sibling_builder.rs b/src/obikphylo/src/siblings/extensions/sibling_builder.rs index 1464917e..7d2cb0d4 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_builder.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_builder.rs @@ -1,7 +1,8 @@ //! [`SiblingBuilder`] — construction triggered *internally*, as a side //! effect of a [`crate::siblings::extensions::SiblingExt`] method (e.g. -//! `shannon_entropy_csv`/`family_overlap` ensuring their annexes exist -//! before reading them), never called directly from outside this crate. +//! `family_overlap` ensuring its annex exists before reading it, or the +//! sampler ensuring the entropy/quartet/pair-weight annexes it draws from), +//! never called directly from outside this crate. //! Unlike `SiblingExt` (explicit, CLI-facing actions), nothing here is //! meant to be its own CLI flag — same `pub`/`pub(crate)` split //! `obikindexer` uses for its own `IndexBuilder`/`PrivateBuilder`. @@ -10,44 +11,75 @@ use obikidxcache::index_cache::IndexCache; use obikindex::{OKIError, OKIResult}; use obisys::progress_bar; -use crate::siblings::algorithms::{ensure_layer_entropy_annex, is_fast_mode}; -use crate::siblings::{FAMILY_OVERLAP_FILE_NAME, FamilyOverlapAccumulator}; +use crate::siblings::algorithms::{ + AnnexRequest, ensure_layer_family_annexes, ensure_layer_pair_weight_annexes, + inverse_overlap_matrix, is_fast_mode, +}; +use crate::siblings::{FamilyOverlap, FamilyOverlapAccumulator, family_overlap_file_name}; pub(crate) trait SiblingBuilder { - /// Build every cached layer's entropy annex, and the whole index's - /// family-overlap annex, that don't already exist — one call covers - /// both, whichever of the two a caller actually needs - /// (`shannon_entropy_csv`/entropy-biased sampling on one side, - /// `family_overlap`/`--min-shared-family` on the other): a missing - /// annex always triggers a scan that feeds both, per - /// `ensure_layer_entropy_annex`'s own docs. A no-op, cheap - /// existence-check pass, once both already exist everywhere. - fn ensure_entropy_and_overlap_annexes(&self) -> OKIResult<()>; + /// Build, for every cached layer, the annexes `request` asks for that + /// don't already exist (entropy, quartet and pair weight, each per + /// `--no-ambiguity` counting convention), and the index-level + /// family-overlap annexes it asks for or that a pair weight is computed + /// from. One call covers all of them, whichever a caller actually needs: + /// a missing entropy, quartet or overlap annex triggers one scan that + /// feeds every missing one, per `ensure_layer_family_annexes`'s own docs, + /// then a second scan builds the missing pair weights, which need the + /// finished overlap matrix. A no-op, cheap existence-check pass, once + /// everything requested already exists. + fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()>; } impl SiblingBuilder for IndexCache { - fn ensure_entropy_and_overlap_annexes(&self) -> OKIResult<()> { + fn ensure_family_annexes(&self, request: &AnnexRequest) -> OKIResult<()> { let n_genomes = self.meta().genomes().len(); let fast_mode = is_fast_mode(self); + let overlap_path = |no_ambiguity: bool| self.index().dir().join(family_overlap_file_name(no_ambiguity)); - let overlap_path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME); - let mut overlap_acc = - (!overlap_path.exists()).then(|| FamilyOverlapAccumulator::new(n_genomes)); - - let pb = progress_bar("entropy_overlap_annex", self.n_partition() as u64, "partitions"); + // Sweep 1: entropy, quartet, and every family-overlap annex still missing. + let mut overlap_accs: Vec<(bool, FamilyOverlapAccumulator)> = request + .overlap_conventions() + .into_iter() + .filter(|&no_ambiguity| !overlap_path(no_ambiguity).exists()) + .map(|no_ambiguity| (no_ambiguity, FamilyOverlapAccumulator::new(n_genomes, no_ambiguity))) + .collect(); + let pb = progress_bar("family_annexes", self.n_partition() as u64, "partitions"); for part in self.partitions() { let n_layer = self.n_layer(part).unwrap_or(0); for l in 0..n_layer { - ensure_layer_entropy_annex(self, part, l, n_genomes, fast_mode, overlap_acc.as_mut())?; + ensure_layer_family_annexes(self, part, l, n_genomes, fast_mode, request, &mut overlap_accs)?; + } + pb.inc(1); + } + pb.finish_and_clear(); + for (no_ambiguity, acc) in overlap_accs { + acc.write(&overlap_path(no_ambiguity)).map_err(OKIError::Io)?; + } + + // Sweep 2: pair weights, from the now-complete overlap annexes. + if request.pair_weight.is_empty() { + return Ok(()); + } + let matrices = request + .pair_weight_conventions() + .into_iter() + .map(|no_ambiguity| { + FamilyOverlap::open(&overlap_path(no_ambiguity)) + .map(|overlap| (no_ambiguity, inverse_overlap_matrix(&overlap))) + .map_err(OKIError::Io) + }) + .collect::>>()?; + let pb = progress_bar("pair_weight_annex", self.n_partition() as u64, "partitions"); + for part in self.partitions() { + let n_layer = self.n_layer(part).unwrap_or(0); + for l in 0..n_layer { + ensure_layer_pair_weight_annexes(self, part, l, n_genomes, fast_mode, &matrices)?; } pb.inc(1); } pb.finish_and_clear(); - if let Some(acc) = overlap_acc { - acc.write(&overlap_path).map_err(OKIError::Io)?; - } - Ok(()) } } diff --git a/src/obikphylo/src/siblings/extensions/sibling_ext.rs b/src/obikphylo/src/siblings/extensions/sibling_ext.rs index b1812c1b..3e4066e4 100644 --- a/src/obikphylo/src/siblings/extensions/sibling_ext.rs +++ b/src/obikphylo/src/siblings/extensions/sibling_ext.rs @@ -14,12 +14,15 @@ use obikindex::{OKIError, OKIResult}; use obisys::progress_bar; use crate::siblings::algorithms::{ - Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, family_entropy_4, - is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, sibling_annex_stats, - sibling_family_size_histogram, + AnnexRequest, Selection, SiblingAnnexStats, build_layer_sibling_annex, family_entropy, + family_entropy_4, is_fast_mode, non_monomorphic_reindex_layer, scan_layer_families, + sibling_annex_stats, sibling_family_size_histogram, }; use crate::siblings::extensions::SiblingBuilder; -use crate::siblings::{ENTROPY_ANNEX_FILE_NAME, FAMILY_OVERLAP_FILE_NAME, FamilyOverlap}; +use crate::siblings::{ + FAMILY_OVERLAP_FILE_NAMES, FamilyOverlap, QUARTET_ANNEX_FILE_NAMES, ScalarKind, + family_overlap_file_name, +}; pub trait SiblingExt { /// Build the sibling-count/minorant annex for every layer of every @@ -81,8 +84,10 @@ pub trait SiblingExt { fn sibling_annex_stats(&self) -> OKIResult; /// The Family Overlap annex — number of shared *variable* families per - /// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs). - /// Built (alongside the entropy annex) if missing, same as every other + /// genome pair, index-wide (see [`FamilyOverlap`]'s own module docs), + /// under the `--no-ambiguity` counting convention `no_ambiguity` selects + /// (genomes carrying several members of a family do not count as + /// carrying it when `true`). Built if missing, same as every other /// annex-backed method here. Feeds `--family-overlap`'s own matrix /// output directly, and `--min-shared-family`'s auto-exclusion /// (`FamilyOverlap::mean_row(g) < threshold` — same statistic @@ -90,7 +95,7 @@ pub trait SiblingExt { /// triggers adding `g` to the excluded-genome set consumed by /// `algorithms::scan::sampled_scan`'s callers; this needs no /// computation of its own beyond that check). - fn family_overlap(&self) -> OKIResult; + fn family_overlap(&self, no_ambiguity: bool) -> OKIResult; } impl SiblingExt for IndexCache { @@ -115,16 +120,24 @@ impl SiblingExt for IndexCache { .expect("cache-consistent: n_layer(part) already bounds l"); part_slots += build_layer_sibling_annex(self, &layer, l, n_parts, fast_mode)?; - // Invalidate this layer's entropy annex, if any — it was - // built against the sibling annex's *previous* - // `family_idx` numbering (`non_monomorphic_reindex_layer` - // reads the sibling annex), now stale. - // `ensure_entropy_and_overlap_annexes` below only builds - // what's *missing*, so the old file must be removed here - // for it to be rebuilt rather than silently left stale. - let entropy_path = layer.dir().join(ENTROPY_ANNEX_FILE_NAME); - if entropy_path.exists() { - std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?; + // Invalidate this layer's entropy and quartet annexes, if + // any — they were built against the sibling annex's + // *previous* `family_idx` numbering + // (`non_monomorphic_reindex_layer` reads the sibling + // annex), now stale. `ensure_family_annexes` below only + // builds what's *missing*, so the old files must be removed + // here for them to be rebuilt rather than silently left + // stale. + for name in ScalarKind::Entropy + .file_names() + .into_iter() + .chain(ScalarKind::PairWeight.file_names()) + .chain(QUARTET_ANNEX_FILE_NAMES) + { + let entropy_path = layer.dir().join(name); + if entropy_path.exists() { + std::fs::remove_file(&entropy_path).map_err(OKIError::Io)?; + } } } total_slots += part_slots; @@ -136,13 +149,16 @@ impl SiblingExt for IndexCache { pb.finish_and_clear(); tracing::info!("sibling annex built — {total_slots} kmers across {n_parts} partitions"); - // The family-overlap annex is index-level, not per-layer (see its - // own module docs) — any layer's sibling annex changing makes the - // whole thing stale, so it's invalidated once here rather than - // per-layer above. - let overlap_path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME); - if overlap_path.exists() { - std::fs::remove_file(&overlap_path).map_err(OKIError::Io)?; + // The family-overlap annexes are index-level, not per-layer (see + // their own module docs) — any layer's sibling annex changing makes + // them all stale (and, with them, every pair weight computed from + // them, removed per layer above), so they are invalidated once here + // rather than per-layer above. + for name in FAMILY_OVERLAP_FILE_NAMES { + let overlap_path = self.index().dir().join(name); + if overlap_path.exists() { + std::fs::remove_file(&overlap_path).map_err(OKIError::Io)?; + } } // Precompute the entropy and family-overlap annexes too, now that @@ -152,19 +168,17 @@ impl SiblingExt for IndexCache { // rather than leaving them to be built lazily on first // `--shannon`/`--entropy-bias`/`--min-shared-family`/ // `--family-overlap` use. - tracing::info!("building entropy and family-overlap annexes"); - self.ensure_entropy_and_overlap_annexes()?; + tracing::info!("building entropy and quartet (both --no-ambiguity conventions) and family-overlap annexes"); + self.ensure_family_annexes(&AnnexRequest::everything())?; Ok(()) } fn shannon_entropy_csv(&self, path: &Path) -> OKIResult<()> { - // Entropy/family-overlap annexes aren't this method's own output, - // but ensuring they exist is a cheap prerequisite check, not a - // reason to reuse their scan here — see below, this method - // deliberately runs its own, differently-scoped one. - self.ensure_entropy_and_overlap_annexes()?; - + // Needs only the sibling annex (`non_monomorphic_reindex_layer` + // below): this method runs its own, differently-scoped scan and + // recomputes every value it writes, so it neither reads nor + // requires the entropy/family-overlap annexes. let n_genomes = self.meta().genomes().len(); let fast_mode = is_fast_mode(self); @@ -237,9 +251,9 @@ impl SiblingExt for IndexCache { sibling_annex_stats(self) } - fn family_overlap(&self) -> OKIResult { - self.ensure_entropy_and_overlap_annexes()?; - let path = self.index().dir().join(FAMILY_OVERLAP_FILE_NAME); + fn family_overlap(&self, no_ambiguity: bool) -> OKIResult { + self.ensure_family_annexes(&AnnexRequest { overlap: vec![no_ambiguity], ..Default::default() })?; + let path = self.index().dir().join(family_overlap_file_name(no_ambiguity)); FamilyOverlap::open(&path).map_err(OKIError::Io) } } diff --git a/src/obikphylo/src/siblings/family_overlap.rs b/src/obikphylo/src/siblings/family_overlap.rs index a1e1ed2e..2f19159e 100644 --- a/src/obikphylo/src/siblings/family_overlap.rs +++ b/src/obikphylo/src/siblings/family_overlap.rs @@ -24,20 +24,43 @@ use std::path::Path; use memmap2::Mmap; -use crate::siblings::helpers::triangle_index; +use crate::siblings::helpers::{is_carrier, triangle_index}; const MAGIC: [u8; 4] = *b"POVL"; -// Header: magic(4) + _pad(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair, -// upper triangle) follows. +// Header: magic(4) + flags(4) + n_genomes(8) = 16 bytes. Data (8 bytes/pair, +// upper triangle) follows. `flags` bit 0 = `no_ambiguity` (see +// `FamilyOverlapAccumulator::new`); older files have `flags == 0`, i.e. +// `no_ambiguity == false`, which is what they were computed under. const HEADER_SIZE: usize = 16; +const FLAG_NO_AMBIGUITY: u32 = 1; pub(crate) const FAMILY_OVERLAP_FILE_NAME: &str = "family_overlap.povl"; +/// Same annex, built with `no_ambiguity == true`. +pub(crate) const FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME: &str = "family_overlap.noamb.povl"; + +/// Every family-overlap file name, whatever its counting convention — for +/// invalidating them all together when the sibling annex they derive from is +/// rebuilt. +pub(crate) const FAMILY_OVERLAP_FILE_NAMES: [&str; 2] = + [FAMILY_OVERLAP_FILE_NAME, FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME]; + +/// File name of the family-overlap annex built under the given +/// `--no-ambiguity` counting convention. The two never share a file. +pub(crate) fn family_overlap_file_name(no_ambiguity: bool) -> &'static str { + if no_ambiguity { + FAMILY_OVERLAP_NO_AMBIGUITY_FILE_NAME + } else { + FAMILY_OVERLAP_FILE_NAME + } +} + /// Read-only, mmap-backed view of an already-built Family Overlap annex. pub struct FamilyOverlap { mmap: Mmap, n_genomes: usize, + no_ambiguity: bool, } impl FamilyOverlap { @@ -54,13 +77,22 @@ impl FamilyOverlap { if mmap.len() < HEADER_SIZE + n_pairs * 8 { return Err(io::Error::new(io::ErrorKind::InvalidData, "POVL file truncated")); } - Ok(Self { mmap, n_genomes }) + let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap()); + Ok(Self { mmap, n_genomes, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 }) } pub fn n_genomes(&self) -> usize { self.n_genomes } + /// The counting convention these counts were computed under: `true` + /// when a genome carrying several members of a family was *not* counted + /// as carrying it (`--no-ambiguity`), `false` when any non-empty + /// state counts as carrying it. + pub fn no_ambiguity(&self) -> bool { + self.no_ambiguity + } + /// Number of variable families shared by genomes `i` and `j` — `0` for /// `i == j` (not stored; a genome trivially "shares" every one of its /// own families with itself, not a meaningful statistic here). @@ -95,24 +127,30 @@ impl FamilyOverlap { /// `EntropyAnnexBuilder` do for their own, much larger, per-k-mer data. pub(crate) struct FamilyOverlapAccumulator { n_genomes: usize, + no_ambiguity: bool, counts: Vec, } impl FamilyOverlapAccumulator { - pub(crate) fn new(n_genomes: usize) -> Self { + /// `no_ambiguity` is the counting convention: `false` counts every + /// genome with a non-empty state as carrying the family, `true` only the + /// genomes carrying exactly one member of it — the others are missing + /// data in the alignment (`--no-ambiguity`), so they share nothing there. + pub(crate) fn new(n_genomes: usize, no_ambiguity: bool) -> Self { let n_pairs = n_genomes.saturating_sub(1) * n_genomes / 2; - Self { n_genomes, counts: vec![0u64; n_pairs] } + Self { n_genomes, no_ambiguity, counts: vec![0u64; n_pairs] } } /// One variable family's contribution: every pair of genomes both - /// carrying it (`genome_mask[g] != 0`) gets its count incremented once. + /// carrying it (per the counting convention) gets its count incremented + /// once. pub(crate) fn add_family(&mut self, genome_mask: &[u8]) { for i in 0..self.n_genomes { - if genome_mask[i] == 0 { + if !is_carrier(genome_mask[i], self.no_ambiguity) { continue; } for j in (i + 1)..self.n_genomes { - if genome_mask[j] != 0 { + if is_carrier(genome_mask[j], self.no_ambiguity) { self.counts[triangle_index(self.n_genomes, i, j)] += 1; } } @@ -122,7 +160,8 @@ impl FamilyOverlapAccumulator { pub(crate) fn write(&self, path: &Path) -> io::Result<()> { let mut buf = Vec::with_capacity(HEADER_SIZE + self.counts.len() * 8); buf.extend_from_slice(&MAGIC); - buf.extend_from_slice(&[0u8; 4]); + let flags = if self.no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 }; + buf.extend_from_slice(&flags.to_le_bytes()); buf.extend_from_slice(&(self.n_genomes as u64).to_le_bytes()); for &c in &self.counts { buf.extend_from_slice(&c.to_le_bytes()); @@ -140,7 +179,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test.povl"); - let mut acc = FamilyOverlapAccumulator::new(3); + let mut acc = FamilyOverlapAccumulator::new(3, false); acc.add_family(&[1, 1, 0]); // genomes 0,1 share this family acc.add_family(&[1, 1, 1]); // all three share this one acc.add_family(&[0, 1, 1]); // genomes 1,2 share this family @@ -160,8 +199,27 @@ mod tests { fn single_genome_mean_row_is_zero() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("test.povl"); - FamilyOverlapAccumulator::new(1).write(&path).unwrap(); + FamilyOverlapAccumulator::new(1, false).write(&path).unwrap(); let overlap = FamilyOverlap::open(&path).unwrap(); assert_eq!(overlap.mean_row(0), 0.0); } + + #[test] + fn no_ambiguity_does_not_count_multi_member_genomes_as_carriers() { + let dir = tempfile::tempdir().unwrap(); + let masks: [&[u8]; 2] = [&[1, 1, 0b0011], &[1, 0b0100, 0b0100]]; + for no_ambiguity in [false, true] { + let path = dir.path().join(family_overlap_file_name(no_ambiguity)); + let mut acc = FamilyOverlapAccumulator::new(3, no_ambiguity); + for m in masks { + acc.add_family(m); + } + acc.write(&path).unwrap(); + let overlap = FamilyOverlap::open(&path).unwrap(); + assert_eq!(overlap.no_ambiguity(), no_ambiguity); + // genome 2 carries two members in the first family: only counted without the flag + assert_eq!(overlap.get(0, 2), if no_ambiguity { 1 } else { 2 }); + assert_eq!(overlap.get(0, 1), 1 + 1); + } + } } diff --git a/src/obikphylo/src/siblings/family_scalar_annex.rs b/src/obikphylo/src/siblings/family_scalar_annex.rs new file mode 100644 index 00000000..c5679f9f --- /dev/null +++ b/src/obikphylo/src/siblings/family_scalar_annex.rs @@ -0,0 +1,247 @@ +//! Persisted, per-layer `f32` value per **non-monomorphic minorant family**: +//! the Shannon entropy of the family ([`ScalarKind::Entropy`], see +//! `algorithms::entropy`) or its pair-balance weight +//! ([`ScalarKind::PairWeight`], see `algorithms::pair_weight`). Compact: +//! indexed by the dedicated ordinal `algorithms::non_monomorphic_reindex_layer` +//! computes (skips monomorphic minorants entirely, ~98% of minorants on real +//! data), never by `scan_layer_families`'s own `family_idx` (which counts +//! every minorant, monomorphic included). +//! +//! Exists so repeated reads of the same layer's values (entropy-biased or +//! pair-balanced family selection) are a plain positional mmap lookup +//! instead of re-paying `scan_layer_families`'s cross-partition resolution — +//! building a file costs exactly that same full scan the first time, it does +//! not make any single computation of its value faster. +//! +//! Mirrors `SiblingAnnex`'s on-disk convention (magic + header + flat +//! per-entry data, read-only after build), just with `f32` payloads and a +//! different (compact) numbering. The two kinds share the format and differ +//! by magic and file name only, so one kind can never be read as the other. + +use std::fs::{File, OpenOptions}; +use std::io; +use std::path::Path; + +use memmap2::{Mmap, MmapMut}; + +// Header: magic(4) + flags(4) + n(8) = 16 bytes. Data (4 bytes/entry) follows. +// `flags` bit 0 = `no_ambiguity` (the counting convention the values were +// computed under, see `FamilyScalarAnnexBuilder::new`); every file written +// before the flag existed has `flags == 0`, i.e. `no_ambiguity == false`, +// which is what it was computed under. +const FLAG_NO_AMBIGUITY: u32 = 1; +const HEADER_SIZE: usize = 16; + +/// Sentinel: not yet computed. Every stored value is `>= 0.0`, so any +/// negative value is unambiguous — chosen over `NaN` to avoid +/// `NaN`-comparison footguns at every read site. +const SENTINEL: f32 = -1.0; + +/// What the stored `f32` means. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScalarKind { + /// Shannon entropy of the family (bits). + Entropy, + /// Mean, over the pairs of genomes carrying the family, of 1 / (number of + /// families the pair shares) — see `algorithms::pair_weight`. + PairWeight, +} + +impl ScalarKind { + fn magic(self) -> [u8; 4] { + match self { + ScalarKind::Entropy => *b"PENT", + ScalarKind::PairWeight => *b"PPWT", + } + } + + /// File name of this kind's annex built under the given `--no-ambiguity` + /// counting convention. The two conventions never share a file: + /// switching between them builds the missing annex without discarding the + /// other. + pub(crate) fn file_name(self, no_ambiguity: bool) -> &'static str { + match (self, no_ambiguity) { + (ScalarKind::Entropy, false) => "entropy.pent", + (ScalarKind::Entropy, true) => "entropy.noamb.pent", + (ScalarKind::PairWeight, false) => "pairweight.ppwt", + (ScalarKind::PairWeight, true) => "pairweight.noamb.ppwt", + } + } + + /// Every file name of this kind, whatever its counting convention — for + /// invalidating them all together (e.g. when the sibling annex they were + /// numbered against is rebuilt). + pub(crate) fn file_names(self) -> [&'static str; 2] { + [self.file_name(false), self.file_name(true)] + } +} + +pub struct FamilyScalarAnnex { + mmap: Mmap, + n: usize, + no_ambiguity: bool, +} + +impl FamilyScalarAnnex { + pub(crate) fn open(path: &Path, kind: ScalarKind) -> io::Result { + let mmap = unsafe { Mmap::map(&File::open(path)?)? }; + if mmap.len() < HEADER_SIZE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file too short")); + } + if mmap[0..4] != kind.magic() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("bad magic: not a {kind:?} annex"), + )); + } + let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize; + if mmap.len() < HEADER_SIZE + n * 4 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "scalar annex file truncated")); + } + let flags = u32::from_le_bytes(mmap[4..8].try_into().unwrap()); + Ok(Self { mmap, n, no_ambiguity: flags & FLAG_NO_AMBIGUITY != 0 }) + } + + pub fn len(&self) -> usize { + self.n + } + + /// The counting convention these values were computed under: `true` + /// when genomes carrying several members of a family were left out of + /// the count (`--no-ambiguity`), `false` when they were counted as a + /// state of their own. + pub fn no_ambiguity(&self) -> bool { + self.no_ambiguity + } + + /// `None` for the sentinel (not yet computed — shouldn't happen against + /// a fully-built annex). + pub fn get(&self, idx: usize) -> Option { + let off = HEADER_SIZE + idx * 4; + let v = f32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap()); + (v >= 0.0).then_some(v) + } +} + +pub struct FamilyScalarAnnexBuilder { + mmap: MmapMut, +} + +impl FamilyScalarAnnexBuilder { + /// Create a new annex of `n` entries (a layer's non-monomorphic + /// minorant count) at `path`, pre-initialised to the sentinel. + /// `no_ambiguity` records, in the header, the counting convention the + /// values will be computed under (see [`FamilyScalarAnnex::no_ambiguity`]). + pub(crate) fn new(n: usize, path: &Path, kind: ScalarKind, no_ambiguity: bool) -> io::Result { + let file_size = HEADER_SIZE + n * 4; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(path)?; + file.set_len(file_size as u64)?; + let mut mmap = unsafe { MmapMut::map_mut(&file)? }; + mmap[0..4].copy_from_slice(&kind.magic()); + let flags = if no_ambiguity { FLAG_NO_AMBIGUITY } else { 0 }; + mmap[4..8].copy_from_slice(&flags.to_le_bytes()); + mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes()); + for i in 0..n { + let off = HEADER_SIZE + i * 4; + mmap[off..off + 4].copy_from_slice(&SENTINEL.to_le_bytes()); + } + Ok(Self { mmap }) + } + + pub(crate) fn set(&mut self, idx: usize, value: f32) { + debug_assert!(value >= 0.0, "stored values must be non-negative, got {value}"); + let off = HEADER_SIZE + idx * 4; + self.mmap[off..off + 4].copy_from_slice(&value.to_le_bytes()); + } + + pub(crate) fn close(self) -> io::Result<()> { + self.mmap.flush() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + const E: ScalarKind = ScalarKind::Entropy; + + #[test] + fn sentinel_is_negative_and_unset_entries_read_as_none() { + let dir = tempdir().unwrap(); + let path = dir.path().join("test.pent"); + FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap().close().unwrap(); + let annex = FamilyScalarAnnex::open(&path, E).unwrap(); + for i in 0..4 { + assert_eq!(annex.get(i), None); + } + } + + #[test] + fn roundtrip_including_zero_value() { + let dir = tempdir().unwrap(); + let path = dir.path().join("test.pent"); + let mut builder = FamilyScalarAnnexBuilder::new(4, &path, E, false).unwrap(); + let values = [0.0f32, 1.5, 3.9, 0.0001]; + for (i, &v) in values.iter().enumerate() { + builder.set(i, v); + } + builder.close().unwrap(); + let annex = FamilyScalarAnnex::open(&path, E).unwrap(); + for (i, &v) in values.iter().enumerate() { + assert_eq!(annex.get(i), Some(v), "entry {i}"); + } + } + + #[test] + fn mixed_sentinel_and_real_values() { + let dir = tempdir().unwrap(); + let path = dir.path().join("test.pent"); + let mut builder = FamilyScalarAnnexBuilder::new(3, &path, E, false).unwrap(); + builder.set(1, 2.0); + builder.close().unwrap(); + let annex = FamilyScalarAnnex::open(&path, E).unwrap(); + assert_eq!(annex.get(0), None); + assert_eq!(annex.get(1), Some(2.0)); + assert_eq!(annex.get(2), None); + assert_eq!(annex.len(), 3); + } + + #[test] + fn counting_convention_roundtrips_through_the_header() { + let dir = tempdir().unwrap(); + for kind in [ScalarKind::Entropy, ScalarKind::PairWeight] { + for no_ambiguity in [false, true] { + let path = dir.path().join(kind.file_name(no_ambiguity)); + FamilyScalarAnnexBuilder::new(2, &path, kind, no_ambiguity).unwrap().close().unwrap(); + assert_eq!(FamilyScalarAnnex::open(&path, kind).unwrap().no_ambiguity(), no_ambiguity); + } + assert_ne!(kind.file_name(false), kind.file_name(true)); + } + } + + #[test] + fn one_kind_cannot_be_read_as_the_other() { + let dir = tempdir().unwrap(); + let path = dir.path().join("w.ppwt"); + FamilyScalarAnnexBuilder::new(1, &path, ScalarKind::PairWeight, false).unwrap().close().unwrap(); + assert!(FamilyScalarAnnex::open(&path, ScalarKind::Entropy).is_err()); + assert!(FamilyScalarAnnex::open(&path, ScalarKind::PairWeight).is_ok()); + } + + #[test] + fn file_with_zero_flags_reads_as_ambiguity_counted() { + // files written before the flag existed carried four zero bytes there + let dir = tempdir().unwrap(); + let path = dir.path().join("old.pent"); + FamilyScalarAnnexBuilder::new(1, &path, E, false).unwrap().close().unwrap(); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[4..8], &[0u8; 4]); + assert!(!FamilyScalarAnnex::open(&path, E).unwrap().no_ambiguity()); + } +} diff --git a/src/obikphylo/src/siblings/helpers.rs b/src/obikphylo/src/siblings/helpers.rs index b9d92499..d924ceb5 100644 --- a/src/obikphylo/src/siblings/helpers.rs +++ b/src/obikphylo/src/siblings/helpers.rs @@ -34,3 +34,15 @@ pub(super) fn triangle_index(n: usize, i: usize, j: usize) -> usize { debug_assert!(j < n && i != j); (i * (2 * n - i - 1)) / 2 + (j - i - 1) } + +/// Whether a genome with this per-family `mask` (bit `b` set iff it carries +/// the member whose central base is `b`) counts as carrying the family, under +/// the `--no-ambiguity` counting convention: any non-empty state does, except +/// — with `no_ambiguity` — a genome carrying several members, which is +/// missing data in the alignment and so shares nothing there. One definition +/// for every annex that counts carriers (family overlap, pair weight, +/// quartet), so they can never disagree. +#[inline] +pub(crate) fn is_carrier(mask: u8, no_ambiguity: bool) -> bool { + mask != 0 && !(no_ambiguity && mask.count_ones() > 1) +} diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index c15d0901..258f4e80 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -20,17 +20,21 @@ //! reverse. pub mod algorithms; -mod entropy_annex; pub mod extensions; mod family_overlap; +mod family_scalar_annex; mod helpers; pub mod iter; +mod quartet_annex; mod siblingannex; -pub(crate) use entropy_annex::ENTROPY_ANNEX_FILE_NAME; -pub use entropy_annex::{EntropyAnnex, EntropyAnnexBuilder}; -pub(crate) use family_overlap::{FAMILY_OVERLAP_FILE_NAME, FamilyOverlapAccumulator}; +pub(crate) use family_overlap::{ + FAMILY_OVERLAP_FILE_NAMES, FamilyOverlapAccumulator, family_overlap_file_name, +}; +pub(crate) use family_scalar_annex::ScalarKind; +pub use family_scalar_annex::{FamilyScalarAnnex, FamilyScalarAnnexBuilder}; pub use family_overlap::FamilyOverlap; +pub(crate) use quartet_annex::QUARTET_ANNEX_FILE_NAMES; pub use iter::{ MinorantBatchIter, MinorantIter, SiblingBatchIter, SiblingEntry, SiblingIter, SiblingLayerExt, }; @@ -38,7 +42,8 @@ pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use algorithms::{ AlignmentReducer, BasePairTally, CardinalityTally, EntropyBias, GammaShape, LayerReducer, - PairwiseReducer, RawSnpDistanceOutput, SankoffBundle, SiblingAnnexStats, SnpAlignment, + PairwiseReducer, RawSnpDistanceOutput, SamplingSpec, SankoffBundle, SiblingAnnexStats, + SnpAlignment, SnpDistanceKind, cardinality_transition_probs, composition_transition_probs, distance_matrix, pairwise_cost_matrix, sampled_scan, }; diff --git a/src/obikphylo/src/siblings/quartet_annex.rs b/src/obikphylo/src/siblings/quartet_annex.rs new file mode 100644 index 00000000..6eb0d590 --- /dev/null +++ b/src/obikphylo/src/siblings/quartet_annex.rs @@ -0,0 +1,59 @@ +//! Persisted, per-layer quartet-informativeness flag: one bit per +//! non-monomorphic minorant family, stored as a plain +//! [`obicompactvec::PersistentBitVec`] (no format of its own) and numbered +//! by the same compact ordinal as [`crate::siblings::EntropyAnnex`] +//! (`algorithms::non_monomorphic_reindex_layer`), never by +//! `scan_layer_families`'s own `family_idx`. +//! +//! A family is quartet-informative when its carrier genomes show at least +//! two distinct called states, each carried by at least two genomes: the +//! pattern that can separate two pairs of genomes from each other +//! (`AB|CD`). Anything weaker — a singleton variant, or a family carried by +//! fewer than four genomes — fits every topology and tells a tree nothing. +//! See `algorithms::masking::is_quartet_informative` for the exact rule. +//! +//! Which genomes count as carriers depends on `--no-ambiguity`, so each +//! convention has its own file, and **the file name is the parameter**: a +//! `PersistentBitVec` header has no room for a flag, and changing it would +//! change a shared structure for the sake of one annex. +//! +//! The flag is computed over every genome of the index. A later +//! `--exclude-genome` can only remove carriers, never create a bipartition, +//! so a family flagged `false` here is also uninformative once genomes are +//! excluded: the annex is a safe pre-filter, and the exact check +//! (`algorithms::masking::survives_quartet`) still runs on the resolved +//! family. + +pub(crate) const QUARTET_ANNEX_FILE_NAME: &str = "quartet.pbiv"; + +/// Same annex, built with `no_ambiguity == true`. +pub(crate) const QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME: &str = "quartet.noamb.pbiv"; + +/// Every quartet-annex file name, whatever its counting convention — for +/// invalidating them all together when the sibling annex they were numbered +/// against is rebuilt. +pub(crate) const QUARTET_ANNEX_FILE_NAMES: [&str; 2] = + [QUARTET_ANNEX_FILE_NAME, QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME]; + +/// File name of the quartet annex built under the given `--no-ambiguity` +/// convention. +pub(crate) fn quartet_annex_file_name(no_ambiguity: bool) -> &'static str { + if no_ambiguity { + QUARTET_ANNEX_NO_AMBIGUITY_FILE_NAME + } else { + QUARTET_ANNEX_FILE_NAME + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_two_conventions_never_share_a_file() { + assert_ne!(quartet_annex_file_name(false), quartet_annex_file_name(true)); + for no_ambiguity in [false, true] { + assert!(QUARTET_ANNEX_FILE_NAMES.contains(&quartet_annex_file_name(no_ambiguity))); + } + } +} -- 2.54.1