docs: document architecture analysis and refactoring plans for siblings

Documents architectural analysis, identified performance bottlenecks including hardcoded selection flags and redundant full-index scans causing I/O-bound stalls. Details planned pipeline refactoring to unify stages around a single shared selection for a fused single-pass scan, noting future dependencies on entropy-biased family selection.

.gitea/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+pname: CI
 
 on:
   pull_request:
@@ -25,8 +25,8 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             src/target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('src/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-cargo-
+          key: ${{ runner.os }}-cargo-v2-${{ hashFiles('src/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-v2-
 
       - name: Build
         run: cargo build --release

.gitignore

@@ -16,6 +16,7 @@ benchmark/simulated_data
 benchmark/specimen_index_presence
 benchmark/specimen_index_count
 benchmark/global_index_presence
+benchmark/global_index_presence_sav
 benchmark/all_specific
 benchmark/global_index_count
 benchmark/stats
@@ -24,3 +25,16 @@ benchmark/reference_dist
 benchmark/obikmer_dist
 benchmark/specific_index_count
 benchmark/specific_index_presence
+TNT
+phyg
+biblio
+*.tnt
+*.tre
+*.phy
+*.treefile
+*.bionj
+*.iqtree
+*.mldist
+*.parstree
+*.ckp.gz
+*.model

Makefile

@@ -10,6 +10,11 @@ DOC_FILE := mkdocs.yml
 DOC_SITE := doc
 DOC_PORT := 8001
 
+DOC_USER_DIR  := UserDocMD
+DOC_USER_FILE := mkdocs-user.yml
+DOC_USER_SITE := doc-user
+DOC_USER_PORT := 8002
+
 # ── virtualenv ────────────────────────────────────────────────────────────────
 
 $(VENV)/bin/activate:
@@ -60,8 +65,22 @@ doc-serve: $(MKDOCS)
 clean-doc:
 	rm -rf $(DOC_SITE)/
 
+.PHONY: doc-user
+doc-user: $(MKDOCS)
+	$(MKDOCS) build -f $(DOC_USER_FILE)
+
+.PHONY: doc-user-serve
+doc-user-serve: $(MKDOCS)
+	$(MKDOCS) serve -f $(DOC_USER_FILE) \
+	                --dev-addr=127.0.0.1:$(DOC_USER_PORT) \
+	                --livereload
+
+.PHONY: clean-doc-user
+clean-doc-user:
+	rm -rf $(DOC_USER_SITE)/
+
 .PHONY: clean
-clean: clean-doc
+clean: clean-doc clean-doc-user
 	rm -rf $(VENV)
 
 # ── release ───────────────────────────────────────────────────────────────────
@@ -86,7 +105,7 @@ bump-version:
 
 .PHONY: release
 release: bump-version
-	@jj auto-describe
+	@jj auto-doc
 	@jj git push --change @
 	@new_version=$$(grep '^version = ' $(CARGO_TOML) | head -n 1 | sed 's/version = "\(.*\)"/\1/'); \
 	git_hash=$$(jj log -r @ --no-graph -T 'commit_id'); \

README.md

@@ -66,9 +66,11 @@ Non-ACGT characters act as hard breaks between k-mer segments in all formats.
                Annotates each sequence with per-genome k-mer match counts
                and optional per-position coverage vectors (--detail).
                Parallel over sequence chunks.
-    distance   Compute a pairwise Bray-Curtis or Jaccard distance matrix
-               between all indexed genomes.
-               Optionally outputs a Newick NJ or UPGMA tree.
+    phylo      Compute pairwise evolutionary-distance proxies between all
+               indexed genomes (Bray-Curtis, Jaccard, etc.), optionally
+               a Newick NJ/UPGMA tree, and optionally a central-position
+               SNP/Sankoff calibration with exports for external
+               phylogenetic tools (TNT, PhyG, IQ-TREE).
     annotate   Add or update genome metadata (taxonomy, etc.) from a CSV
                file; or dump the current metadata as CSV.
     estimate   Dry-run: resolve and print approximate-index parameters
@@ -106,7 +108,7 @@ obikmer reindex --approx -z 5 --evidence-bits 8 index/
 obikmer query index/ reads.fq.gz > annotated.fa
 
 # Pairwise distances
-obikmer distance index/ > distances.tsv
+obikmer phylo index/ > distances.tsv
 ```
 
 ## Parameter constraints

UserDocMD/architecture.md

@@ -0,0 +1,33 @@
+# Architecture notes for advanced use
+
+This page describes execution-level behavior relevant to sizing and running `obikmer` on large datasets or multi-socket machines. It complements the [index format](formats/index_layout.md) and [theory](theory/indexing_architecture.md) pages.
+
+## Sequence invariant
+
+Every input sequence is treated purely as a compact representation of a set of overlapping kmers:
+
+- Only the `A`/`C`/`G`/`T` alphabet (case-insensitive) is recognized; a sequence is cut at any other character (including IUPAC ambiguity codes), so runs containing them are not represented in the index.
+- Sequences are internally processed in chunks of at most 256 nucleotides; a chunk shorter than k is dropped. This is invisible to the user beyond the ACGT-only, minimum-length-k constraints above.
+- Kmers are always handled in canonical form (see [DNA encoding](theory/encoding.md)), so the tool is strand-agnostic throughout: a kmer and its reverse complement are always the same entry.
+
+## Index dimensioning
+
+An index directory is organized as `KmerIndex → partitions → layers`, with a canonical kmer belonging to exactly one (partition, layer) pair. This is what makes set operations (merge, filter, distance) parallel and coordination-free across partitions.
+
+- **Partition count** (`-p`/`--partitions`, rounded up to a power of 2) is the main dimensioning knob: more partitions means more independent parallel units and a smaller working set per partition, at the cost of more open files during construction.
+- **Layers** accumulate as an index grows through successive merges; per-partition query cost grows with the number of layers (worst case linear, expected constant since most kmer lookups resolve in the first layer they could plausibly be in).
+- Genome columns (count or presence data) are kept at a consistent width across every layer and partition after a merge, which is what allows whole-index aggregate distances (Jaccard, Bray-Curtis, Euclidean, Hellinger, …) to be computed as a two-pass cascade (local partial sums per partition, then a global combination) with no double counting.
+
+## Parallel execution and NUMA awareness
+
+Partition-level work (index construction, `merge`, `filter`, `reindex`, `select`, `phylo`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc:
+
+- On a multi-socket / multi-NUMA-node machine, one thread pool is pinned per NUMA node, and each partition is processed entirely by threads pinned to one node — keeping the memory a partition touches local to that node's DRAM. This matters because touching kmer data across NUMA nodes without pinning can degrade throughput by an order of magnitude or more on large multi-socket machines.
+- On a single-socket machine, Apple Silicon, or if hwloc cannot report NUMA topology, all cores are treated as one node with no pinning and negligible overhead — this is the default behavior on macOS.
+- Within a node, the number of active worker threads ramps up progressively rather than being fixed up front: it starts conservatively and grows in steps, but only as long as measured CPU efficiency or disk I/O throughput keeps improving. If neither improves after a growth step, the runner stops adding workers — avoiding oversubscription on stages that are memory-bandwidth-bound rather than CPU- or I/O-bound. Ramp speed scales with the number of cores per node, so a single-node machine ramps just as fast as a large multi-node one.
+
+No CLI flag controls this directly; it is fully automatic at runtime. NUMA-aware pinning can be compiled out (Cargo feature `numa`, on by default), in which case a plain global thread pool is used instead.
+
+## Kmer filtering (`filter`)
+
+[`filter`](usage/filter.md) evaluates predicates against the genome metadata matrix directly whenever every active filter can be expressed as a column-level test (e.g. "any outgroup column non-zero"), producing a per-slot keep/drop decision without touching kmer sequence data at all. If any active filter cannot be expressed this way, evaluation falls back to a per-kmer, row-level check. Either way, the result is always written as a single, freshly compacted layer (`unitigs.bin` and the MPHF are rebuilt from the surviving kmers), never as an additional layer on top of the source index.

UserDocMD/ecology-letters.csl

@@ -0,0 +1,230 @@
+<?xml version="1.0" encoding="utf-8"?>
+<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" demote-non-dropping-particle="sort-only" default-locale="en-US">
+  <info>
+    <title>Ecology Letters</title>
+    <id>http://www.zotero.org/styles/ecology-letters</id>
+    <link href="http://www.zotero.org/styles/ecology-letters" rel="self"/>
+    <link href="http://www.zotero.org/styles/apa" rel="template"/>
+    <link href="http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291461-0248/homepage/ForAuthors.html" rel="documentation"/>
+    <author>
+      <name>David Kaplan</name>
+      <email>david.kaplan@ird.fr</email>
+    </author>
+    <contributor>
+      <name>Sebastian Karcher</name>
+    </contributor>
+    <category citation-format="author-date"/>
+    <category field="biology"/>
+    <issn>1461-023X</issn>
+    <eissn>1461-0248</eissn>
+    <updated>2023-10-11T10:45:32+00:00</updated>
+    <rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
+  </info>
+  <macro name="container">
+    <choose>
+      <if type="chapter paper-conference" match="any">
+        <text term="in" text-case="capitalize-first" suffix=": "/>
+        <text variable="container-title" font-style="italic"/>
+        <text variable="collection-title" prefix=", "/>
+        <names variable="editor translator" prefix=" (" delimiter=", " suffix=")">
+          <label form="short" suffix=" "/>
+          <name name-as-sort-order="all" and="symbol" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="never"/>
+        </names>
+      </if>
+      <else>
+        <group delimiter=", ">
+          <text variable="container-title" font-style="italic" form="short"/>
+          <text variable="collection-title"/>
+        </group>
+      </else>
+    </choose>
+  </macro>
+  <macro name="author">
+    <names variable="author">
+      <name name-as-sort-order="all" and="symbol" sort-separator=", " initialize-with="." delimiter=", " delimiter-precedes-last="never"/>
+      <label form="short" prefix=" (" suffix=")" text-case="capitalize-first"/>
+      <et-al font-style="italic"/>
+      <substitute>
+        <names variable="editor"/>
+        <names variable="translator"/>
+        <text macro="title"/>
+      </substitute>
+    </names>
+  </macro>
+  <macro name="author-short">
+    <names variable="author">
+      <name form="short" and="symbol" delimiter=", " initialize-with=". "/>
+      <et-al font-style="italic"/>
+      <substitute>
+        <names variable="editor"/>
+        <names variable="translator"/>
+        <choose>
+          <if type="bill book graphic legal_case legislation motion_picture report song" match="any">
+            <text variable="title" form="short" font-style="italic"/>
+          </if>
+          <else>
+            <text variable="title" form="short" quotes="true"/>
+          </else>
+        </choose>
+      </substitute>
+    </names>
+  </macro>
+  <macro name="access">
+    <choose>
+      <if type="webpage">
+        <group>
+          <text term="available at" text-case="capitalize-first" suffix=": "/>
+          <text variable="URL" suffix="."/>
+        </group>
+        <text value="Last accessed" prefix=" " suffix=" "/>
+        <date variable="accessed">
+          <date-part name="day" suffix=" "/>
+          <date-part name="month" suffix=" "/>
+          <date-part name="year"/>
+        </date>
+      </if>
+    </choose>
+  </macro>
+  <macro name="title">
+    <choose>
+      <if type="report" match="any">
+        <text variable="title" font-style="italic"/>
+        <group prefix=" (" suffix=")">
+          <text variable="genre"/>
+          <text variable="number" prefix=" No. "/>
+        </group>
+      </if>
+      <else-if type="bill book graphic legal_case legislation motion_picture report song speech" match="any">
+        <text variable="title" font-style="italic"/>
+      </else-if>
+      <else-if type="webpage">
+        <text variable="title" font-style="italic"/>
+      </else-if>
+      <else>
+        <text variable="title"/>
+      </else>
+    </choose>
+  </macro>
+  <macro name="publisher">
+    <choose>
+      <if type="report" match="any">
+        <group delimiter=", ">
+          <text variable="publisher"/>
+          <text variable="publisher-place"/>
+        </group>
+      </if>
+      <else>
+        <text variable="genre" suffix=". "/>
+        <group delimiter=", ">
+          <text variable="publisher"/>
+          <text variable="publisher-place"/>
+        </group>
+      </else>
+    </choose>
+  </macro>
+  <macro name="event">
+    <choose>
+      <if variable="event">
+        <text term="presented at" text-case="capitalize-first" suffix=" "/>
+        <text variable="event"/>
+      </if>
+    </choose>
+  </macro>
+  <macro name="issued">
+    <choose>
+      <if variable="issued">
+        <date variable="issued">
+          <date-part name="year"/>
+        </date>
+      </if>
+      <else-if variable="accessed">
+        <choose>
+          <if type="webpage">
+            <date variable="accessed">
+              <date-part name="year"/>
+            </date>
+          </if>
+          <else>
+            <text term="no date" form="short"/>
+          </else>
+        </choose>
+      </else-if>
+      <else>
+        <text term="no date" form="short"/>
+      </else>
+    </choose>
+  </macro>
+  <macro name="edition">
+    <choose>
+      <if is-numeric="edition">
+        <group delimiter=" ">
+          <number variable="edition" form="ordinal"/>
+          <text value="edn"/>
+        </group>
+      </if>
+      <else>
+        <text variable="edition" suffix="."/>
+      </else>
+    </choose>
+  </macro>
+  <macro name="locators">
+    <choose>
+      <if type="article-journal article-magazine article-newspaper" match="any">
+        <group prefix=", " delimiter=", ">
+          <group>
+            <text variable="volume"/>
+          </group>
+          <text variable="page"/>
+        </group>
+      </if>
+      <else-if type="bill book graphic legal_case legislation motion_picture report song thesis" match="any">
+        <group delimiter=". " prefix=". ">
+          <text macro="edition"/>
+          <text macro="event"/>
+          <text macro="publisher"/>
+        </group>
+      </else-if>
+      <else-if type="chapter paper-conference" match="any">
+        <group delimiter=", " prefix=". ">
+          <text macro="event"/>
+          <text macro="publisher"/>
+          <group>
+            <label variable="page" form="short" suffix=" "/>
+            <text variable="page"/>
+          </group>
+        </group>
+      </else-if>
+    </choose>
+  </macro>
+  <citation et-al-min="3" et-al-use-first="1" disambiguate-add-year-suffix="true" collapse="year-suffix" year-suffix-delimiter=", ">
+    <sort>
+      <key macro="author"/>
+      <key macro="issued"/>
+    </sort>
+    <layout prefix="(" suffix=")" delimiter="; ">
+      <group delimiter=" ">
+        <text macro="author-short"/>
+        <text macro="issued"/>
+      </group>
+    </layout>
+  </citation>
+  <bibliography et-al-min="7" et-al-use-first="6" entry-spacing="0" hanging-indent="true">
+    <sort>
+      <key macro="author"/>
+      <key macro="issued" sort="ascending"/>
+      <key macro="title"/>
+    </sort>
+    <layout>
+      <group suffix=".">
+        <text macro="author" suffix="."/>
+        <text macro="issued" prefix=" (" suffix="). "/>
+        <group delimiter=". ">
+          <text macro="title"/>
+          <text macro="container"/>
+        </group>
+        <text macro="locators"/>
+        <text macro="access" prefix=". "/>
+      </group>
+    </layout>
+  </bibliography>
+</style>

UserDocMD/formats/index_layout.md

@@ -0,0 +1,56 @@
+# Index construction and on-disk layout
+
+## Construction pipeline
+
+Building an index ([`index`](../usage/index_command.md)) proceeds through a fixed sequence of phases, each operating independently per partition (see [Partitioning and indexing architecture](../theory/indexing_architecture.md)):
+
+1. **Scatter.** A single streaming pass over the input. Each sequence fragment is cut at non-ACGT bases, passed through the low-complexity entropy filter (see [Low-complexity kmer filter](../theory/entropy_filter.md)), and any resulting segment shorter than k is dropped. Surviving segments are decomposed into super-kmers, canonicalized, and routed by `hash(minimizer) mod n_partitions` into one file per partition.
+2. **Dereplication.** Within each partition, identical super-kmer sequences are merged and their occurrence counts summed. This count is per super-kmer, not per kmer — a kmer's true abundance is the sum of the counts of every super-kmer containing it.
+3. **Exact counting.** Every kmer in every dereplicated super-kmer is enumerated and its exact total count computed. A per-genome kmer frequency spectrum is produced at this stage.
+4. **Quorum filtering.** Kmers outside the `--min-abundance`/`--max-abundance` range are dropped, and super-kmers are recompacted around the surviving kmer set.
+5. **Local assembly.** The surviving kmers of each partition are assembled into unitigs — maximal non-branching runs of a local de Bruijn graph — such that every kmer appears exactly once, at one (unitig, offset) location.
+6. **MPHF and evidence construction.** A minimal perfect hash function is built over the canonical kmers of each partition, together with the evidence structure needed to verify that a queried kmer was genuinely indexed (see below). Per-genome counts or presence bits are recorded alongside if requested.
+
+Phases 1–5 are independent per partition and run in parallel; phase 6 finalizes each partition once its kmer set is fixed.
+
+## Minimal perfect hash function (MPHF)
+
+Each partition's surviving kmers are mapped to a dense range of integer slots by a minimal perfect hash function: no collisions, near-optimal space (a few bits per key), O(1) lookup. Because an MPHF maps *any* input to some slot — including kmers that were never indexed — a lookup alone cannot distinguish a genuinely indexed kmer from an arbitrary one; every lookup is followed by an evidence check.
+
+## Evidence: exact vs. approximate
+
+Two verification modes are available, selected at build time (`index --approx`) and convertible afterwards ([`reindex`](../usage/reindex.md)):
+
+- **Exact** (default): the hashed slot stores a pointer back into the partition's unitig data. At query time the kmer is reconstructed from that location and compared directly to the query. Zero false positives, at the cost of one extra random read per lookup.
+- **Approximate** (`--approx`): the slot stores a short fingerprint (`--evidence-bits` bits) instead of a pointer; verification is a single fingerprint comparison. This trades a small, bounded false-positive rate ($1/2^b$ per kmer, reduced further to about $1/2^{b \cdot z}$ for a read requiring $z$ consecutive matching kmers via the `-z`/`--findere-z` parameter) for lower memory and disk usage, since no reconstruction index is needed. See [`estimate`](../usage/estimate.md) to explore this trade-off before building.
+
+## On-disk layout
+
+```
+<index_root>/
+  index.meta                    global configuration (k, minimizer size, partition count,
+                                 evidence mode, whether counts are stored) and genome list/metadata
+  scatter.done / count.done / index.done      build-progress sentinels
+  spectrums/<label>.json        per-genome kmer frequency histogram
+  partitions/
+    part_00000/ ... part_NNNNN/
+      index/
+        meta.json                number of layers in this partition
+        layer_0/
+          unitigs.bin             reconstructible kmer sequence data — always kept
+          unitigs.bin.idx         random-access index into unitigs.bin (exact evidence only)
+          mphf.bin                 the minimal perfect hash function
+          evidence.bin              exact evidence (exact mode only)
+          fingerprint.bin           approximate evidence (approximate mode only)
+          counts/                   per-genome kmer counts (if counts were requested)
+          presence/                 per-genome presence/absence bits
+        layer_1/, layer_2/, ...     added by later merges, same internal structure
+```
+
+`unitigs.bin` is the only file from which the indexed kmer content can be fully recovered; it is always retained. Every other file (MPHF, evidence, counts) is derived from it.
+
+A **layer** corresponds to one increment of kmer content added to a partition — most commonly, one [`merge`](../usage/merge.md) operation that introduces kmers not already present in the index. Genomes already present in the index simply gain new columns in the existing layers' count/presence data; only genuinely new kmer content is assembled into a new layer. Because of this, merging cost scales with the novel kmer content being added, not with the accumulated size of the index. A query against an index with several layers checks each layer's MPHF in turn.
+
+Sources merged together must share the same kmer size, minimizer size, partition count, and evidence mode (including matching approximate-mode parameters); mismatches are rejected rather than silently reconciled — [`reindex`](../usage/reindex.md) one of the sources first if needed.
+
+`obikmer pack` consolidates a partition's per-column files (counts/presence) into a single file, reducing the number of file opens needed at query time.

UserDocMD/index.md

@@ -0,0 +1,69 @@
+# obikmer
+
+`obikmer` is a command-line tool for counting, indexing, querying and comparing DNA sequences represented as kmer sets. It targets individual genome datasets of tens of gigabases, with an emphasis on computational, memory, and disk efficiency.
+
+All functionality is exposed through a single binary, `obikmer`, organized as subcommands.
+
+## Core principles
+
+- Kmers are of fixed, odd length $k$, chosen at index-construction time in the range $[11, 31]$ (see [Kmers and super-kmers](theory/kmers_and_superkmers.md)).
+- Each kmer fits in a 64-bit word using a 2-bit-per-base encoding (see [DNA encoding](theory/encoding.md)).
+- Kmers are handled in **canonical form** ($\text{canonical}(kmer) = \min(kmer, \text{revcomp}(kmer))$), making counting strand-independent.
+- Sequences are decomposed into **super-kmers** before storage, anchored on a hash-selected **minimizer** (see [Minimizer selection](theory/minimizer_selection.md)), then routed to one of several **partitions** for parallel, memory-bounded processing (see [Partitioning and indexing architecture](theory/indexing_architecture.md)).
+- Low-complexity kmers can be filtered out at index-construction time using an entropy-based score (see [Low-complexity kmer filter](theory/entropy_filter.md)).
+
+## Commands
+
+| Command | Purpose |
+|---|---|
+| [`superkmer`](usage/superkmer.md) | Extract super-kmers from a sequence file and write them to stdout |
+| [`index`](usage/index_command.md) | Build a genome index |
+| [`merge`](usage/merge.md) | Merge multiple indexes into one |
+| [`filter`](usage/filter.md) | Retain only kmers matching ingroup/outgroup predicates |
+| [`select`](usage/select.md) | Project and/or aggregate genome columns of an index |
+| [`query`](usage/query.md) | Query an index with sequences and annotate matches |
+| [`dump`](usage/dump.md) | Dump indexed kmers as CSV |
+| [`annotate`](usage/annotate.md) | Add, update, or dump genome metadata |
+| [`phylo`](usage/phylo.md) | Compute pairwise evolutionary-distance proxies, trees, and phylogenetic exports |
+| [`name-tree`](usage/name-tree.md) | Translate a TNT/PhyG numeric-label tree export back to real taxon names |
+| [`unitig`](usage/unitig.md) | Dump the unitigs of an index as FASTA |
+| [`estimate`](usage/estimate.md) | Estimate approximate-index parameters before indexing |
+| [`reindex`](usage/reindex.md) | Convert an index's evidence representation (exact ↔ approximate) |
+| [`utils`](usage/utils.md) | Miscellaneous index maintenance and inspection utilities |
+| [`pack`](usage/pack.md) | Pack per-column matrix files into a single-file format |
+
+See [Genome predicates and taxonomy paths](usage/predicates.md) for the selection language shared by `filter`, `select`, `dump`, and `unitig`.
+
+## Further reading
+
+- [Index construction and on-disk layout](formats/index_layout.md)
+- [Architecture notes for advanced use](architecture.md) — parallel execution, NUMA awareness, index dimensioning
+
+## Input formats
+
+- `superkmer` and `index`: FASTA (`.fa`, `.fasta`), FASTQ (`.fq`, `.fastq`), GenBank flat file (`.gb`, `.gbk`, `.gbff`), all optionally gzip-compressed; directories are expanded recursively; streaming stdin via `-` or when no input path is given.
+- `query`: FASTA or FASTQ, optionally gzip-compressed; streaming stdin the same way.
+
+## Parameter constraints
+
+These constraints are checked at startup; an invalid value exits immediately with an error.
+
+| Parameter | Constraint | Reason |
+|---|---|---|
+| $k$ (`--kmer-size`) | odd, $k \in [11, 31]$ | odd length guarantees the canonical form is always well defined; the range keeps a kmer within a 64-bit word while retaining specificity |
+| $m$ (`--minimizer-size`) | odd, $3 \le m \le k-1$ | same palindrome argument as $k$; must be strictly shorter than the kmer |
+| $z$ (`-z`, approximate evidence only) | $z \le k-1$ | the effective indexed kmer size is $k-z+1$ |
+
+## Genome label constraints
+
+Genome labels are arbitrary Unicode strings, with the following restrictions:
+
+| Character | Forbidden | Reason |
+|---|---|---|
+| `/` | yes | filesystem path separator |
+| `=` | yes | separator used by `--new-label` |
+| `\0` | yes | null byte |
+| `\n`, `\r`, `\t` | yes | would break CSV output |
+| spaces | allowed | quote in the shell, e.g. `--new-label 'new label=old label'` |
+
+Empty labels are rejected. A label derived automatically from the input file name (when `--label` is omitted) is not validated, since it is already filesystem-safe.

UserDocMD/installation.md

@@ -0,0 +1,84 @@
+# Installation
+
+## Prerequisites
+
+### Rust toolchain
+
+`obikmer` requires **Rust 1.85 or later** (edition 2024). Install or update via [rustup](https://rustup.rs):
+
+```bash
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+rustup update stable
+```
+
+### C build environment (required for hwloc)
+
+`obikmer` embeds [hwloc](https://www.open-mpi.org/projects/hwloc/) (Hardware Locality) for NUMA-aware thread placement on multi-socket machines. hwloc is built from source at compile time, which requires a standard C build environment.
+
+#### Linux (Debian/Ubuntu)
+
+```bash
+apt install build-essential automake libtool autoconf pkg-config
+```
+
+#### Linux (RHEL/Rocky/AlmaLinux)
+
+```bash
+dnf install gcc make automake libtool autoconf pkgconfig
+```
+
+#### HPC clusters
+
+Most HPC clusters provide these tools via the module system:
+
+```bash
+module load gcc automake libtool autoconf
+```
+
+If in doubt, check that `autoreconf --version` and `libtool --version` return successfully.
+
+#### macOS
+
+```bash
+brew install automake libtool autoconf pkg-config
+```
+
+## Building
+
+```bash
+git clone <repository-url>
+cd obikmer/src
+cargo build --release
+```
+
+The compiled binary is at `target/release/obikmer`.
+
+### Building on HPC clusters (network filesystems)
+
+HPC home directories are typically on a network filesystem (Lustre, NFS) optimized for large sequential reads, not for the many small file operations Cargo generates during compilation. Building directly on such a filesystem can be extremely slow.
+
+Redirect the build directory to a local scratch disk:
+
+```bash
+CARGO_TARGET_DIR=/scratch/$USER/cargo-target cargo build --release
+```
+
+Adapt the path to the scratch space available on your cluster (`/var/tmp`, `/tmp`, `/scratch/local`, etc.). Once built, copy the binary to a permanent location:
+
+```bash
+cp /scratch/$USER/cargo-target/release/obikmer ~/bin/
+```
+
+## NUMA support
+
+NUMA-aware thread placement is active automatically on multi-socket Linux machines, detected at runtime via hwloc. No build flag is required — it falls back gracefully to a single-pool strategy on:
+
+- macOS (Apple Silicon, unified memory)
+- single-socket Linux machines
+- any system where hwloc reports only one NUMA node
+
+## Verifying the installation
+
+```bash
+obikmer --help
+```

UserDocMD/references.bib

@@ -0,0 +1,261 @@
+%% This BibTeX bibliography file was created using BibDesk.
+%% https://bibdesk.sourceforge.io/
+
+%% Created for Eric Coissac at 2026-04-18 08:19:36 +0200
+
+
+%% Saved with string encoding Unicode (UTF-8)
+
+
+
+@article{Zheng2020-ji,
+	abstract = {MOTIVATION: Minimizers are methods to sample k-mers from a
+               string, with the guarantee that similar set of k-mers will be
+               chosen on similar strings. It is parameterized by the k-mer
+               length k, a window length w and an order on the k-mers.
+               Minimizers are used in a large number of softwares and pipelines
+               to improve computation efficiency and decrease memory usage.
+               Despite the method's popularity, many theoretical questions
+               regarding its performance remain open. The core metric for
+               measuring performance of a minimizer is the density, which
+               measures the sparsity of sampled k-mers. The theoretical optimal
+               density for a minimizer is 1/w, provably not achievable in
+               general. For given k and w, little is known about asymptotically
+               optimal minimizers, that is minimizers with density O(1/w).
+               RESULTS: We derive a necessary and sufficient condition for
+               existence of asymptotically optimal minimizers. We also provide a
+               randomized algorithm, called the Miniception, to design
+               minimizers with the best theoretical guarantee to date on density
+               in practical scenarios. Constructing and using the Miniception is
+               as easy as constructing and using a random minimizer, which
+               allows the design of efficient minimizers that scale to the
+               values of k and w used in current bioinformatics software
+               programs. AVAILABILITY AND IMPLEMENTATION: Reference
+               implementation of the Miniception and the codes for analysis can
+               be found at https://github.com/kingsford-group/miniception.
+               SUPPLEMENTARY INFORMATION: Supplementary data are available at
+               Bioinformatics online.},
+	author = {Zheng, Hongyu and Kingsford, Carl and Mar{\c c}ais, Guillaume},
+	doi = {10.1093/bioinformatics/btaa472},
+	issn = {1367-4803,1367-4811},
+	journal = {Bioinformatics (Oxford, England)},
+	language = {en},
+	month = jul,
+	number = {Suppl_1},
+	pages = {i119--i127},
+	pmc = {PMC8248892},
+	pmid = 32657376,
+	publisher = {Oxford University Press (OUP)},
+	title = {Improved design and analysis of practical minimizers},
+	url = {http://dx.doi.org/10.1093/bioinformatics/btaa472},
+	volume = 36,
+	year = 2020,
+	bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btaa472}}
+
+@article{Zheng2021-cc,
+	abstract = {MOTIVATION: Minimizers are efficient methods to sample k-mers
+               from genomic sequences that unconditionally preserve sufficiently
+               long matches between sequences. Well-established methods to
+               construct efficient minimizers focus on sampling fewer k-mers on
+               a random sequence and use universal hitting sets (sets of k-mers
+               that appear frequently enough) to upper bound the sketch size. In
+               contrast, the problem of sequence-specific minimizers, which is
+               to construct efficient minimizers to sample fewer k-mers on a
+               specific sequence such as the reference genome, is less studied.
+               Currently, the theoretical understanding of this problem is
+               lacking, and existing methods do not specialize well to sketch
+               specific sequences. RESULTS: We propose the concept of polar
+               sets, complementary to the existing idea of universal hitting
+               sets. Polar sets are k-mer sets that are spread out enough on the
+               reference, and provably specialize well to specific sequences.
+               Link energy measures how well spread out a polar set is, and with
+               it, the sketch size can be bounded from above and below in a
+               theoretically sound way. This allows for direct optimization of
+               sketch size. We propose efficient heuristics to construct polar
+               sets, and via experiments on the human reference genome, show
+               their practical superiority in designing efficient
+               sequence-specific minimizers. AVAILABILITY AND IMPLEMENTATION: A
+               reference implementation and code for analyses under an
+               open-source license are at
+               https://github.com/kingsford-group/polarset. SUPPLEMENTARY
+               INFORMATION: Supplementary data are available at Bioinformatics
+               online.},
+	author = {Zheng, Hongyu and Kingsford, Carl and Mar{\c c}ais, Guillaume},
+	doi = {10.1093/bioinformatics/btab313},
+	issn = {1367-4803,1367-4811},
+	journal = {Bioinformatics (Oxford, England)},
+	language = {en},
+	month = jul,
+	number = {Suppl\_1},
+	pages = {i187--i195},
+	pmc = {PMC8686682},
+	pmid = 34252928,
+	publisher = {Oxford University Press (OUP)},
+	title = {Sequence-specific minimizers via polar sets},
+	url = {http://dx.doi.org/10.1093/bioinformatics/btab313},
+	volume = 37,
+	year = 2021,
+	bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btab313}}
+
+@article{Pan2024-hb,
+	abstract = {MOTIVATION: The minimizer concept is a data structure for
+               sequence sketching. The standard canonical minimizer selects a
+               subset of k-mers from the given DNA sequence by comparing the
+               forward and reverse k-mers in a window simultaneously according
+               to a predefined selection scheme. It is widely employed by
+               sequence analysis such as read mapping and assembly. k-mer
+               density, k-mer repetitiveness (e.g. k-mer bias), and
+               computational efficiency are three critical measurements for
+               minimizer selection schemes. However, there exist trade-offs
+               between kinds of minimizer variants. Generic, effective, and
+               efficient are always the requirements for high-performance
+               minimizer algorithms. RESULTS: We propose a simple minimizer
+               operator as a refinement of the standard canonical minimizer. It
+               takes only a few operations to compute. However, it can improve
+               the k-mer repetitiveness, especially for the lexicographic order.
+               It applies to other selection schemes of total orders (e.g.
+               random orders). Moreover, it is computationally efficient and the
+               density is close to that of the standard minimizer. The refined
+               minimizer may benefit high-performance applications like binning
+               and read mapping. AVAILABILITY AND IMPLEMENTATION: The source
+               code of the benchmark in this work is available at the github
+               repository https://github.com/xp3i4/mini\_benchmark.},
+	author = {Pan, Chenxu and Reinert, Knut},
+	doi = {10.1093/bioinformatics/btae045},
+	issn = {1367-4803,1367-4811},
+	journal = {Bioinformatics (Oxford, England)},
+	language = {en},
+	month = feb,
+	number = 2,
+	pmc = {PMC10868324},
+	pmid = 38269626,
+	publisher = {Oxford University Press (OUP)},
+	title = {A simple refined DNA minimizer operator enables 2-fold faster computation},
+	url = {http://dx.doi.org/10.1093/bioinformatics/btae045},
+	volume = 40,
+	year = 2024,
+	bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btae045}}
+
+@article{Kille2023-px,
+	abstract = {MOTIVATION: The Jaccard similarity on k-mer sets has shown to be
+               a convenient proxy for sequence identity. By avoiding expensive
+               base-level alignments and comparing reduced sequence
+               representations, tools such as MashMap can scale to massive
+               numbers of pairwise comparisons while still providing useful
+               similarity estimates. However, due to their reliance on minimizer
+               winnowing, previous versions of MashMap were shown to be biased
+               and inconsistent estimators of Jaccard similarity. This directly
+               impacts downstream tools that rely on the accuracy of these
+               estimates. RESULTS: To address this, we propose the minmer
+               winnowing scheme, which generalizes the minimizer scheme by use
+               of a rolling minhash with multiple sampled k-mers per window. We
+               show both theoretically and empirically that minmers yield an
+               unbiased estimator of local Jaccard similarity, and we implement
+               this scheme in an updated version of MashMap. The minmer-based
+               implementation is over 10 times faster than the minimizer-based
+               version under the default ANI threshold, making it well-suited
+               for large-scale comparative genomics applications. AVAILABILITY
+               AND IMPLEMENTATION: MashMap3 is available at
+               https://github.com/marbl/MashMap.},
+	author = {Kille, Bryce and Garrison, Erik and Treangen, Todd J and Phillippy, Adam M},
+	doi = {10.1093/bioinformatics/btad512},
+	issn = {1367-4803,1367-4811},
+	journal = {Bioinformatics (Oxford, England)},
+	language = {en},
+	month = sep,
+	number = 9,
+	pmc = {PMC10505501},
+	pmid = 37603771,
+	publisher = {Oxford University Press (OUP)},
+	title = {Minmers are a generalization of minimizers that enable unbiased local Jaccard estimation},
+	url = {http://dx.doi.org/10.1093/bioinformatics/btad512},
+	volume = 39,
+	year = 2023,
+	bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btad512}}
+
+@incollection{Golan2025-xf,
+	address = {Cham},
+	author = {Golan, Shay and Shur, Arseny M},
+	booktitle = {Lecture Notes in Computer Science},
+	doi = {10.1007/978-3-031-82670-2\_25},
+	isbn = {9783031826696,9783031826702},
+	issn = {0302-9743,1611-3349},
+	language = {en},
+	pages = {347--360},
+	publisher = {Springer Nature Switzerland},
+	series = {Lecture Notes in Computer Science},
+	title = {Expected density of random minimizers},
+	url = {http://dx.doi.org/10.1007/978-3-031-82670-2_25},
+	year = 2025,
+	bdsk-url-1 = {http://dx.doi.org/10.1007/978-3-031-82670-2_25},
+	bdsk-url-2 = {http://dx.doi.org/10.1007/978-3-031-82670-2%5C_25}}
+
+@article{Mohamadi2017-ok,
+	abstract = {Motivation: Many bioinformatics algorithms are designed for the
+               analysis of sequences of some uniform length, conventionally
+               referred to as k -mers. These include de Bruijn graph assembly
+               methods and sequence alignment tools. An efficient algorithm to
+               enumerate the number of unique k -mers, or even better, to build
+               a histogram of k -mer frequencies would be desirable for these
+               tools and their downstream analysis pipelines. Among other
+               applications, estimated frequencies can be used to predict genome
+               sizes, measure sequencing error rates, and tune runtime
+               parameters for analysis tools. However, calculating a k -mer
+               histogram from large volumes of sequencing data is a challenging
+               task. Results: Here, we present ntCard, a streaming algorithm for
+               estimating the frequencies of k -mers in genomics datasets. At
+               its core, ntCard uses the ntHash algorithm to efficiently compute
+               hash values for streamed sequences. It then samples the
+               calculated hash values to build a reduced representation
+               multiplicity table describing the sample distribution. Finally,
+               it uses a statistical model to reconstruct the population
+               distribution from the sample distribution. We have compared the
+               performance of ntCard and other cardinality estimation
+               algorithms. We used three datasets of 480 GB, 500 GB and 2.4 TB
+               in size, where the first two representing whole genome shotgun
+               sequencing experiments on the human genome and the last one on
+               the white spruce genome. Results show ntCard estimates k -mer
+               coverage frequencies >15× faster than the state-of-the-art
+               algorithms, using similar amount of memory, and with higher
+               accuracy rates. Thus, our benchmarks demonstrate ntCard as a
+               potentially enabling technology for large-scale genomics
+               applications. Availability and Implementation: ntCard is written
+               in C ++ and is released under the GPL license. It is freely
+               available at https://github.com/bcgsc/ntCard. Contact:
+               hmohamadi@bcgsc.ca or ibirol@bcgsc.ca. Supplementary information:
+               Supplementary data are available at Bioinformatics online.},
+	author = {Mohamadi, Hamid and Khan, Hamza and Birol, Inanc},
+	date-modified = {2026-04-18 08:19:36 +0200},
+	doi = {10.1093/bioinformatics/btw832},
+	issn = {1367-4803,1367-4811},
+	journal = {Bioinformatics (Oxford, England)},
+	language = {en},
+	month = may,
+	number = 9,
+	pages = {1324--1330},
+	pmc = {PMC5408799},
+	pmid = 28453674,
+	publisher = {Oxford University Press (OUP)},
+	title = {ntCard: a streaming algorithm for cardinality estimation in genomics data},
+	url = {http://dx.doi.org/10.1093/bioinformatics/btw832},
+	volume = 33,
+	year = 2017,
+	bdsk-url-1 = {http://dx.doi.org/10.1093/bioinformatics/btw832}}
+
+@misc{Mash-distances-doc,
+	author = {{Marbl Lab}},
+	howpublished = {Mash documentation},
+	title = {Mash Distance},
+	url = {https://mash.readthedocs.io/en/latest/distances.html},
+	urldate = {2026-07-09},
+	year = 2026}
+
+@article{Fan2015-mash-formula,
+	author = {Fan, Huan and Ives, Anthony R and Surget-Groba, Yann and Cannon, Charles H},
+	doi = {10.1186/s12864-015-1647-5},
+	journal = {BMC Genomics},
+	number = 1,
+	title = {An assembly and alignment-free method of phylogeny reconstruction from next-generation sequencing data},
+	url = {https://doi.org/10.1186/s12864-015-1647-5},
+	volume = 16,
+	year = 2015}

UserDocMD/theory/encoding.md

@@ -0,0 +1,28 @@
+# DNA encoding
+
+## 2-bit nucleotide encoding
+
+Every nucleotide is encoded on 2 bits, most-significant-bit first within each word:
+
+| Base | Encoding |
+|------|----------|
+| A    | `00`     |
+| C    | `01`     |
+| G    | `10`     |
+| T    | `11`     |
+
+The Watson-Crick complement of a base is its bitwise NOT on 2 bits: $\text{complement}(base) = \lnot base \mathbin{\&} \texttt{0b11}$.
+
+## Kmer encoding
+
+A kmer of length $k$ ($k \le 31$) fits in a single 64-bit word. The first nucleotide occupies the two most significant bits, each following nucleotide occupies the next two bits, and unused low-order bits are zero. Extracting nucleotide i (0-indexed from the 5′ end) is a shift-and-mask operation.
+
+Reverse complement is computed by bit manipulation directly on the packed word, without any lookup table: complement every base, reverse the byte order, then reverse the order of 2-bit groups within each byte in two more passes, and finally realign the result to the most-significant bits.
+
+## Canonical form
+
+The canonical form of a kmer is the lexicographic minimum of the kmer and its reverse complement:
+
+$$\text{canonical}(kmer) = \min\big(kmer,\ \text{revcomp}(kmer)\big)$$
+
+Using the canonical form halves the kmer space and makes counting strand-independent: a kmer and its reverse complement are always treated as the same entity, regardless of which DNA strand was sequenced.

UserDocMD/theory/entropy_filter.md

@@ -0,0 +1,36 @@
+# Low-complexity kmer filter
+
+Low-complexity kmers (homopolymer runs, tandem repeats) can dominate an index without carrying useful information. `obikmer` detects and excludes them during index construction using a normalized Shannon entropy score.
+
+## Sub-word frequencies
+
+For a kmer of length $k$ and a sub-word size $ws$ ($1 \le ws \le ws_{\max}$, default $ws_{\max} = 6$), the kmer is decomposed into its $k - ws + 1$ overlapping sub-words of length $ws$ by sliding a window across it. Each sub-word is tallied under its raw 2-bit-packed value, with no canonicalization.
+
+## Corrected Shannon entropy
+
+Let $f_j$ be the observed count of raw sub-word $j$, and $n_{\text{words}} = k - ws + 1$ the total number of sub-words. The entropy is:
+
+$$H_{\text{corr}} = \log(n_{\text{words}}) - \frac{1}{n_{\text{words}}} \sum_j f_j \log f_j$$
+
+## Small-sample correction
+
+Because only $n_{\text{words}}$ sub-words are observed among up to $4^{ws}$ possible values, the achievable maximum entropy $H_{\max}$ is bounded below $\log(4^{ws})$ for small samples. $H_{\max}$ is computed from the most uniform integer distribution achievable with $n_{\text{words}}$ observations over $4^{ws}$ categories. The normalized entropy is:
+
+$$\hat{H}(ws) = \frac{H_{\text{corr}}}{H_{\max}} \in [0, 1]$$
+
+A value near 0 indicates low complexity (e.g. a homopolymer run); near 1 indicates high complexity, characteristic of a random sequence.
+
+## Final score
+
+The filter evaluates $\hat{H}(ws)$ for every word size from 1 to ws_max and keeps the minimum:
+
+$$\text{entropy}(kmer) = \min_{ws=1}^{ws_{\max}} \hat{H}(ws)$$
+
+Taking the minimum across word sizes ensures that repetition at any scale is detected: a homopolymer is caught at $ws=1$, a dinucleotide repeat at $ws=2$, and so on. A kmer is rejected if its entropy score falls below a threshold $\theta$ (default 0.7), a configurable collection parameter.
+
+## Properties
+
+The entropy score depends only on the kmer sequence itself, not on where or how many times it occurs:
+
+- **Orientation invariance**: a kmer and its reverse complement always receive the same score.
+- **Context independence**: a given kmer is always accepted or always rejected, regardless of which genome or read it appears in. The filter defines a fixed partition of the kmer space into low-complexity and valid kmers.

UserDocMD/theory/indexing_architecture.md

@@ -0,0 +1,32 @@
+# Partitioning and indexing architecture
+
+An index is split into a fixed number of **partitions**, each handling an independent, disjoint slice of the kmer space. Partitioning keeps the working set of each stage small enough to process efficiently and enables parallel construction and querying.
+
+## Routing
+
+The canonical minimizer of a super-kmer (see [Minimizer selection](minimizer_selection.md)) is hashed to produce a $p$-bit routing value that selects the destination partition:
+
+```
+canonical minimizer → hash(minimizer) → p-bit value → partition index
+```
+
+The routing value is recomputed whenever it is needed (during construction and again at query time) rather than stored — it is not part of the on-disk super-kmer representation.
+
+Within a partition, kmers are indexed as plain values via a minimal perfect hash function (see [On-disk storage](../formats/index_layout.md)); the minimizer plays no further role once a super-kmer has reached its partition.
+
+## Why hashing is necessary
+
+A canonical minimizer is an m-mer ($m \in \{9, 11, 13, 15\}$), and its distribution over all possible m-mer values is not uniform — as the lexicographic minimum of a window, small values are systematically over-represented [@Zheng2020-ji; @Zheng2021-cc; @Pan2024-hb; @Kille2023-px; @Golan2025-xf]. Routing directly on the raw minimizer value would therefore produce badly unbalanced partitions.
+
+Hashing the minimizer before routing redistributes this skewed distribution uniformly across partitions. This works reliably because the number of partition-index bits $p$ is chosen well below the number of bits available in the minimizer ($2m$): even with strong bias in the minimizer distribution, the hash has enough entropy margin to absorb it, provided the number of distinct minimizers actually observed is much larger than the number of partitions.
+
+## Parameter guidance
+
+| Minimizer size $m$ | Minimizer bits ($2m$) | Typical partition-index bits $p$ | Partitions |
+|----|-----------|-----------|------------|
+| 9  | 18        | 6–8       | 64–256     |
+| 11 | 22        | 8–10      | 256–1 024  |
+| 13 | 26        | 10–12     | 1 024–4 096|
+| 15 | 30        | 10–14     | 1 024–16 384|
+
+The number of partitions must satisfy $p \le 2m$, and in practice $p$ is chosen well below that bound to leave a comfortable entropy margin. For $k=31$, $m=13$, $p=10$ (1024 partitions), partition load is well balanced on real genomic data.

UserDocMD/theory/kmers_and_superkmers.md

@@ -0,0 +1,24 @@
+# Kmers and super-kmers
+
+## Kmers
+
+A **kmer** is a DNA subsequence of fixed length $k$. Two constraints apply to $k$, both enforced when a command starts (an invalid value exits immediately with an error):
+
+- $k \in [11, 31]$: long enough to be specific, short enough to fit in a single 64-bit word at 2 bits/base ($k \le 32$ is the hard limit; $k < 11$ gives insufficient specificity).
+- $k$ **is odd**: an odd-length sequence can never equal its own reverse complement, so the two orientations of any kmer are always distinct. This is required for the canonical form (see [DNA encoding](encoding.md)) to be well defined.
+
+## Super-kmers
+
+A **super-kmer** is a maximal run of consecutive, overlapping kmers from a read that share the same canonical minimizer (see [Minimizer selection](minimizer_selection.md)). Each kmer in the run overlaps the next by $k-1$ nucleotides. A super-kmer is capped at 256 nucleotides; a longer run is split at that boundary.
+
+For a random minimizer of length $m$ over kmers of length $k$, the expected length of a super-kmer is approximately [@Zheng2020-ji; @Golan2025-xf]:
+
+$$L_{\text{nt}} \approx \frac{k-m+2}{2} + k - 1$$
+
+For $k=31$, $m=13$ this is about 40 nucleotides; in practice super-kmers rarely exceed a few dozen nucleotides.
+
+### Canonical super-kmers
+
+A **canonical super-kmer** is the lexicographic minimum of a super-kmer and its reverse complement. When a read and its reverse complement are both encountered, they produce super-kmers that are reverse complements of each other; both reduce to the same canonical super-kmer, so a genomic region is represented once regardless of which strand was read.
+
+Super-kmers are the unit of work used throughout construction and querying: sequences are decomposed into super-kmers first, and every downstream step (partition routing, deduplication, counting) operates on them rather than on individual kmers.

UserDocMD/theory/minimizer_selection.md

@@ -0,0 +1,42 @@
+# Minimizer selection
+
+## Definition
+
+A **minimizer** of a kmer window is the m-mer ($m < k$) that is smallest, among all $k - m + 1$ overlapping m-mers in the window, under a chosen ordering. The minimizer is always taken in canonical form (lexicographic minimum of forward and reverse complement) so that selection is strand-independent.
+
+The minimizer partitions a sequence into super-kmers: maximal runs of overlapping kmers that share the same minimizer (see [Kmers and super-kmers](kmers_and_superkmers.md)).
+
+## Hash-based ("random") minimizer
+
+`obikmer` selects minimizers by hash order rather than plain lexicographic order. Ordering m-mers lexicographically on their 2-bit encoding systematically favors AT-rich m-mers (an all-A m-mer always encodes to 0), which causes low-complexity regions to dominate as minimizers and produces unbalanced partitions.
+
+Instead, a well-distributed hash function $H$ is applied to the canonical (lexicographically minimal) form of each m-mer, and the m-mer with the smallest $H$ value wins. Because $H$ is a bijection with good avalanche properties, every distinct m-mer in a window has an equal chance of holding the minimum hash value, independent of its nucleotide composition.
+
+The canonical form used as input to $H$ is still the lexicographic minimum of forward/reverse-complement — hashing is applied on top of it, not used to redefine it. Defining canonicity by hash value instead would bias the *distribution of hash values themselves* toward small values (the minimum of two independent hashes is not uniformly distributed), reintroducing a bias one layer down.
+
+### Hash function
+
+The hash function is a 64-bit mixing function (splitmix64-style finalizer) applied to the m-mer XORed with a fixed non-zero seed:
+
+$$H(x) = \text{mix64}(x \oplus s), \quad s = \lfloor 2^{64}/\varphi \rfloor = \texttt{0x9e3779b97f4a7c15}$$
+
+```
+H(x):
+    x ← x  ⊕  0x9e3779b97f4a7c15
+    x ← x  ⊕  (x >> 30)
+    x ← x  ×  0xbf58476d1ce4e5b9
+    x ← x  ⊕  (x >> 27)
+    x ← x  ×  0x94d049bb133111eb
+    return x ⊕ (x >> 31)
+```
+
+The XOR seed avoids the finalizer's fixed point at 0 ($\text{mix64}(0) = 0$), which would otherwise make an all-A m-mer (canonical value 0) win every window comparison.
+
+## Partition routing is independent of minimizer selection
+
+The hash used to select a minimizer within a window (the minimum of several hash values) and the hash used to route a super-kmer to a storage partition are computed separately:
+
+- **Selection** uses $H$ applied to every candidate m-mer in the window, keeping the minimum.
+- **Partition routing** recomputes $H$ on the single selected minimizer only, once its position is fixed. This is a hash of one specific value, not the minimum of several, so it is uniformly distributed and safe to use directly for routing.
+
+See [Partitioning and indexing architecture](indexing_architecture.md) for how the routing value is turned into a partition index.

UserDocMD/usage/annotate.md

@@ -0,0 +1,25 @@
+# annotate
+
+Add or update genome metadata of an index from a CSV file, or dump the current metadata as CSV.
+
+```bash
+obikmer annotate INDEX --csv FILE [OPTIONS]
+obikmer annotate INDEX --dump
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory to annotate (modified in place) |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `--csv` | — | CSV file of metadata to apply (must contain an id column); required unless `--dump` is used |
+| `--sep` | `,` | CSV field separator |
+| `--id-col` | `id` | Name of the column containing genome labels |
+| `--na-value` | `NA` | Value meaning "remove this field" (deletes the existing key if present) |
+| `--no-overwrite` | off | Do not overwrite existing metadata keys |
+| `--dump` | off | Print all genome metadata as CSV to stdout instead of applying a file |

UserDocMD/usage/dump.md

@@ -0,0 +1,25 @@
+# dump
+
+Dump all kmers of an index as CSV, one row per kmer, with per-genome counts or presence.
+
+```bash
+obikmer dump INDEX [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory to dump |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `--force-presence` | off | Output presence/absence (0/1) even if the index stores counts |
+| `--debug` | off | Prefix each row with the partition and layer columns |
+| `--head N` | none | Limit output to the first N kmers |
+
+`dump` also accepts the shared [predicate options](filter.md#predicate-options) (`--ingroup`, `--outgroup`, `--min-count`, etc.) to restrict which kmers are dumped.
+
+Output is CSV on stdout.

UserDocMD/usage/estimate.md

@@ -0,0 +1,18 @@
+# estimate
+
+Estimate approximate-index parameters (z, evidence bits, false-positive rate) before building an index with `--approx`, without touching any files.
+
+```bash
+obikmer estimate [OPTIONS]
+```
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `-k, --kmer-size` | `31` | Kmer size used at query time (matches `index`'s `--kmer-size`) |
+| `-z, --findere-z` | none | Findere z parameter |
+| `--evidence-bits` | none | Fingerprint bits per slot (b) |
+| `--fp` | none | Target false-positive rate per z-window |
+
+Any two of `-z`, `--evidence-bits`, `--fp` may be given; the third is derived using the same model as `index --approx` and `reindex --approx` ($FP = 1 / 2^{b \cdot z}$). The report printed to stdout includes: query $k$, effective indexed $k$ ($k-z+1$), $z$, evidence bits, per-kmer false-positive rate, and per-z-window false-positive rate.

UserDocMD/usage/filter.md

@@ -0,0 +1,47 @@
+# filter
+
+Apply row-level selection to an index: retain only kmers matching ingroup/outgroup predicates over genome membership, plus optional total-count and complexity thresholds. The output is a new, single-layer index.
+
+```bash
+obikmer filter SOURCE -o OUTPUT [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `SOURCE` | Source index directory |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `-o, --output` | — (required) | Output index directory |
+| `-f, --force` | off | Overwrite an existing output directory |
+| `--presence` | off | Output presence/absence instead of counts |
+| `--min-total-count` | none | Minimum total count across all genomes (count index only) |
+| `--max-total-count` | none | Maximum total count across all genomes |
+| `--min-complexity` | none | Minimum normalized entropy (same score as `--theta` at index build time), recomputed from the stored unitig sequences |
+| `--complexity-level-max` | `6` | Maximum sub-word size for the complexity score (used only with `--min-complexity`) |
+
+## Predicate options
+
+| Option | Default | Description |
+|---|---|---|
+| `--ingroup` | none | Ingroup predicate (repeatable; each occurrence is ANDed) |
+| `--outgroup` | none | Outgroup predicate (repeatable; each occurrence is ORed) |
+| `--min-count` | 0, or group size + N if negative | Minimum number of ingroup genomes carrying the kmer |
+| `--max-count` | ingroup group size | Maximum number of ingroup genomes carrying the kmer |
+| `--min-frac` | `1.0` if `--ingroup` given without an explicit quorum, else `0.0` | Minimum fraction of ingroup genomes |
+| `--max-frac` | `1.0` | Maximum fraction of ingroup genomes |
+| `--min-outgroup-count` | `0` | Minimum number of outgroup genomes carrying the kmer |
+| `--max-outgroup-count` | `0` if `--outgroup` given without an explicit quorum, else outgroup group size | Maximum number of outgroup genomes |
+| `--min-outgroup-frac` | `0.0` | Minimum fraction of outgroup genomes |
+| `--max-outgroup-frac` | `1.0` | Maximum fraction of outgroup genomes |
+| `--presence-threshold` | `0` | Minimum count for a genome to be considered a carrier of a kmer |
+
+See [Genome predicates and taxonomy paths](predicates.md) for the predicate syntax used by `--ingroup`/`--outgroup`.
+
+A negative `--min-count`/`--max-count` is interpreted as an offset from the group size — e.g. `--min-count=-1` means "all but one".
+
+Declaring `--ingroup` with no explicit ingroup quorum flag implicitly sets `--min-frac 1.0` (present in every ingroup genome). Declaring `--outgroup` with no explicit outgroup quorum flag implicitly sets `--max-outgroup-count 0` (absent from every outgroup genome). Any explicit quorum flag for a group disables that group's implicit default.

UserDocMD/usage/index_command.md

@@ -0,0 +1,50 @@
+# index
+
+Build a genome index from one or more sequence files. Construction proceeds in phases (scatter → dereplicate → count → layered MPHF), described in [On-disk storage](../formats/index_layout.md).
+
+```bash
+obikmer index -o OUTPUT [OPTIONS] [INPUTS...]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INPUTS...` | Input sequence files or directories (FASTA/FASTQ/GenBank, gzip optional). If omitted, reads from stdin. |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `-o, --output` | — (required) | Output index directory |
+| `--force` | off | Overwrite an existing output directory |
+| `--label` | input file name without extension | Genome label stored in the index |
+| `--meta KEY=VALUE` | none | Attach a categorical metadata field to the genome (repeatable) |
+| `-k, --kmer-size` | `31` | Kmer size (odd, in [11, 31]) |
+| `-m, --minimizer-size` | `11` | Minimizer size (odd, in $[3, k-1]$) |
+| `--theta` | `0.7` | Entropy threshold for the low-complexity filter |
+| `--level-max` | `6` | Maximum sub-word size for the entropy score |
+| `-p, --partitions` | `256` | Number of partitions (rounded up to a power of 2) |
+| `-T, --threads` | detected core count | Number of worker threads |
+| `--max-open-files` | `threads / 4` (min 1) | Maximum number of input files open simultaneously |
+| `--min-abundance` | `1` | Minimum abundance (inclusive) for a kmer to be retained |
+| `--max-abundance` | none | Maximum abundance (inclusive) |
+| `--with-counts` | off | Store per-kmer counts; otherwise only presence/absence is stored |
+| `--keep-intermediate` | off | Keep intermediate build files instead of deleting them after construction |
+| `--approx` | off | Use approximate evidence (Findere fingerprint) instead of exact evidence |
+| `-z, --findere-z` | see below | Findere z parameter: number of consecutive kmers that must all match (approximate evidence only) |
+| `--evidence-bits` | see below | Fingerprint bits per slot (b), approximate evidence only |
+| `--fp` | see below | Target false-positive rate per z-window, approximate evidence only |
+| `--block-size` | `1` | Block size, in unitigs, for the exact on-disk index (rounded up to a power of 2) |
+
+## Exact vs. approximate evidence
+
+By default, an index stores **exact** evidence: a kmer is either present or absent (or has an exact count with `--with-counts`), with no false positives.
+
+With `--approx`, evidence is stored as a compact **fingerprint** instead, trading a small, tunable false-positive rate for reduced memory/disk usage. The false-positive model is:
+
+$$FP = \frac{1}{2^{b \cdot z}}$$
+
+where $b$ is `--evidence-bits` and $z$ is `--findere-z`. Any two of `-z`, `--evidence-bits`, `--fp` can be given and the third is derived; if none are given, defaults are $b=8$, $z=1$ ($FP \approx 1/256$). See [`estimate`](estimate.md) to explore this trade-off before building an index, and [`reindex`](reindex.md) to convert an existing index between the two representations.
+
+`z` must be strictly less than k: the effective indexed kmer length under approximate evidence is k−z+1.

UserDocMD/usage/merge.md

@@ -0,0 +1,29 @@
+# merge
+
+Merge multiple built indexes into a single index.
+
+```bash
+obikmer merge -o OUTPUT SOURCE... [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `SOURCE...` | Index directories to merge (at least one required) |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `-o, --output` | — (required) | Output index directory |
+| `--force` | off | Overwrite an existing output directory |
+| `--force-presence` | off | Store the merged index as presence/absence even if all sources have counts |
+| `--rename-duplicates` | off | Disambiguate duplicate genome labels (`.1`, `.2`, …) instead of failing |
+| `--budget-fraction` | `0.5` | Fraction of available RAM reserved as the memory budget for parallel partition merging |
+
+## Behaviour
+
+The output mode is chosen automatically: if every source index stores counts, the merged index stores counts too; otherwise it is presence/absence. `--force-presence` forces presence/absence regardless of the sources.
+
+By default, merging two indexes that share a genome label fails with an error; `--rename-duplicates` instead appends a numeric suffix to keep both copies.

UserDocMD/usage/name-tree.md

@@ -0,0 +1,21 @@
+# name-tree
+
+Translate a numerically-labelled tree export (TNT, PhyG, or any plain Newick file with bare `1`, `2`, `3`, … leaf labels) back to real taxon names, reading the label order from the FASTA that produced it.
+
+```bash
+obikmer name-tree TREE --fasta FASTA -o OUTPUT
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `TREE` | Tree file to translate — a TNT-style NEXUS export (`tree NAME = [&U] ...;`) or a plain Newick file |
+| `--fasta` | FASTA file whose record order gives the numeric taxon labels (1-based) — typically the `_sankoff.fasta`/`_snp.fasta` used to produce `TREE` |
+| `-o, --output` | Output NEXUS file path |
+
+## Output
+
+A NEXUS file with a `taxa` block, a `translate` table (numeric label → taxon name, from `--fasta`'s header order), and every tree found in `TREE`, topology unchanged — readable directly in FigTree, PearTree, `ape` (R), etc.
+
+`--tnt`'s and `--phyg`'s exports (see [phylo](phylo.md)) both number taxa `1..N` in the same order as the pseudo-alignment FASTA they were built from (`<prefix>_sankoff.fasta`), so pass that same file as `--fasta` here.

UserDocMD/usage/pack.md

@@ -0,0 +1,15 @@
+# pack
+
+Pack an index's per-column matrix files into a single-file format to reduce query-time I/O (fewer file opens per query).
+
+```bash
+obikmer pack INDEX
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory to pack (modified in place) |
+
+The index directory is locked for exclusive access while packing.

UserDocMD/usage/phylo.md

@@ -0,0 +1,192 @@
+# phylo
+
+Compute pairwise evolutionary-distance proxies between the genomes stored in an index — a plain distance matrix, optionally trees (NJ/UPGMA), and optionally a central-position SNP model with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE).
+
+```bash
+obikmer phylo INDEX [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory |
+
+## Distance matrix
+
+| Option | Default | Description |
+|---|---|---|
+| `--metric` | `jaccard` | One of `jaccard`, `mash`, `hamming`, `bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean` |
+| `--presence-threshold` | `1` | Minimum count for a kmer to be considered present, for Jaccard/Mash on a count index |
+| `--shared-kmers` | off | Also write the shared-kmer count matrix |
+| `--nj` | off | Compute and write a Neighbor-Joining tree (Newick) |
+| `--upgma` | off | Compute and write a UPGMA tree (Newick) |
+| `-o, --output` | none (stdout) | Output file prefix; without it, the distance matrix is printed to stdout as CSV |
+
+`hamming` requires a presence/absence index. All other metrics work on either index type; on a presence index, `jaccard`/`mash`/`hamming` are the only ones available.
+
+### Metric definitions
+
+- **jaccard**: $D = 1 - \dfrac{|A \cap B|}{|A \cup B|}$ over the sets of kmers present in each genome.
+- **mash**: derived from the Jaccard distance via $D = -\dfrac{1}{k} \ln\!\left(\dfrac{2J}{1+J}\right)$ where $J = 1 - D_{\text{jaccard}}$ and $k$ is the index's kmer size; clamped to 1.0 when $J \le 0$.
+- **hamming**: number of kmer positions where presence differs between the two genomes (presence index only, not normalized): $D = \sum_i \mathbb{1}[a_i \ne b_i]$.
+- **bray-curtis**: $D = 1 - \dfrac{2 \sum_i \min(c_i^A, c_i^B)}{\sum_i c_i^A + \sum_i c_i^B}$ on raw per-kmer counts.
+- **relfreq-bray-curtis**: the same formula computed on per-genome relative frequencies $p_i = c_i / \sum_j c_j$ instead of raw counts.
+- **euclidean**: $D = \sqrt{\sum_i (c_i^A - c_i^B)^2}$ on raw counts.
+- **relfreq-euclidean**: the same formula on relative frequencies.
+- **hellinger**: $D = \dfrac{1}{\sqrt{2}} \sqrt{\sum_i \left(\sqrt{p_i^A} - \sqrt{p_i^B}\right)^2}$ on relative frequencies, bounded in $[0, 1]$.
+- **hellinger-euclidean**: the unnormalized variant, $D = \sqrt{2} \times D_{\text{hellinger}}$.
+
+## Central-position SNP model
+
+This is a separate operation from the distance-matrix computation above: if any option below is used, no `--metric` matrix is computed in the same invocation.
+
+A **family** is the set of up to 4 kmers that share identical flanking sequence and differ only at the exact central base. Because $k$ is odd, the central position is well defined and maps to itself under reverse complementation. All computations below first require building the **sibling annex**, an index-wide record of which of the 4 possible central bases are observed at each family, across every genome.
+
+| Option | Default | Description |
+|---|---|---|
+| `--sibling-annex` | off | Build the sibling presence-mask annex (prerequisite for every option below) |
+| `--exclude-genome LABEL` | none | Exclude a genome (repeatable) from every SNP/Sankoff/export computation below |
+| `--min-shared-family N` | none | Auto-exclude any genome whose mean shared-family count against every other genome (see `--family-overlap`) falls below `N` — same exclusion as `--exclude-genome`, applied on top of it |
+| `--sibling-stats` | off | Write the family-size (sibling count) distribution, per genome and globally |
+| `--raw-snp-distance` | off | Write the single-copy central-SNP p-distance matrix |
+| `--raw-snp-counts` | off | Write per-pair diagnostic counts (n_snp, n_shared, n_eligible) instead of a matrix |
+| `--snp` | off | Write a SNP-only pseudo-alignment in FASTA, IUPAC-coded |
+| `--family-overlap` | off | Write an NxN matrix of, for each genome pair, how many variable families both genomes actually carry a call for; the diagonal holds each genome's own total family count |
+
+### Locus eligibility
+
+A family is eligible for a genome pair $(i, j)$ only if genome $i$ carries exactly one of the family's observed forms (single-copy, unambiguous) and genome $j$ also carries exactly one. A genome carrying more than one form at a locus makes that locus ineligible for any pair involving it.
+
+`--raw-snp-distance` tallies, over every eligible locus of every genome pair, $n_{\text{snp}}$ (the two genomes' single forms differ) versus $n_{\text{shared}}$ (they agree — this includes invariant families). The output ratio is $\hat{p} = \dfrac{n_{\text{snp}}}{n_{\text{snp}} + n_{\text{shared}}}$.
+
+`--snp` restricts itself to *variable* families (family size $\ge 2$) and writes one FASTA record per genome, one column per family, IUPAC-coded from each genome's presence mask at that family (a single form → the plain base; several forms → the matching IUPAC ambiguity code; no form → `-`).
+
+`--exclude-genome` removes a genome from these computations, re-checking column variability among the remaining genomes so that a column made monomorphic by the exclusion is dropped rather than kept artificially. It does not affect the `--metric` distance-matrix path.
+
+### Family overlap and low-coverage genomes
+
+`--family-overlap` writes, for every genome pair, how many variable families both genomes actually carry a call for (neither is absent) — a direct measure of how much informative content two genomes actually share. On genome-skim or otherwise incomplete-coverage collections, a genome with very little overlap with everything else has almost nothing left to constrain its position in a tree, and tends to end up placed unstably (near-zero branch length, grafted inside an unrelated clade) by `--tnt`/`--iqtree`.
+
+`--min-shared-family N` automates the fix: it excludes, before any computation, every genome whose mean shared-family count against all other genomes (the same statistic, averaged per row of the `--family-overlap` matrix) falls below `N`. There is no universal value for `N` — it depends on how divergent and how completely covered the genome collection is; inspect `--family-overlap`'s own output to find where the real gap sits before choosing a threshold.
+
+## Sankoff calibration and phylogenetic exports
+
+| Option | Default | Description |
+|---|---|---|
+| `--sankoff` | off | Calibrate a 16-state parsimony cost matrix and matching pseudo-alignment |
+| `--sankoff-ratio-ceiling` | `0.5` | Exclude genome pairs whose raw SNP ratio exceeds this value from the calibration |
+| `--free-loss` | off | Recode a family's non-detection as the `?` missing-data symbol instead of an ordinary, costed state, in `--sankoff`'s pseudo-alignment and every export built from it |
+| `--tnt` | off | Also write a TNT script (implies `--sankoff`) |
+| `--phyg` | off | Also write PhyG input files (implies `--sankoff`) |
+| `--iqtree` | off | Also write an IQ-TREE custom model and alignment (implies `--sankoff`) |
+| `--sankoff-cost-scale` | `100` | Integer scaling factor applied to costs before rounding (required by TNT/PhyG's integer-only cost commands) |
+
+### The 16-state model
+
+Each family is treated as a character with 16 possible states: one per subset of the 4 possible central bases actually observed (including the empty subset). Calibration combines two tallies, both restricted to genome pairs at or below `--sankoff-ratio-ceiling`:
+
+- a $5 \times 5$ transition matrix over family cardinality (0–4 observed forms) between paired genomes, and
+- a $4 \times 4$ base-substitution transition matrix from unambiguous single-copy loci,
+
+which are combined into a row-normalized $16 \times 16$ transition probability matrix $P$, converted to a symmetric cost matrix via $\text{cost}(a,b) = -\ln P(a,b)$.
+
+`--sankoff` alone writes the cost matrix, the calibration parameters, and a pseudo-alignment recoded so the empty state uses the symbol `0` (never a gap character, to avoid ambiguity with external tools' own gap semantics). It does not run any external tool.
+
+With `--free-loss`, the empty state is recoded to `?` instead — TNT/PhyG/IQ-TREE's own missing-data symbol — rather than an ordinary, costed 16th state. This matters for genome-skim or otherwise incomplete-coverage collections, where non-detection of a family is dominated by sampling failure rather than true evolutionary loss: scoring it as a real state risks grouping genomes by shared undersampling instead of shared ancestry. `?` rather than `-` because `-` still carries gap/indel semantics in these tools, and a non-detected family is not an observed deletion. `--free-loss` also drops the cardinality-transition cost between any two states, not just to/from the empty one: whether a genome shows 1 vs. 2 (etc.) detected members of a family it does carry is exactly as vulnerable to sampling failure as whether the family was detected at all, so gaining or losing a sibling is priced the same way — for free — as gaining or losing the whole family. Combine with `--min-shared-family`/`--family-overlap` above: `--free-loss` removes the false signal from non-detection, but a genome left with too little real overlap with everything else will still be placed unstably — excluding it is the other half of the fix.
+
+### Exports
+
+All three exports reuse the `--sankoff` calibrated matrix and pseudo-alignment, recoded for the target tool:
+
+- **`--tnt`**: a self-contained TNT script (alignment recoded to TNT's fixed 16-symbol alphabet, integer-scaled cost matrix re-closed to a metric, a default search block).
+- **`--phyg`**: a custom cost-matrix file plus a PhyG script reusing the `--sankoff` alignment directly.
+- **`--iqtree`**: a custom substitution-model file (exchangeability matrix recovered as $R(a,b) = e^{-\text{cost}(a,b)}$, plus empirical state frequencies) and a matching alignment, for maximum-likelihood inference with real branch lengths (unlike the parsimony step-counts from TNT/PhyG). Only states actually occurring in the alignment are kept and compactly renumbered.
+
+TNT and PhyG both write trees with bare numeric leaf labels (`1`, `2`, …, in the same order as `<prefix>_sankoff.fasta`). Use [`name-tree`](name-tree.md) on the tool's own tree output plus that same FASTA to get a NEXUS file with real taxon names.
+
+## Output files
+
+With `-o/--output PREFIX`, the relevant subset of the files below is written. Without `-o`, only the plain `--metric` distance matrix is produced, on stdout. All matrices use genome labels (from the index metadata) as row/column headers, in index order; all CSVs are comma-separated with a header row.
+
+### Distance matrix
+
+| File | Written by | Format | Content |
+|---|---|---|---|
+| `<prefix>_dist.csv` | always | CSV matrix | the `--metric` distance, 6 decimals, symmetric, diagonal 0 |
+| `<prefix>_shared.csv` | `--shared-kmers` | CSV matrix | shared-kmer count per genome pair (integers) |
+| `<prefix>_nj.nwk` | `--nj` | Newick | Neighbor-Joining tree, branch lengths from the `--metric` matrix |
+| `<prefix>_upgma.nwk` | `--upgma` | Newick | UPGMA tree, same matrix |
+
+Matrix layout (`_dist.csv`, `_shared.csv`, and every other "CSV matrix" below): header `genome,<label1>,<label2>,...`, one data row per genome, `<label>,<value1>,<value2>,...`.
+
+### Central-position SNP model
+
+| File | Written by | Format | Content |
+|---|---|---|---|
+| `<prefix>_siblings.csv` | `--sibling-stats` | CSV table | family-size distribution, per genome and global |
+| `<prefix>_rawsnp.csv` | `--raw-snp-distance` | CSV matrix | single-copy central-SNP p-distance ($\hat p$), or `NA` |
+| `<prefix>_rawsnp_counts.csv` | `--raw-snp-counts` | CSV table | per-pair diagnostic counts behind `_rawsnp.csv` |
+| `<prefix>_snp.fasta` | `--snp` | FASTA | SNP-only pseudo-alignment, IUPAC-coded |
+| `<prefix>_family_overlap.csv` | `--family-overlap` | CSV matrix | variable families both genomes of a pair carry a call for |
+
+**`_siblings.csv`** — family size = number of distinct central bases observed at a family (1–4), not "sibling count" (0–3).
+
+| Column | Meaning |
+|---|---|
+| `genome` | genome label, or the literal `global` for the last row |
+| `1`, `2`, `3`, `4` | for a genome row: number of families of that size where the genome carries ≥ 1 member. For the `global` row: the actual deduplicated family-size histogram — **not** the sum of the rows above (a family shared by several genomes would otherwise be counted once per genome) |
+
+**`_rawsnp.csv`** — same matrix layout as `_dist.csv`; each cell is $\hat p = n_{\text{snp}}/(n_{\text{snp}}+n_{\text{shared}})$, 6 decimals, or `NA` when the pair has zero eligible loci (distinguishes "identical everywhere eligible" from "nothing eligible at all").
+
+**`_rawsnp_counts.csv`** — one row per unordered genome pair (not a matrix), the counts `_rawsnp.csv`'s ratio is computed from:
+
+| Column | Meaning |
+|---|---|
+| `genome_a`, `genome_b` | the pair |
+| `n_snp` | eligible loci where the two genomes' single forms differ |
+| `n_shared` | eligible loci where they agree (includes invariant families) |
+| `n_eligible` | `n_snp + n_shared` |
+| `ratio` | $\hat p$ = `n_snp / n_eligible`, or `NA` if `n_eligible = 0` |
+
+**`_snp.fasta`** — one record per non-excluded genome, one column per variable family (family size ≥ 2), header carries an `n_sites` annotation. Each site is IUPAC-coded from the genome's presence mask at that family: single observed form → plain base; several forms → matching IUPAC ambiguity code; no form → `-`.
+
+**`_family_overlap.csv`** — same matrix layout as `_dist.csv`; cell `[i][j]` = number of `_snp.fasta` columns where both genome `i` and `j` carry a call (neither is `-`). Diagonal `[i][i]` is kept (not skipped): it holds genome `i`'s own total variable-family count.
+
+### Sankoff calibration and exports
+
+| File | Written by | Format | Content |
+|---|---|---|---|
+| `<prefix>_sankoff_matrix.csv` | `--sankoff`/`--tnt`/`--phyg`/`--iqtree` | CSV matrix | calibrated 16×16 cost matrix |
+| `<prefix>_sankoff_params.yaml` | same flags | YAML | calibration report (raw tallies + derived probabilities) |
+| `<prefix>_sankoff.fasta` | same flags | FASTA | Sankoff-recoded pseudo-alignment |
+| `<prefix>_sankoff.tnt` | `--tnt` | TNT script | ready-to-run parsimony search |
+| `<prefix>_sankoff.tcm` | `--phyg` | PhyG TCM | cost matrix in PhyG's own format |
+| `<prefix>_sankoff.pg` | `--phyg` | PhyG script | ready-to-run parsimony search |
+| `<prefix>_iqtree.model` | `--iqtree` | IQ-TREE model file | custom ML substitution model |
+| `<prefix>_iqtree.fasta` | `--iqtree` | FASTA | alignment recoded for that model |
+
+**`_sankoff_matrix.csv`** — header `state,0,A,C,M,G,R,S,V,T,W,Y,H,K,D,B,N`: the 16 symbols are IUPAC codes for the 16 subsets of the 4 possible central bases (bit 0=A, 1=C, 2=G, 3=T), `0` standing for the empty/absent state (not `-`, to avoid colliding with external tools' own gap syntax). One row per source state, one value per destination state, cost $-\ln P(a,b)$, 4 decimals.
+
+**`_sankoff_params.yaml`** — everything the calibration estimated, structured so it can be reloaded rather than re-parsed:
+
+| Key | Meaning |
+|---|---|
+| `ratio_ceiling` | the `--sankoff-ratio-ceiling` value used |
+| `cardinality_transitions` | 5×5 list of `{from, to, count, probability}`, family cardinality (0–4 observed forms) |
+| `composition_transitions` | 4×4 list of `{from, to, count, probability}`, base letters `A/C/G/T`, single-copy substitutions |
+
+**`_sankoff.fasta`** — same sites as `_snp.fasta`, recoded to match `_sankoff_matrix.csv`'s alphabet: absent state is `0` (or `?` under `--free-loss`). Excluded genomes dropped; columns left monomorphic by that exclusion are re-checked and dropped too.
+
+**`_sankoff.tnt`** (`--tnt`) — self-contained TNT script: `xread` block (alignment recoded to TNT's fixed `0-9A-F` alphabet), an integer-scaled (`--sankoff-cost-scale`) and metric-closed `smatrix`, a default `hold 20; mult; export` search. Run with `printf 'proc <path>;\nquit;\n' | tnt`. Produces `<prefix>_sankoff.tre` (bare numeric leaf labels, order matching `_sankoff.fasta`) — feed both into [`name-tree`](name-tree.md) to recover taxon names.
+
+**`_sankoff.tcm`** (`--phyg`) — first line: the 16-symbol alphabet plus a trailing gap symbol (17 total). Each following line: one row of the integer-scaled, metric-closed cost matrix (17 values — the extra gap column/row reuses the cost to/from the empty state `0`, since it's never actually triggered).
+
+**`_sankoff.pg`** (`--phyg`) — script: `read(prefasta:..., tcm:...)` against `_sankoff.fasta`/`_sankoff.tcm`, a default 300s/4-instance `search`, `report(...)` writing `<prefix>_sankoff.tre` (bare numeric labels, as for `--tnt`). Run with `phyg` from the output directory (the script uses relative file names). Feed the tree plus `_sankoff.fasta` into [`name-tree`](name-tree.md) for taxon names.
+
+**`_iqtree.model`** (`--iqtree`) — lower-triangular exchangeability matrix $R(a,b) = e^{-\text{cost}(a,b)}$ (one row of increasing length per state, whitespace-separated, PAML order), followed by one line of empirical state frequencies. Only states actually occurring in the alignment are kept, compactly renumbered `0..k-1`.
+
+**`_iqtree.fasta`** (`--iqtree`) — alignment recoded to that same compact `0..k-1` alphabet (symbols `0-9A-F`). Under `--free-loss`, non-detection becomes `?` and columns left non-informative once missing calls are ignored are dropped first (required for `+ASC`). Run with:
+```
+iqtree3 -s <prefix>_iqtree.fasta --seqtype MORPH -m <prefix>_iqtree.model+ASC --prefix <prefix>_iqtree -T AUTO
+```

UserDocMD/usage/predicates.md

@@ -0,0 +1,46 @@
+# Genome predicates and taxonomy paths
+
+Several commands ([`filter`](filter.md), [`select`](select.md), [`dump`](dump.md), [`unitig`](unitig.md)) select or group genomes using the same predicate language over genome metadata (see [`annotate`](annotate.md) for attaching metadata to a genome).
+
+## Predicate syntax
+
+| Form | Meaning |
+|---|---|
+| `*` or `all` | Matches every genome (case-insensitive) |
+| `key=v1\|v2` | Genome's `key` metadata equals one of the listed values |
+| `key!=v` | Genome's `key` metadata does not equal `v` |
+| `key~path` | Genome's `key` metadata (a taxonomy path) matches `path` (ancestry match) |
+| `key!~path` | Genome's `key` metadata does not match `path` |
+
+A genome whose metadata does not contain `key` at all cannot be classified by that predicate and is excluded from the relevant group's quorum count.
+
+Multiple `--ingroup` predicates are combined with AND; multiple `--outgroup` predicates are combined with OR. When both an ingroup and an outgroup predicate would match the same genome, ingroup classification wins.
+
+## Taxonomy paths
+
+A metadata value is treated as a taxonomy path when it starts with the literal prefix `taxonomy:/`; any other value is treated as a plain string and only supports `=`/`!=`.
+
+```
+taxonomy:/segment1@rank1/segment2@rank2/...
+```
+
+Each segment is a name, optionally annotated with a rank (e.g. `@family`, `@genus`, `@species`); ranks are optional and can be mixed within a path. The `@` character is reserved inside taxonomy paths and cannot appear in segment names or rank labels.
+
+### Path matching (`~` / `!~`)
+
+Matching compares segment names only (ranks are informational, not part of the match), with anchoring controlled by leading/trailing `/`:
+
+| Pattern | Matches |
+|---|---|
+| `A/B` | anywhere in the path |
+| `/A/B` | at the start of the path (prefix) |
+| `A/B$` | at the end of the path (suffix) |
+| `/A/B$` | the entire path (exact) |
+
+A rank-qualified query, `key@rank=value`, matches only when the path's segment at that specific rank equals `value`.
+
+### Example
+
+```bash
+obikmer filter source -o output --ingroup "taxon~/Betulaceae/Betula"
+```

UserDocMD/usage/query.md

@@ -0,0 +1,38 @@
+# query
+
+Query an index with sequences and annotate each query with the kmer matches found.
+
+```bash
+obikmer query INDEX INPUTS... [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory to query against |
+| `INPUTS...` | Input sequence files (FASTA/FASTQ, gzip optional); at least one required |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `--detail` | off | Report per-position, per-genome coverage vectors in the output |
+| `--count-missing` | off | Also count query kmers absent from the index |
+| `--force-presence` | off | Report presence (0/1) per genome instead of raw counts |
+| `--presence-threshold` | `1` | Minimum accumulated count to declare a genome present (implies `--force-presence`) |
+| `-z, --findere-z` | derived from the index metadata | Override the Findere z parameter |
+| `-T, --threads` | detected core count | Number of worker threads |
+| `--chunk-size` | auto-sized (available RAM ÷ threads, clamped to 4–256 MiB) | I/O chunk size, in MiB |
+| `--max-open-files` | `threads / 4` (min 1) | Maximum number of input files open simultaneously |
+
+## Output
+
+FASTA on stdout, one record per query, annotated in the OBITools-style header format `>id {"key":value,...}`:
+
+- `kmer_count`: total number of kmers matched
+- `kmer_missing`: number of query kmers absent from the index (only with `--count-missing`)
+- `kmer_strict_matches`: per-genome match counts
+- `coverage`: per-position, per-genome coverage vectors (only with `--detail`)
+
+`--mismatch` is accepted by the CLI but not currently functional; using it produces a warning and is ignored.

UserDocMD/usage/reindex.md

@@ -0,0 +1,25 @@
+# reindex
+
+Convert an existing index's evidence representation in place, between exact and approximate.
+
+```bash
+obikmer reindex INDEX [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory to convert (modified in place) |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `--approx` | off | Convert to approximate evidence (default direction is approximate → exact); requires `-z`/`--evidence-bits`/`--fp` |
+| `-z, --findere-z` | none | Findere z parameter (≥ 1) |
+| `--evidence-bits` | none | Fingerprint bits per slot (b) |
+| `--fp` | none | Target false-positive rate per z-window |
+| `--block-size` | `1` | Block size for the exact on-disk index (ignored when converting to approximate) |
+
+See [`index`](index_command.md#exact-vs-approximate-evidence) for the exact/approximate trade-off and the underlying false-positive model, and [`estimate`](estimate.md) to explore parameters beforehand. The index directory is locked for exclusive access during conversion.

UserDocMD/usage/select.md

@@ -0,0 +1,35 @@
+# select
+
+Project and/or aggregate the genome columns of an index into a new (or in-place) index. Where [`filter`](filter.md) selects rows (kmers), `select` operates on columns (genomes): grouping several genomes into one aggregated column, reordering columns, or dropping some.
+
+```bash
+obikmer select SOURCE (--output OUTPUT | --in-place) [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `SOURCE` | Source index directory |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `--output` | — | Output index directory (mutually exclusive with `--in-place`) |
+| `--in-place` | off | Rewrite the source index in place (mutually exclusive with `--output`) |
+| `-f, --force` | off | Overwrite an existing output directory |
+| `--group NAME:PRED` | none | Define a named group of genomes by predicate (repeatable; mutually exclusive with `--aggregate-by`) |
+| `--group-op NAME:OP` | none | Aggregation operator for a named group |
+| `--aggregate-by KEY` | none | Automatically create one group per distinct value of a metadata key (mutually exclusive with `--group`) |
+| `--aggregate-op OP` | none | Aggregation operator applied to every auto-generated group |
+| `--select COL,...` | all columns | Output columns, in order (group names or genome labels) |
+| `--presence-threshold` | `0` | Minimum count for a genome to be considered a carrier (logical operators only) |
+
+## Aggregation operators
+
+`any`, `all`, `none` (logical, evaluated against `--presence-threshold`), `sum`, `min`, `max` (numeric, count index only). If a group's operator is left unspecified, it defaults to `any` when the source is a presence/absence index and `sum` when it stores counts.
+
+A `select` never changes the underlying kmer set — only the per-genome data (counts or presence) is rewritten, so an unaggregated pass-through column (a plain genome label in `--select`) is a cheap copy.
+
+At least one of `--output`/`--in-place` is required, and at least one output column must be defined; every name listed in `--select` must resolve to either a defined group or an existing genome label. See [Genome predicates and taxonomy paths](predicates.md) for the predicate syntax used by `--group`.

UserDocMD/usage/superkmer.md

@@ -0,0 +1,27 @@
+# superkmer
+
+Extract super-kmers from one or more sequence files and write them to stdout, without building a full index. Useful for inspecting or piping the super-kmer decomposition of a dataset.
+
+```bash
+obikmer superkmer [OPTIONS] [INPUTS...]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INPUTS...` | Input sequence files or directories (FASTA/FASTQ/GenBank, gzip optional). If omitted, reads from stdin. |
+
+## Options
+
+| Option | Default | Description |
+|---|---|---|
+| `-k, --kmer-size` | `31` | Kmer size (must be odd, in [11, 31]) |
+| `-m, --minimizer-size` | `11` | Minimizer size (must be odd, in $[3, k-1]$) |
+| `--theta` | `0.7` | Entropy threshold; kmers with a normalized entropy at or below this value are excluded |
+| `--level-max` | `6` | Maximum sub-word size used for the entropy score |
+| `-p, --partitions` | `256` | Number of partitions (rounded up to the next power of 2) |
+| `-T, --threads` | detected core count | Number of worker threads |
+| `--max-open-files` | `threads / 4` (min 1) | Maximum number of input files open simultaneously |
+
+Output is written to stdout in the internal scatter format used by `index`; it is primarily intended to be piped into other tools or inspected for debugging.

UserDocMD/usage/unitig.md

@@ -0,0 +1,19 @@
+# unitig
+
+Dump the unitigs of an index as FASTA. A unitig is a maximal non-branching path through the de Bruijn graph implied by the index's kmers; the concatenation of every unitig reconstructs every stored kmer exactly once.
+
+```bash
+obikmer unitig INDEX [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory |
+
+## Options
+
+`unitig` accepts the shared [predicate options](filter.md#predicate-options) (`--ingroup`, `--outgroup`, `--min-count`, etc.) to restrict which kmers are included before the unitigs are enumerated.
+
+Output is FASTA on stdout.

UserDocMD/usage/utils.md

@@ -0,0 +1,26 @@
+# utils
+
+Miscellaneous index maintenance and inspection utilities.
+
+```bash
+obikmer utils INDEXES... [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEXES...` | One or more index directories |
+
+## Options
+
+| Option | Scope | Description |
+|---|---|---|
+| `--new-label NEW=OLD` | single index only | Rename a genome label |
+| `--upgrade-index` | single index only | Add any missing layer metadata files to an older index |
+| `--bits-per-kmer` | single index only | Print bits-per-kmer statistics |
+| `--stats` | single index only | Print per-genome kmer counts as CSV |
+| `--partition-stats` | one or more indexes | Print a partition-size distribution report |
+| `--csv FILE` | with `--partition-stats` | Also write raw per-(partition, source) data to FILE as CSV |
+
+At least one operation option must be given. All options except `--partition-stats` require exactly one index directory.

benchmark/Makefile

@@ -89,50 +89,50 @@ $(REF_DIST_CSVS) &: $(REF_NPZS) build_reference_dist.py
 
 reference_dist: $(REF_DIST_CSVS)
 
-# ── obikmer distance (presence index) ────────────────────────────────────────
+# ── obikmer phylo (presence index) ──────────────────────────────────────────
 
 $(OBIKMER_PRESENCE_DIST) &: global_index_presence/index.done $(BINARY)
 	mkdir -p obikmer_dist/presence
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/presence/jaccard \
 		--metric jaccard --shared-kmers --nj \
 		global_index_presence
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/presence/hamming \
 		--metric hamming --nj \
 		global_index_presence
 
 obikmer_dist_presence: $(OBIKMER_PRESENCE_DIST)
 
-# ── obikmer distance (count index) ───────────────────────────────────────────
+# ── obikmer phylo (count index) ─────────────────────────────────────────────
 
 $(OBIKMER_COUNT_DIST) &: global_index_count/index.done $(BINARY)
 	mkdir -p obikmer_dist/count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/jaccard \
 		--metric jaccard --shared-kmers --nj \
 		global_index_count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/bray_curtis \
 		--metric bray-curtis --nj \
 		global_index_count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/relfreq_bray_curtis \
 		--metric relfreq-bray-curtis --nj \
 		global_index_count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/euclidean \
 		--metric euclidean --nj \
 		global_index_count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/relfreq_euclidean \
 		--metric relfreq-euclidean --nj \
 		global_index_count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/hellinger \
 		--metric hellinger --nj \
 		global_index_count
-	$(BINARY) distance \
+	$(BINARY) phylo \
 		--output obikmer_dist/count/hellinger_euclidean \
 		--metric hellinger-euclidean --nj \
 		global_index_count

benchmark/build_reference_dist.py

@@ -2,10 +2,10 @@
 """Compute reference pairwise distance matrices from per-specimen .npz kmer indexes.
 
 Reads all .npz files in reference_index/ (each containing sorted uint64 `kmers`
-and uint32 `counts`), computes all distance metrics supported by `obikmer distance`,
+and uint32 `counts`), computes all distance metrics supported by `obikmer phylo`,
 and writes one CSV per metric to reference_dist/.
 
-Output CSV format matches `obikmer distance --output`:
+Output CSV format matches `obikmer phylo --output`:
   - first row: "genome", then specimen names
   - subsequent rows: specimen name, then float or int values
 

benchmark/compare_all_dist.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-"""Compare all reference distance matrices against obikmer distance outputs.
+"""Compare all reference distance matrices against obikmer phylo outputs.
 
 Reads from:
   reference_dist/   — ground-truth matrices computed by build_reference_dist.py
-  obikmer_dist/     — matrices produced by `obikmer distance`
+  obikmer_dist/     — matrices produced by `obikmer phylo`
 
 Handles label reordering: both matrices are sorted by genome label before
 element-wise comparison, so column/row order differences are irrelevant.

docmd/architecture/siblings.md

@@ -0,0 +1,620 @@
+# Sibling annex — architecture (discussion)
+
+Status: architecture decided (2026-08-14). Implementation not yet mandated.
+
+## Two index spaces, uncorrelated
+
+Every kmer stored in a `Layer` lives in two independent index spaces:
+
+- **Iteration order**: its position when enumerating `unitigs.bin` (the
+  superkmer file), deterministic but arbitrary with respect to slot.
+- **MPHF slot**: `MphfLayer::index(kmer)`, the number the MPHF assigns.
+
+The two are not correlated by any formula. Converting from one to the other
+requires either recomputing the MPHF (kmer → slot) or scanning the iteration
+stream (kmer → order). There is no `slot → kmer` operation: the MPHF is a
+one-way function, not an invertible bijection with a stored inverse. Any
+method that reconstructs a kmer from a bare slot number is wrong by
+construction, regardless of the mechanism used (MPHF re-hash, or evidence
+decode + direct unitig read). See `MphfLayer::kmer_at`
+(`obilayeredmap/src/mphf_layer.rs`) — flagged for removal, currently called
+from `obikphylo/siblings/build.rs` and `family_scan.rs` (since removed — see
+"Pending work" status below).
+
+## Two pipelines, never mixed
+
+| | origin of the kmer | membership known? | correct mapping |
+|---|---|---|---|
+| **query pipeline** | external (caller-supplied) | no | `query`/`find`/`find_strict` — MPHF + evidence check |
+| **iteration pipeline** | enumerated from this layer's own `unitigs.bin` | yes, by construction | `index`/`index_batch` — MPHF only, no evidence |
+
+Evidence exists solely to answer "is this external kmer a member of the
+layer" for the query pipeline. Using it (or the MPHF) to go the other way —
+recover a kmer from a slot, or re-verify a kmer that was just produced by
+iterating the layer — is a conceptual error: evidence can be probabilistic
+(`Approx` mode), so any slot→kmer attempt is unsound in general, and
+pointless even in `Exact`/`Hybrid` mode since the kmer was already known.
+
+## Sibling annex: an iteration-pipeline artifact only
+
+The sibling annex (`FamilyMask`/`SiblingAnnex`, `.psib`,
+`obicompactvec/src/siblingannex.rs`) records, per kmer, whether it is a
+family minorant and which family members are present in the index. Its only
+consumers (`obikphylo/siblings/stats.rs`, `family_scan.rs`) enumerate it
+exhaustively (`0..annex.len()`); no query-pipeline code path touches it.
+
+**Decision**: the annex must be persisted in iteration order, not slot
+order. This lets readers zip-iterate `Layer::iter_kmers()` and the annex
+file directly — one linear, cache-friendly pass, no MPHF/slot indirection,
+no `kmer_at`. It also enables specialized iterators building on this zip:
+minorants-only iteration, batch-of-kmers → batch-of-family-members, etc.
+
+Today the annex is built and stored in **slot** order
+(`build_layer_sibling_annex`, `siblings/build.rs`): `slot_kmer` is populated
+via `(0..n_slots).map(|slot| mphf.kmer_at(slot))`, and the origin `slot` is
+threaded through the whole cross-partition reconciliation pipeline (variant
+generation, `query_partition_with`, final `mask[slot].fetch_or(...)`). This
+must change to iterating `iter_kmers()`/`enumerate_kmers()` and threading
+the **iteration index** instead of the slot end to end — eliminating
+`kmer_at` from the build path entirely, not just the read path. No
+slot-indexed intermediate is needed even during construction; the
+iteration-order id is sufficient throughout.
+
+The cross-partition side of the same pipeline is unaffected: checking
+whether a generated family-variant kmer exists in another partition is a
+genuine query-pipeline operation (the variant's membership in the *target*
+partition is unknown) and must keep going through
+`KmerPartition::query_partition_with` (MPHF + evidence), never a raw
+`index()`.
+
+## Pending work — done
+
+The plan above shipped: `obikphylo` (a new crate — phylo-domain extension
+traits over `obikindex::KmerIndex`/`obilayeredmap::Layer<D>`, replacing the
+old `obikindex::siblings` module) builds and reads the annex purely in
+iteration order (`SiblingLayerExt::iter_siblings`/`iter_minorants`, both with
+batch variants, mirroring `Layer<D>`'s own `KmerIter`/`KmerBatchIter`
+shape). `MphfLayer::kmer_at` has no remaining callers.
+
+A separate, unrelated bug surfaced during this work and was fixed
+(2026-08-14): `MphfLayer::enumerate_kmers_batch` computed its
+`batch_start_index` via the stdlib `.enumerate()` adapter, which counts
+*batches* (0, 1, 2…), not the cumulative k-mer offset the annex is actually
+keyed on — every batch past the first wrote its mask/annex entries at the
+wrong iteration-order position. Fixed by tracking a running offset instead;
+regression tests added (`sibling_annex_no_empty_masks_after_build`,
+`sibling_histogram_does_not_panic_on_partial_last_batch`).
+
+## Performance: `build_sibling_annex` parallelism (2026-08-14)
+
+Investigated on a real multi-genome run (`phyloskims_sal_vac`, k=31/m=11).
+Baseline: mostly one active core, with short multi-core bursts — average
+~3 cores.
+
+**Fixes that helped, kept:**
+
+- `CanonicalKmerOf::minimizer()` (`obikseq/src/kmer.rs`) — a direct O(k)
+  bit-arithmetic minimiser for a single isolated k-mer, replacing a
+  `RollingStat` instance fed byte-by-byte through an ASCII round-trip (used
+  by `helpers::partition_of`, called for every generated family variant).
+  ~3x wall-clock improvement on its own, confirmed by sampling
+  (`obiskbuilder::rolling_stat`/`obikentropy` frames disappeared from the
+  hot path). `CanonicalKmerOf::partition()` added alongside it (wraps
+  `minimizer().seq_hash() & mask`, the same routing rule
+  `KmerPartition`/`RoutableSuperKmer` use).
+- Cross-partition resolution (`outgoing.par_iter()` in
+  `build_layer_sibling_annex`) parallelised at the *partition* level — one
+  Rayon task per non-empty `outgoing[dest]` bucket. For k=31/m=11, a
+  central-base substitution changes the winning minimiser (and thus the
+  destination partition) only when that window overlaps the central base:
+  ~11 of the 21 possible windows do, so ~10/21 (≈48%) of generated variants
+  route right back to the partition already being built. That self bucket
+  ends up far larger than any other, so the per-partition split pinned one
+  thread to it alone while the rest of the pool finished instantly —
+  confirmed by sampling: one thread solid in `MphfLayer::find`, everyone
+  else idle. Fixed by splitting each non-empty bucket into
+  `total_queries / n_workers` (capped 4096) chunks *before* `par_iter()`,
+  preserving per-partition mmap locality (each chunk stays contiguous
+  within one partition) while letting Rayon spread an oversized bucket
+  across several threads. Net effect of both fixes together: ~3 cores
+  average → ~10-13 cores average on the same run, and a projected total
+  build time of ~1h15 down to ~30min on the real `phyloskims_sal_vac` run
+  this was measured against.
+- `TracedBar`'s ETA (`obisys/src/progress.rs`) was silently starved: the
+  custom progress message and the self-computed ETA text used to share one
+  `pb.set_message()` slot, with the ETA holding off for 2s after any custom
+  message — fine when custom messages are rare, broken once
+  `build_sibling_annex`'s per-partition callback fires more often than
+  that. Fixed by keeping the two texts in separate fields, composed
+  together on every render instead of one overwriting the other.
+
+**Tried and reverted — do not repeat blindly:**
+
+- Parallelising the *outer* partition loop in `build_sibling_annex` with
+  `obikindex::PartitionRunner` (already used by `merge`/`build_layers`),
+  splitting a fixed core budget between outer (partition) and inner
+  (pipeline + resolution) concurrency so their product wouldn't exceed the
+  budget. Measured *worse*: throughput dropped over time (26
+  partitions/5min → 38/11-12min) and peak resolution concurrency fell from
+  ~11-12 cores to ~7-8. Cause: this capped the resolution burst — which
+  scales very well on its own — to make room for outer concurrency, and
+  running several partitions' resolution at once scatters access across
+  multiple partitions' mmap regions at once, working against the
+  locality `outgoing`'s per-partition grouping exists for. `PartitionRunner`
+  stayed exported from `obikindex` (`new_capped` too) since it's
+  general-purpose, but nothing in `obikphylo` calls it.
+- Splitting resolution chunks even finer (`/(n_workers*8)`, cap 1024,
+  instead of `/n_workers`, cap 4096) to smooth the residual sawtooth.
+  Measured ~10% *slower*, wider dips, not narrower. Reverted to the
+  original chunk sizing.
+
+**Known remaining limitation, not yet worth fixing:** within one layer, the
+four stages (sequential `unitigs.bin` read → parallel generation →
+parallel resolution → sequential annex write) never overlap — confirmed by
+1s-interval sampling: generation alone occupies ~17 threads evenly, but the
+next layer's read/generation never starts until the current layer's
+resolution and write are both done. This produces a real, periodic (~layer
+duration) alternation between "many cores" and "few cores" that neither of
+the fixes above touches, since both operate *within* one layer's resolution
+step. The only remaining lever is overlapping consecutive layers (e.g. a
+depth-2 pipeline: start layer N+1's read/generation while layer N's
+resolution/write is still running) — a real restructuring, not a parameter
+tweak, and explicitly *not* to be combined with the reverted
+budget-capping idea above (let each phase use however many cores it
+naturally wants; only the *scheduling* needs to overlap). Deferred, not
+started.
+
+## Cross-partition batch resolution — current state vs. the batched-accumulator design (discussion, 2026-08-14)
+
+`family_scan.rs::scan_layer_families` (shared by `snp_pseudo_alignment`,
+`sibling_annex_stats`, `cardinality_tally`, `scan_family_pairs`) already
+implements most of a dispatch/accumulate/resolve pipeline: generation
+(cheap, CPU-only — builds `outgoing[dest_partition]` from `FamilyMask` and
+buckets cross-partition queries) runs on an `obipipeline::throttle` +
+`make_pipe!` stage, decoupled from resolution (I/O-bound, `rayon::par_iter`
+*across partitions*, one generated batch resolved at a time, never several
+concurrently — this ordering is deliberate, see the module's own docs on a
+reverted concurrent-batch-resolution attempt that scattered mmap access).
+The fast/slow mode gate (`PartitionCache::fast_mode`, `cache.rs:162-163`)
+already exists: `n_layers <= 7` (checked once from the first non-empty
+partition's `PartitionMeta::n_layers`, documented as identical across every
+partition of an index — a structural, build-time property, never a
+per-partition state) decides whether `FamilyMask`'s recorded `layer_value`
+can be trusted to skip straight to the right layer
+(`find_presence_batch_fast`) or must fall back to scanning every layer of
+the destination partition (`find_presence_batch`).
+
+**Real gap, confirmed not implemented**: resolution is triggered by the
+*source* batch finishing (`FAMILY_BATCH = 65536` minorants read from the
+scanned layer), not by an *output* accumulator filling up. Since most
+central-base variants of a family route back to the same partition being
+scanned (~48% per the k=31/m=11 measurement above), a `FAMILY_BATCH`'s
+`outgoing[dest]` is large for the local/self partition and thin for the
+other ~255 (or however many) destination partitions — each of those gets
+resolved at low query density every batch instead of being accumulated
+across several source batches until resolving it is worthwhile. This is
+distinct from, and not fixed by, the fast/slow layer gate above.
+
+Redesign sketched (not built): per-destination accumulators decoupled from
+`FAMILY_BATCH`, flushed on reaching a size threshold instead of on source-batch
+completion — a "hot" accumulator for the partition being scanned (sharded
+one-per-generation-worker, no lock, since all `n_workers` pipeline workers
+write to it concurrently — this differs from an earlier, simpler mental
+model of "one thread owns one layer's local collector," which doesn't hold
+here since `n_workers` threads cooperate on scanning *one* layer at a time,
+not one thread per layer) and "cold" mutex-per-partition accumulators for
+the rest, low contention expected since traffic to any single cold
+destination is a small fraction of total.
+
+This breaks the current strict-iteration-order delivery of `on_family`
+(today: a reorder buffer keyed by batch, since a whole `FAMILY_BATCH`
+resolves atomically). With cross-batch accumulation, a family only becomes
+complete once *every* accumulator holding one of its outgoing queries has
+flushed, at unpredictable, independent times — no longer streamable
+strictly in order without a large, unbounded pending buffer. Resolution
+sketched: replace order-dependent consumers with coordinate-addressed
+writes instead of order-dependent appends (see `PseudoAlignment` idea
+below) wherever possible, since `sibling_annex_stats`'s reduction (plain
+counts) is already order-independent and needs nothing here.
+
+**Superseded 2026-08-15** by the `--subsample`/`--shannon` design below,
+which sidesteps the accumulator redesign for now: bounding the number of
+families actually resolved per layer (via sampling) keeps per-layer
+resolution volume small enough that the batch-density problem above stops
+mattering in practice for these two consumers. The accumulator redesign
+remains relevant for a future *unsampled, full-index* run, but is not
+required to ship `--subsample`/`--shannon`.
+
+## Pseudo-alignment at scale — pruning is unavoidable (discussion, 2026-08-14/15)
+
+The reference run (`phyloskims_sal_vac`-scale bacterial test set,
+`iqtree.fasta`) produced a dense alignment for 13 genomes × 383,965 sites
+(4.8 MB) — trivially small. The in-progress plant index is expected to
+carry on the order of 9 billion minorant families; a dense byte-per-cell
+alignment at that column count is unbuildable regardless of genome count
+(hundreds of GB even at a handful of genomes). Long-term ambition is 6,000–
+8,000 genomes on a large machine, which makes the per-cell cost dominant in
+the other dimension too. Pruning the retained family set before
+materializing anything is mandatory, not an optimization.
+
+**Already free**: `family_size() < 2` (no sibling variant registered at
+all) is a zero-cost structural filter, read directly off `FamilyMask` bits,
+already applied in `snp_pseudo_alignment`. Insufficient alone — per the
+~80-85% mono-family estimate from earlier discussion, this only brings 9
+billion down to roughly 1.3-1.8 billion, still unusable.
+
+**Entropy definition — settled 2026-08-15, correcting an earlier wrong
+turn.** The project does **not** encode families as IUPAC ambiguity codes
+interpreted the classical way (Fitch-parsimony subset-compatibility, or
+ML's "one true state, uncertain which"); see
+`docmd/theory/evolutionary_distances.md` ("Why the IUPAC/DNA encoding used
+for the first `--snp` test was wrong") and the Sankoff resolution that
+followed it. The real model is a genuine 16-state alphabet (the powerset of
+`{A,C,G,T}`, `∅` included as a real state) scored with a *calibrated
+pairwise cost matrix* (`obikphylo::cardcomp::pairwise_cost_matrix`,
+`cmd/phylo/sankoff.rs`), not a compatibility/subset relation between
+states. Under that model, each of the 16 states — including multi-bit ones
+like `AC` — is a first-class, independently-costed state, not an
+uncertainty encoding of a single true base. So: **entropy over the 15
+non-empty states (`∅` excluded, matching the earlier decision to exclude
+genomes where the family is absent) is the correct informativeness
+measure** for this project — not a 4-symbol reduction, which would discard
+exactly the cardinality/composition information the calibrated cost matrix
+is built to exploit.
+
+## `--subsample` / `--shannon` — sampling strategy (decided 2026-08-15)
+
+Goal: make both the pseudo-alignment (`--snp`) and a Shannon-entropy
+diagnostic usable at any index scale, from the 13-genome bacterial
+reference run up to the 9-billion-family plant index, without requiring the
+batched-accumulator redesign above.
+
+**`--subsample N`** (integer, families to retain): bounds the pseudo-alignment
+to `N` minorant families, sampled **proportionally per layer** among
+non-monomorphic minorants (`family_size >= 2`) — this sidesteps the need
+for a true global reservoir merge across layers while still approximating a
+uniform sample over the whole index, and directly answers the earlier open
+question of global-vs-per-layer selection scope.
+
+Three passes, in order:
+
+1. **Global count** (cheap, structural, parallel across layers — same shape
+   as the existing `sibling_family_size_histogram`, extended to report a
+   **per-layer** breakdown rather than one index-wide aggregate): for each
+   layer, `count_layer` = number of non-monomorphic minorants. Gives
+   `total_count = Σ count_layer`.
+2. **Per-layer proportional reservoir sampling** (cheap, structural, one
+   pass per layer, no cross-partition resolution): `N_layer = round(N ×
+   count_layer / total_count)`. Since `N_layer` is a proportion of
+   `count_layer`, it can never exceed it as long as `N <= total_count` — the
+   one edge case is `total_count <= N`, in which case sampling is skipped
+   entirely and *every* non-monomorphic minorant of every layer is kept
+   (no reservoir needed, `N` was never a real constraint). Otherwise:
+   Algorithm-R reservoir sampling over the layer's non-monomorphic minorant
+   indices, producing `N_layer` iteration-order indices directly, no
+   intermediate full list ever materialized.
+3. **Filtered resolution** (the expensive step, the existing
+   `scan_layer_families` engine, unchanged): re-scan the layer, generating
+   and resolving cross-partition queries **only** for the indices selected
+   in step 2 (cheap membership test against a small per-layer index set) —
+   this is what keeps `--subsample` cheap even on an unsampled-scale index,
+   since the cross-partition resolution volume is bounded by `N`, not by
+   the layer's true size.
+
+Steps 2 and 3 cannot be merged into one pass: true single-pass reservoir
+sampling would waste step-3's expensive resolution work on candidates later
+evicted by the reservoir. Step 1 must fully complete (every layer) before
+step 2 can start for any layer, since `total_count` is a global quantity.
+
+**`--shannon`** (no argument): emits a CSV of per-family Shannon entropy
+(15 non-empty states, `∅`/absent genomes excluded from the denominator, per
+the settled definition above). Independent of `--subsample` — entropy is
+computed and written per family as soon as its `genome_mask` resolves,
+O(1) memory per family, so it streams fine even unsampled at full index
+scale (a time cost, not a memory one). Combined with `--subsample N`, it
+delivers the original exploratory diagnostic (e.g. `--subsample 1000000
+--shannon`) directly from this general machinery, rather than a
+purpose-built one-off script.
+
+Validated end-to-end (2026-08-15) against real data: `--sibling-hist` on
+`phyloskims_sal_vac` (91 real genomes, k=31/m=11, 256 partitions × 2
+layers) confirms the ~9-billion-family estimate almost exactly (8,925,068,238
+total, 97.9% monomorphic — a sharper mono fraction than the ~80-85% earlier
+guess, corrected here). `--subsample`/`--shannon` on the smaller 20-genome
+bacterial reference (`benchmark/global_index_presence`) produced a sample
+size within rounding of the request (99,742/100,000) and a
+[0.8,1.2)-bucket share (40.7%) matching the full unsampled population
+(41.6%) — the two histograms only diverged wildly (5‰ vs 41.6%) under a
+real bug in `reservoir_sample_layer` (see next section), now fixed.
+
+**Bug found and fixed (2026-08-15): `family_idx` numbering mismatch.**
+`scan_layer_families`'s `family_idx` counts *every minorant* of a layer
+(monomorphic ones included, since `iter_minorants_batch` filters only on
+`is_minorant()`), not the raw annex slot (`SiblingAnnex::get(slot)` spans
+every k-mer, minorant or not) and not a counter over non-monomorphic
+minorants alone. `subsample.rs`'s `reservoir_sample_layer` originally
+stored raw slot numbers in its `HashSet<usize>` selection, which drifts
+away from `family_idx` as soon as *any* monomorphic minorant is seen —
+i.e. almost immediately, since ~98% of minorants are monomorphic. Fixed by
+tracking two separate counters: `family_idx` (every minorant, matching
+`scan_layer_families`) and `seen` (non-monomorphic minorants only, what
+Algorithm R actually samples over) — only `family_idx` values are ever
+stored in the selection set. The existing unit test never exercised this
+(its fixture has exactly one non-monomorphic family, always hitting the
+"keep everything" shortcut) — a stronger fixture with several interleaved
+monomorphic/non-monomorphic families would be needed to catch a regression
+here automatically; not yet written.
+
+## Cheap entropy pre-filtering — row-marginal sums (idea, not implemented, 2026-08-15)
+
+Motivation: on real data (bacterial reference, full unsampled run), only
+~5‰ of non-monomorphic minorants fall in the `[0.5, 1.5]` bit band judged
+phylogenetically interesting (entropy too low = uninformative near-invariant
+site; too high = saturated/noisy, see `family_entropy`'s 15-state
+discussion) — roughly 1 in 10,000 minorants overall. Computing exact
+entropy for every candidate just to discard 99.99% of them is wasteful at
+the full 9-billion scale.
+
+**The idea**: `PersistentBitMatrix::col_view(c)` gives a genome's whole
+presence column as a `BitSliceView` (sequential, no MPHF, no cross-partition
+routing — purely local to one layer's own matrix). Accumulating
+`TempCompactIntVecBuilder::inc_present(col)` (already exists in
+`obicompactvec/src/builder.rs:121`, along with `add`/`min`/`max`/`diff` on
+`IntSliceView` — no new low-level API needed) over every column of a layer
+produces `coverage[slot]`: how many genomes carry each exact k-mer, in one
+sequential per-layer pass, entirely decoupled from family/sibling
+structure. Persisted once per layer, a family's members' coverage could
+then be looked up via a plain local MPHF `index()` (cheap) instead of a
+full cross-partition presence resolution (`find_presence_batch`) — the
+expensive part today is specifically the cross-partition/cross-layer
+routing to a sibling's own matrix, not the bit-reading itself, and
+`coverage[slot]` sidesteps that routing entirely by moving the cost into a
+one-time, purely local, embarrassingly-parallel build step.
+
+**Why not implemented**: `coverage[slot]` is a per-member marginal —
+summing members' coverages to approximate a family's entropy silently
+assumes no genome carries more than one member at once. It cannot
+represent or detect joint co-occurrence (a genome carrying both `A` and
+`C` at once, i.e. a combined 15-symbol state) at all, which is exactly the
+phenomenon `family_entropy`'s 15-state definition exists to capture (see
+`CardinalityTally`/`cardinality_transition_probs`, the project's own
+existing machinery for this same co-occurrence structure, built for the
+Sankoff matrix calibration). A family that is in truth uniformly `AC`
+across every carrying genome would look like a well-balanced 2-state split
+under the marginal approximation (entropy ≈ 1) while its true 15-state
+entropy is 0 — i.e. the marginal proxy's failure mode lands on exactly the
+"saturated, uninformative" tail this pre-filter would need to catch,
+undermining the point. It stays plausible as a coarse filter for the
+*low* tail only (a dominant single member's marginal share reliably
+predicts low true entropy too), but not as a stand-in for the high tail —
+not pursued further for now.
+
+## `--free-loss`/`--tnt` pipeline: four independent scans, three of them unsampled (found 2026-08-15, not yet fixed)
+
+Measured on `phyloskims_sal_vac` (91 genomes): `obikmer phylo --subsample 500000
+--free-loss --tnt` logs four sequential stages —
+`raw_snp_distance` (1413s), `base_pair_tally` (1456s),
+`cardinality_tally` (2078s), `snp_pseudo_alignment` (143s). Reading the
+code (`obikmer/src/cmd/phylo/mod.rs:210-242`,
+`obikphylo/src/siblings/distance.rs`, `cardinality.rs`, `alignment.rs`)
+surfaced two compounding problems, not one:
+
+1. **Four separate full scans of the annex**, each opening its own
+   `KmerPartition`/`PartitionCache` and calling `scan_family_pairs`/
+   `scan_layer_families` independently — nothing computed in one stage is
+   reused by another. `base_pair_tally` is explicitly documented
+   (`distance.rs:138-143`) as a second full pass over the same
+   traversal `raw_snp_distance` already did, needed only because
+   `raw_snp_distance` doesn't keep the resolved bases, only aggregate
+   counts. `cardinality_tally` and `snp_pseudo_alignment` are each a
+   third and fourth independent full pass. Per the module's own earlier
+   profiling note (`family_scan.rs:26-28`, cited already above), this
+   traversal is page-fault/mmap-bound, not compute-bound — the ~10-12%
+   CPU efficiency ("contention" status) observed on these three slow
+   stages is consistent with I/O stalls scaled by repeated full scans,
+   not lock contention (there are no `Mutex`/`RwLock` anywhere in
+   `siblings/*.rs`; shared writes use per-slot `AtomicU8::fetch_or`).
+2. **Worse: `raw_snp_distance` and `cardinality_tally` don't honor
+   `--subsample` at all** — both call `scan_layer_families` with
+   `Selection::All` hardcoded, and `args.subsample` isn't even threaded
+   into their function signatures (`mod.rs:212`, `mod.rs:226`). Only
+   `snp_pseudo_alignment(args.subsample)` builds a real reservoir-sampled
+   `Selection::Some(set)` (via `compute_selections`, `alignment.rs:89`).
+   So today, `--subsample 500000` only bounds the pseudo-alignment step —
+   the SNP-distance matrix, the Sankoff base-pair calibration, and the
+   cardinality histogram are always computed over the **full, unsampled**
+   index regardless of the flag. This is not merely "different subsamples
+   per stage" (which would already be a problem worth fixing) — it's that
+   three of the four stages never subsample, which explains most of the
+   ~10x runtime gap against `snp_pseudo_alignment` on its own.
+
+**Decided requirement (2026-08-15, not yet implemented)**: all four
+stages must consume **one shared selection**, computed once, not each
+stage either scanning everything or drawing its own independent sample.
+Per-genome-pair SNP counts, the Sankoff base-pair calibration, the
+cardinality histogram, and the pseudo-alignment must all describe the
+same set of families — otherwise the calibration and the alignment it's
+meant to calibrate aren't even guaranteed to agree on which sites exist.
+Once selection is unified, doing four independent full scans stops making
+sense on its own terms: a single pass over the shared selection can feed
+all four accumulators (SNP/shared counts, base-pair tally, cardinality
+tally, pseudo-alignment output) at once, which is also the fix for
+problem 1 above — the two issues turn out to be one architectural
+decision, not two.
+
+**Forward-looking complication, explicitly flagged**: the eventual plan
+is to bias family *selection itself* by entropy15 (favor the informative
+mid-entropy band over uniform reservoir sampling — the exact motivation
+behind "Cheap entropy pre-filtering" above). That means selection can no
+longer be an uninformed draw made *during* the single fused scan; entropy
+(exact or a cheap proxy) has to be known *before* selection happens,
+which argues for a first, cheap, entropy-informed selection pass followed
+by one fused, unsampled-relative-to-that-selection scan — tying this
+decision directly to the still-unresolved marginal-proxy limitation
+described in "Cheap entropy pre-filtering" above (which only reliably
+predicts the *low*-entropy tail, not the high one — a real gap, not
+solved by this note, if entropy-biased selection ships before it is).
+
+## Two entropy definitions kept side by side, for comparison (2026-08-15)
+
+`--shannon`'s CSV carries both `entropy15` (`family_entropy` — the settled
+15-non-empty-state definition, see above) and `entropy4` (`family_entropy_4`
+— plain nucleotide reduction), computed from the same already-resolved
+`genome_mask`, not from the marginal approximation above. A genome carrying
+several bases at once contributes to *each* base's count (counted once per
+base present, not fractionally split, not folded into one combined state)
+— a genome polymorphic for the family is present at more than one base by
+construction, so it is expected to count more than once; the denominator is
+the total base-occurrence count, not the genome count (the two coincide
+only when no genome carries more than one base). Kept side by side
+specifically to measure, on real data, how much the two diverge — not yet
+analyzed.
+
+## `PersistentSparseBitMatrix` — implemented and measured (2026-08-15)
+
+A row-major (k-mer-major), deduplicated sparse alternative to
+`obicompactvec::PersistentBitMatrix`, motivated by the same sparsity that
+drove `--subsample`/`--shannon` above, but pursued as a foundational
+storage-layer change rather than an index-level workaround. Full design
+history, rationale, and rejected alternatives (external Elias-Fano crates,
+`cacheline-ef`, a single unsplit `dict_id` array) are in the dedicated
+implementation plan (`vivid-mapping-tiger.md` at the time of writing — the
+content below is the durable summary, not a pointer to a session-scoped
+file). Also directly informed by Alanko, Bille, Gørtz, Navarro, Puglisi,
+"Compact Data Structures for Collections of Sets" (2025,
+`biblio/Alanko et al. - Compact Data Structures for Collections of
+Sets.pdf`) — this design implements only their exact-duplicate special
+case (a plain dedup dictionary), not their full subset-containment
+hierarchy.
+
+**Design**: four on-disk components, each mmap-backed, built once per
+layer (matching how the rest of the build pipeline already works — never
+the whole multi-billion-row index at once): an `is_multi` rank-capable
+flag per row (singleton vs. multi-genome), a fixed-bit-width array for
+singleton rows (genome index directly, `ceil(log2(n_cols))` bits), a
+separate fixed-bit-width array for multi-genome rows (`dict_id`,
+`ceil(log2(n_distinct_multi_sets))` bits — kept apart from the singleton
+array specifically because `n_distinct_multi_sets` can be large in
+absolute terms even when multi-genome rows are a small *fraction* of all
+rows, and a single shared array would force every row, singletons
+included, to pay the wider width), and a deduplicated dictionary of
+distinct multi-genome sets (Elias-Fano-encoded byte offsets + a
+varint-encoded values blob). New low-level primitives added to
+`obicompactvec` to build this: `PersistentFixedIntVec` (arbitrary,
+runtime-parameterized bit width, width 0 included — needed once a real
+bug surfaced, see below), `PersistentRankSelectBitVec` (rank1/rank0/select1
+on top of the crate's existing `count_ones`, using
+`common_traits::SelectInWord`), `EliasFano` (composes the two). A new
+`BinaryMatrix` trait (`n`, `n_cols`, `row`/`fill_row`, `fill_sub_matrix`,
+`count_ones`) unifies dense and sparse at the one call site that needs
+both interchangeably (`obikphylo::siblings::cache::Mat`) — column-oriented
+methods (`col`, `col_view`, the `partial_*_dist_matrix` family) stay
+dense-only.
+
+**Two real bugs caught by tests, not by inspection**: (1) `EliasFano::open`
+re-derived its low-bits width from the persisted low-vector file's own
+width byte; the zero-width case was built with a dummy 1-bit placeholder
+(the builder rejected true width 0), so every reopened value silently
+doubled. Fixed by making `PersistentFixedIntVec` genuinely support width 0
+(no storage, `get` always 0) instead of working around the limitation in
+`EliasFano`. (2) An empty row (cardinality 0 — not expected on a real
+built index, but not guarded against either) was recorded as a singleton
+at genome 0, indistinguishable on read-back from a *real* singleton at
+genome 0. Fixed by routing cardinality-0 rows through the dictionary path
+(a genuine empty entry) instead of the singleton shortcut. Both caught by
+`obicompactvec`'s test suite (142 tests, including disk-reopen round-trips
+that drop every builder/mmap before reopening fresh), not by manual
+review — worth remembering next time a "this edge case can't happen in
+practice" shortcut is tempting.
+
+**Measured on real data** (`layer_1` of `phyloskims_sal_vac`'s
+`part_00018`, 30,246,774 rows, 91 genomes — `#[ignore]`d benchmarks in
+`obikphylo/src/siblings/tests.rs`):
+
+| | dense | sparse | ratio |
+|---|---|---|---|
+| on-disk size | 328.1MB | 43.7MB | **7.5x** smaller |
+| build time / peak RSS | — | 4.26s / 628MB | (per-layer, in-memory construction — comfortable) |
+| row access, sequential (2M reads) | 43ns/row | 32ns/row | sparse **faster** (smaller structure, better cache fit) |
+| row access, random (2M reads) | 409ns/row | 85ns/row | sparse **~4.8x faster** (the real `--shannon`/family-lookup shape) |
+| column access, one full column (30.2M rows) | 11.5ms | 993ms | sparse **86x slower** (no native column method — every read decodes a full row to keep one bit) |
+
+The row-access wins (both directions) weren't the design's stated goal —
+compactness was — but turn out real: dense's genome-major layout scatters
+a single row read across a much bigger file, which costs more than
+sparse's rank/select/varint decode once the file is this much smaller.
+The column-access cost is the flip side of the same layout choice, and is
+exactly what the next item below exists to fix.
+
+**Next, not yet planned**: rewrite `partial_jaccard_dist_matrix`/
+`partial_hamming_dist_matrix`/etc. (`obicompactvec/src/bitmatrix/pairwise.rs`)
+as a row-major co-occurrence accumulation (`O(Σ_rows k²)`, per-row
+increments into an `NxN` genome-pair counter — the known alternative to
+today's column-fold, plausibly cheaper on data this sparse, not just a
+fallback) so `obikindex`'s `--metric`/distance-matrix path can use the
+sparse type without the measured 86x column-access penalty. Needs its own
+design pass (in particular how it plugs into the `BitPartials`/
+`ColumnWeights` traits so both matrix types keep serving `--metric`)
+before implementation — not just "port the loop", a genuinely different
+algorithm.
+
+Also still deferred, unchanged from the implementation plan: full Alanko
+et al. subset-hierarchy compression (only the exact-duplicate special case
+is built), a sparse `PersistentCompactIntMatrix` (count matrices), and
+BRWT-style column-correlation exploitation.
+
+## Wired into `pack` and the sibling-annex build path (2026-08-15)
+
+`PersistentSparseBitMatrix` went from a validated but unused type to a
+real, selectable on-disk format:
+
+- **Generic `Layer<D>`**: `obilayeredmap::Layer<D>`'s presence-only methods
+  (`n_cols`, `sub_matrix`, `fill_sub_matrix`) are generic over any
+  `D: LayerData<Item = Box<[bool]>> + BinaryMatrix`, not hardcoded to
+  `PersistentBitMatrix` — `PersistentSparseBitMatrix` implements
+  `LayerData` (`open`/`read`) the same way. `find_slot`/`index_batch` were
+  already generic over any `D: LayerData`, so they needed no change.
+  Verified by `obilayeredmap`'s
+  `presence_layer_generic_over_sparse_matches_dense` test: build a dense
+  presence layer, convert it to sparse via `build_from_dense`, open both
+  as `Layer<PersistentBitMatrix>`/`Layer<PersistentSparseBitMatrix>` on
+  the same directory, assert `n_cols`/`sub_matrix`/`find_slot` agree.
+  (This test must stay at `k=4` with mutually non-colliding canonical
+  4-mers across its input sequences — `K`/`M` are process-wide
+  `AtomicUsize`s in test builds, not thread-local, so a test using a
+  different `k` races every other test in the same crate binary; a k=11
+  version of this test passed alone but failed under the full
+  `obilayeredmap` suite for exactly that reason before being fixed.)
+- **`obikphylo::siblings::cache::Mat`** gained a third variant,
+  `SparsePresence(Layer<PersistentSparseBitMatrix>)`, alongside `Count`
+  and `Presence` — every method (`find_slot`, `index_batch`,
+  `iter_minorants_batch`, `n_cols`, `fill_sub_matrix_carries`) dispatches
+  to it identically to `Presence`, since both go through the same generic
+  `Layer<D>` code. `PartitionCache::build` picks the variant per layer by
+  checking for `presence/is_multi.prsb` (the sparse format's own marker
+  file, see the design section above) before falling back to the dense
+  open path.
+- **`pack_sparse_bit_matrix`** (new, `obicompactvec::bitmatrix::sparse`):
+  `pack --sparse`'s entry point. Idempotent (checks `is_multi.prsb`
+  first); packs to dense `matrix.pbmx` first if that hasn't happened yet
+  (the dense→sparse transpose needs random row access, which only the
+  packed/columnar dense forms give), then `build_from_dense`s the sparse
+  form into the same directory and deletes `matrix.pbmx` — old-format
+  files are removed only after the new format is fully written, mirroring
+  `pack_bit_matrix`'s own crash-safety convention.
+- **CLI**: `obikmer pack --sparse` threads a `sparse: bool` through
+  `KmerIndex::pack_matrices` (all other call sites — `select`, `merge`,
+  `finalize_indexed` — pass `false`, unchanged dense behaviour). Count
+  matrices are untouched by `--sparse` (no sparse `PersistentCompactIntMatrix`
+  — see "still deferred" above).
+- **End-to-end coverage**: `obikphylo::siblings::tests::
+  sibling_annex_works_after_pack_sparse` builds a two-genome index, packs
+  it `--sparse`, asserts `is_multi.prsb` exists, then runs
+  `build_sibling_annex` and checks the resulting `FamilyMask`s match the
+  dense-path test (`sibling_annex_one_sibling_each`) exactly — proves the
+  sparse format round-trips through the real build pipeline
+  (`PartitionCache` sparse-detection included), not just the
+  `obicompactvec`/`obilayeredmap` unit layers below it.
+
+Full workspace `cargo test` (all crates, unit + doc tests) green after
+this change.

docmd/implementation/filtering.md

@@ -289,14 +289,14 @@ obikmer dump myindex --head 100
 obikmer dump myindex --head 20 --ingroup "species=Betula_nana" --min-count 1
 ```
 
-### `distance --presence-threshold N`
+### `phylo --presence-threshold N`
 
 When computing Jaccard distance on a **count index**, a k-mer is considered present in a genome if its count is ≥ N (default 1).
 This option is independent of the `--presence-threshold` used in filtering.
 
 ```sh
 # Jaccard treating kmers with count ≥ 2 as present
-obikmer distance myindex --metric jaccard --presence-threshold 2
+obikmer phylo myindex --metric jaccard --presence-threshold 2
 ```
 
 This parameter has no effect on presence/absence indexes (where values are already 0/1) or on metrics other than Jaccard.

docmd/index.md

@@ -13,7 +13,7 @@
 | `query`     | Query an index with sequences and annotate matches |
 | `dump`      | Dump all indexed k-mers as CSV (kmer + per-genome counts or presence); supports the shared [kmer filtering](implementation/filtering.md) system; `--head N` limits output to the first N k-mers |
 | `annotate`  | Add or update genome metadata from a CSV file; or dump metadata as CSV |
-| `distance`  | Compute pairwise distance matrix between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1) |
+| `phylo`     | Compute pairwise evolutionary-distance proxies between genomes (`--metric jaccard\|mash\|hamming\|bray-curtis\|relfreq-bray-curtis\|euclidean\|relfreq-euclidean\|hellinger\|hellinger-euclidean`); optionally build NJ/UPGMA trees; `--presence-threshold N` sets the minimum count to consider a k-mer present when computing Jaccard/Mash on count indexes (default 1); optionally a central-position SNP/Sankoff calibration with exports for TNT/PhyG/IQ-TREE (see [evolutionary distances](theory/evolutionary_distances.md)) |
 | `unitig`    | Build a global de Bruijn graph across all partitions and enumerate its unitigs as FASTA; supports the shared [kmer filtering](implementation/filtering.md) system |
 | `select`    | Project and/or aggregate genome columns into a new or in-place index; the column-axis counterpart of `filter` (see [select](implementation/select.md)) |
 | `estimate`  | Estimate approximate-index parameters (z, evidence bits, FP rates) before indexing |

iqtree.contree

@@ -0,0 +1 @@
+(Escherichia_coli--CFT073:0.7973338454,(Escherichia_coli--EDL933:0.5132866305,(Escherichia_coli--K-12_MG1655:0.2990687226,Escherichia_coli--K-12_W3110:0.3006288477)100:0.1655629834)69:0.0000007189,(((Klebsiella_pneumoniae--ATCC_13883:0.9661675519,(Klebsiella_pneumoniae--MGH_78578:0.3148953385,Yersinia_ruckeri--YRB:0.3576917943)84:0.4568464394)83:0.0727808943,(Klebsiella_pneumoniae--HS11286:0.1397401144,Proteus_mirabilis--HI4320:0.2192824480)77:0.3686486712)76:0.1162295711,((Salmonella_enterica--AKU_12601:0.6158730439,(Salmonella_enterica--LT2:0.4371443730,Salmonella_enterica--P125109:0.5815549929)100:0.1012287522)100:0.0714169372,Salmonella_enterica--CT18:0.2058143021)76:0.3641489914)100:0.2900565429);

iqtree.splits.nex

@@ -0,0 +1,62 @@
+#nexus
+
+BEGIN Taxa;
+DIMENSIONS ntax=13;
+TAXLABELS
+[1] 'Escherichia_coli--CFT073'
+[2] 'Escherichia_coli--EDL933'
+[3] 'Escherichia_coli--K-12_MG1655'
+[4] 'Escherichia_coli--K-12_W3110'
+[5] 'Klebsiella_pneumoniae--ATCC_13883'
+[6] 'Klebsiella_pneumoniae--HS11286'
+[7] 'Klebsiella_pneumoniae--MGH_78578'
+[8] 'Proteus_mirabilis--HI4320'
+[9] 'Salmonella_enterica--AKU_12601'
+[10] 'Salmonella_enterica--CT18'
+[11] 'Salmonella_enterica--LT2'
+[12] 'Salmonella_enterica--P125109'
+[13] 'Yersinia_ruckeri--YRB'
+;
+END; [Taxa]
+
+BEGIN Splits;
+DIMENSIONS ntax=13 nsplits=34;
+FORMAT labels=no weights=yes confidences=no intervals=no;
+MATRIX
+	100	 2,
+	100	 3,
+	100	 4,
+	100	 3 4,
+	69	 2 3 4,
+	100	 5,
+	100	 7,
+	16	 5 7,
+	100	 6,
+	100	 13,
+	16	 6 13,
+	23	 5 6 7 13,
+	100	 8,
+	100	 10,
+	23	 8 10,
+	100	 9,
+	100	 11,
+	100	 12,
+	100	 11 12,
+	100	 9 11 12,
+	23	 8 9 10 11 12,
+	100	 1 2 3 4,
+	100	 1,
+	84	 7 13,
+	83	 5 7 13,
+	77	 6 8,
+	76	 5 6 7 8 13,
+	76	 9 10 11 12,
+	31	 1 2,
+	1	 5 6 7 8,
+	1	 10 13,
+	1	 9 10 11 12 13,
+	1	 5 6 13,
+	1	 5 6 8,
+;
+END; [Splits]
+

mkdocs-user.yml

@@ -0,0 +1,62 @@
+site_name: obikmer — User Guide
+docs_dir: UserDocMD
+site_dir: doc-user
+
+theme:
+  name: material
+
+plugins:
+  - mermaid2:
+  - bibtex:
+      bib_file: UserDocMD/references.bib
+      csl_file: UserDocMD/ecology-letters.csl
+      enable_inline_citations: false
+
+markdown_extensions:
+  - admonition
+  - footnotes
+  - tables
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
+  - pymdownx.arithmatex:
+      generic: true
+
+extra_javascript:
+  - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js
+
+nav:
+  - Home: index.md
+  - Installation: installation.md
+  - Theory:
+      - Kmers and super-kmers: theory/kmers_and_superkmers.md
+      - DNA encoding: theory/encoding.md
+      - Low-complexity kmer filter: theory/entropy_filter.md
+      - Minimizer selection: theory/minimizer_selection.md
+      - Partitioning and indexing architecture: theory/indexing_architecture.md
+  - Usage:
+      - superkmer: usage/superkmer.md
+      - index: usage/index_command.md
+      - merge: usage/merge.md
+      - filter: usage/filter.md
+      - select: usage/select.md
+      - query: usage/query.md
+      - dump: usage/dump.md
+      - annotate: usage/annotate.md
+      - phylo: usage/phylo.md
+      - name-tree: usage/name-tree.md
+      - unitig: usage/unitig.md
+      - estimate: usage/estimate.md
+      - reindex: usage/reindex.md
+      - utils: usage/utils.md
+      - pack: usage/pack.md
+      - Predicates and taxonomy paths: usage/predicates.md
+  - Formats:
+      - Index construction and on-disk layout: formats/index_layout.md
+  - Architecture notes: architecture.md
+
+watch:
+  - UserDocMD
+  - mkdocs-user.yml

mkdocs.yml

@@ -58,6 +58,7 @@ nav:
   - Architecture:
       - Sequences: architecture/sequences/invariant.md
       - Kmer index: architecture/index_architecture.md
+      - Sibling annex (discussion): architecture/siblings.md
       - NUMA-aware worker pools: architecture/numa_worker_pools.md
       - NUMA-aware partition runner: architecture/numa_partition_runner.md
 

nj.nwk

@@ -0,0 +1 @@
+(((((((((((((Salmonella_enterica--P125109:0.009029074806027838,Salmonella_enterica--LT2:0.010374535686866677):0.00299392644685007,Salmonella_enterica--AKU_12601:0.010129090864056669):0.008889984754373365,Salmonella_enterica--CT18:0.029563522158784489):0.05178212639099835,(Escherichia_coli--CFT073:0.018265758526676266,(Escherichia_coli--EDL933:0.01586939152374907,(Escherichia_coli--K-12_MG1655:0.0043321525684933419,Escherichia_coli--K-12_W3110:0.004373581717010194):0.007835004272817602):0.005522415854494232):0.05395833543383322):0.007199364670842333,(Klebsiella_pneumoniae--HS11286:0.027306681424389149,(Klebsiella_pneumoniae--ATCC_13883:0.015176538442299506,Klebsiella_pneumoniae--MGH_78578:0.026462018414958045):0.0015885400482290244):0.060738462006352359):0.029481404183583916,Yersinia_ruckeri--YRB:0.10269826524749153):0.004316815340436764,Proteus_mirabilis--HI4320:0.11745645944715785):0.04439101156727093,Wolbachia_endosymbiont--GCF_000306885.1_ASM30688v1:0.17151356406519997):0.011344480816758762,Acidobacterium_capsulatum--ATCC_51196:0.17121217909071463):0.01869709592355495,Shouchella_clausii--KSM-K16:0.1554729916092201):0.04397682726757515,Bacillus_subtilis--168:0.10403127799132669):0.08828869671555731,Opitutus_terrae--PB90-1:0.06380512639846864):0.41655625366738377,Candidozyma_auris--GCF_003013715.1_ASM301371v2:0.22735026269091164,Saccharolobus_islandicus--M.16.4:0.21891918239548398);

phylo_archi.md

@@ -0,0 +1,58 @@
+**Nous avons un vrai problème architectural dans la partie phylo, basée sur les sibling.**
+
+La structure de l'index est en quatre parties, à l'intérieur d'un layer:
+
+- Un fichier de superkmers qui permet d'itinérer sur les kmers gérées par le layer.
+- Un MPFH qui permet de convertir un kmer en un numéro de slot compact.
+- Un tableau d'évidence, qui peut être exact ou probabiliste. Ce qui permet de vérifier lorsque l'on interroge le MPFH avec un kmer, si ce kmer est contenu dans le layer.
+  - Il est donc une **erreur conceptuelle grave** d'utiliser cette évidence pour retrouver un kmer à partir de son numéro de slot. Toute tentative en ce sens est vouée à l’échec sur une évidence non exacte.
+- Un tableau de présence ou de comptage par génome. Ranger colonne first : une colonne correspond à un génome, et indexé par numéro de slot
+
+## Concernant le MPFH.
+
+- Le rôle premier du MPFH est de convertir un kmer en un numéro de slot.
+- On peut, en rôle secondaire, en combinant le MPFH et le tableau d'évidence, s'en servir pour mettre en place un test d'appartenance au layer pour un kmer.
+
+L'API du `Layer<D>` expose également des méthodes de mapping brut kmer → slot via le MPHF, sans vérification d'appartenance :
+
+- `index(&self, kmer: CanonicalKmer) -> usize` — retourne le numéro de slot pour un kmer. C'est un mapping pur, équivalent à `MphfOnly::index`.
+- `index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize>` — retourne un vecteur de slots pour un slice de kmers.
+
+Ces méthodes sont distinctes de `query()` et `find()` qui incluent la vérification d'appartenance.
+
+Lorsque l'on itère sur le fichier de superkmer, l'ordre des kmer est déterministe, mais non corrélé avec les numéros de slot correspondants. Il existe donc un numéro d'ordre dans les kmer contenues dans le fichier de superkmer.
+
+L'API du `Layer<D>` expose maintenant quatre itérateurs sur les kmers du layer :
+
+- `iter_kmers(&self) -> KmerIter<'_>` — itérateur nommé sur `CanonicalKmer`, construit à partir de `unitigs.bin`. Plusieurs instances peuvent coexister concurremment tant que le layer vit.
+- `enumerate_kmers(&self) -> Enumerate<KmerIter<'_>>` — variante indexée retournant `(usize, CanonicalKmer)`, où l'index 0 correspond au premier kmer du fichier de superkmers. Construit par composition sur `iter_kmers()` sans duplication de code d'itération.
+- `iter_kmers_batch(&self, n: usize) -> KmerBatchIter<'_>` — itérateur par batch retournant `Vec<CanonicalKmer>` de taille `n`. Le dernier batch peut être tronqué.
+- `enumerate_kmers_batch(&self, n: usize) -> Enumerate<KmerBatchIter<'_>>` — variante indexée retournant `(usize, Vec<CanonicalKmer>)`, où l'index correspond au premier kmer du batch. Construit par composition sur `iter_kmers_batch()`.
+
+L'itération est implémentée via des structs `KmerIter` et `KmerBatchIter` qui encapsulent l'itérateur interne de `UnitigFileReader::iter_indexed_canonical_kmers()`. Les structs sont publics et peuvent être stockés, transmis ou combinés avec d'autres adaptateurs d'itérateur.
+
+## Performance — itérateurs de batch
+
+Les itérateurs `iter_kmers_batch` et `enumerate_kmers_batch` allouent un nouveau `Vec` à chaque batch. Pour des tailles de batch importantes ou des chemins critiques, cela peut générer une pression malloc significative.
+
+**À concevoir** : un système de pool de vecteurs avec réallocation automatique dès qu'un buffer n'est plus référencé, pour réutiliser les allocations entre batches et réduire le nombre d'appels système. Ce pool pourrait être intégré à `KmerBatchIter` ou proposé comme un adaptateur d'itérateur générique.
+
+## Accès aux matrices de présence/comptage
+
+Les types de vecteurs persistants exposent maintenant des méthodes de lookup batch optimisées pour l'accès séquentiel au mmap :
+
+- `PersistentCompactIntVec::get_batch(slots) -> Vec<u32>` et `fill_batch(slots, out)`
+- `PersistentBitVec::get_batch(slots) -> Vec<bool>` et `fill_batch(slots, out)`
+- `IntSliceView::get_batch(slots) -> Vec<u32>` et `fill_batch(slots, out)`
+- `BitSliceView::get_batch(slots) -> Vec<bool>` et `fill_batch(slots, out)`
+
+Toutes ces méthodes trient les slots en interne pour un accès mmap séquentiel, puis réordonnent les résultats selon l'ordre d'origine. `fill_batch` évite l'allocation en remplissant un buffer fourni par le caller.
+
+Les matrices persistantes exposent `sub_matrix(slots) -> Vec<Vec<T>>` et `fill_sub_matrix(slots, out)` :
+
+- `PersistentCompactIntMatrix::sub_matrix(slots)` — retourne `Vec<Vec<u32>>` column-first
+- `PersistentCompactIntMatrix::fill_sub_matrix(slots, out: &mut [Vec<u32>])` — remplit des buffers fournis, réutilise les allocations existantes
+- `PersistentBitMatrix::sub_matrix(slots)` — retourne `Vec<Vec<bool>>` column-first
+- `PersistentBitMatrix::fill_sub_matrix(slots, out: &mut [Vec<bool>])` — remplit des buffers fournis, réutilise les allocations existantes
+
+Le scan est colonne-first pour respecter la layout mémoire. `fill_sub_matrix` trie les slots une seule fois, puis appelle `fill_batch_sorted` sur chaque colonne pour éviter le tri redondant. Ces méthodes sont également disponibles sur `Layer<PersistentCompactIntMatrix>` et `Layer<PersistentBitMatrix>`.

sankoff_params.yaml

@@ -0,0 +1,167 @@
+ratio_ceiling: 0.5
+cardinality_transitions:
+- from: 0
+  to: 0
+  count: 54999793
+  probability: 0.7934021725308564
+- from: 0
+  to: 1
+  count: 8873859
+  probability: 0.12801028195383377
+- from: 0
+  to: 2
+  count: 5305545
+  probability: 0.07653539585976665
+- from: 0
+  to: 3
+  count: 126109
+  probability: 0.001819191475424167
+- from: 0
+  to: 4
+  count: 16149
+  probability: 0.00023295818011898336
+- from: 1
+  to: 0
+  count: 8873859
+  probability: 0.8510749571674425
+- from: 1
+  to: 1
+  count: 1041044
+  probability: 0.09984455215137214
+- from: 1
+  to: 2
+  count: 507272
+  probability: 0.048651493749477304
+- from: 1
+  to: 3
+  count: 4248
+  probability: 0.00040741760918753563
+- from: 1
+  to: 4
+  count: 225
+  probability: 0.00002157932252052625
+- from: 2
+  to: 0
+  count: 5305545
+  probability: 0.9117891245815953
+- from: 2
+  to: 1
+  count: 507272
+  probability: 0.08717767784549091
+- from: 2
+  to: 2
+  count: 5227
+  probability: 0.0008982907041949506
+- from: 2
+  to: 3
+  count: 689
+  probability: 0.00011840870388182915
+- from: 2
+  to: 4
+  count: 96
+  probability: 0.000016498164836945715
+- from: 3
+  to: 0
+  count: 126109
+  probability: 0.9614164824273843
+- from: 3
+  to: 1
+  count: 4248
+  probability: 0.03238545399100404
+- from: 3
+  to: 2
+  count: 689
+  probability: 0.005252725470763132
+- from: 3
+  to: 3
+  count: 98
+  probability: 0.0007471220553480217
+- from: 3
+  to: 4
+  count: 26
+  probability: 0.00019821605550049553
+- from: 4
+  to: 0
+  count: 16149
+  probability: 0.9781344639612356
+- from: 4
+  to: 1
+  count: 225
+  probability: 0.013628104179285281
+- from: 4
+  to: 2
+  count: 96
+  probability: 0.00581465778316172
+- from: 4
+  to: 3
+  count: 26
+  probability: 0.0015748031496062992
+- from: 4
+  to: 4
+  count: 14
+  probability: 0.0008479709267110841
+composition_transitions:
+- from: 'A'
+  to: 'A'
+  count: 161774
+  probability: 0.4701762136303263
+- from: 'A'
+  to: 'C'
+  count: 27788
+  probability: 0.0807624007835592
+- from: 'A'
+  to: 'G'
+  count: 130019
+  probability: 0.3778842157577942
+- from: 'A'
+  to: 'T'
+  count: 24490
+  probability: 0.07117716982832031
+- from: 'C'
+  to: 'A'
+  count: 27788
+  probability: 0.0780165140757087
+- from: 'C'
+  to: 'C'
+  count: 184047
+  probability: 0.5167232390273484
+- from: 'C'
+  to: 'G'
+  count: 19622
+  probability: 0.05508996830263265
+- from: 'C'
+  to: 'T'
+  count: 124724
+  probability: 0.3501702785943102
+- from: 'G'
+  to: 'A'
+  count: 130019
+  probability: 0.3610094570655886
+- from: 'G'
+  to: 'C'
+  count: 19622
+  probability: 0.05448224926003876
+- from: 'G'
+  to: 'G'
+  count: 183951
+  probability: 0.5107565097152884
+- from: 'G'
+  to: 'T'
+  count: 26562
+  probability: 0.07375178395908417
+- from: 'T'
+  to: 'A'
+  count: 24490
+  probability: 0.07335783586895636
+- from: 'T'
+  to: 'C'
+  count: 124724
+  probability: 0.37360076443118473
+- from: 'T'
+  to: 'G'
+  count: 26562
+  probability: 0.07956434611479049
+- from: 'T'
+  to: 'T'
+  count: 158067
+  probability: 0.4734770535850684

src/Cargo.lock

@@ -48,6 +48,15 @@ dependencies = [
  "as-slice",
 ]
 
+[[package]]
+name = "aligned-vec"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b"
+dependencies = [
+ "equator",
+]
+
 [[package]]
 name = "allocator-api2"
 version = "0.2.21"
@@ -497,6 +506,18 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "console"
+version = "0.16.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.10.2"
@@ -820,6 +841,26 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "equator"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc"
+dependencies = [
+ "equator-macro",
+]
+
+[[package]]
+name = "equator-macro"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
@@ -907,6 +948,16 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs4"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8c6b3bd49c37d2aa3f3f2220233b29a7cd23f79d1fe70e5337d25fb390793de"
+dependencies = [
+ "rustix 0.38.44",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "fs_at"
 version = "0.2.1"
@@ -1228,13 +1279,26 @@ version = "0.17.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
 dependencies = [
- "console",
+ "console 0.15.11",
  "number_prefix",
  "portable-atomic",
  "unicode-width",
  "web-time",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.18.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c"
+dependencies = [
+ "console 0.16.4",
+ "portable-atomic",
+ "unicode-width",
+ "unit-prefix",
+ "web-time",
+]
+
 [[package]]
 name = "infer"
 version = "0.19.0"
@@ -1326,12 +1390,9 @@ dependencies = [
 
 [[package]]
 name = "kodama"
-version = "0.2.3"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a44f3a71a44fbf49ce38152db7dc9adf959d4fe5c29344cd1858bdbda8d9091"
-dependencies = [
- "num-traits",
-]
+checksum = "f868feceed4703c842925c14232667c5f29c8059d0354154b2d06ec3bf9daf82"
 
 [[package]]
 name = "lazy_static"
@@ -1365,6 +1426,12 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.12.1"
@@ -1654,6 +1721,7 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 name = "obicompactvec"
 version = "0.1.0"
 dependencies = [
+ "common_traits",
  "memmap2",
  "ndarray",
  "rayon",
@@ -1695,36 +1763,37 @@ version = "0.1.0"
 dependencies = [
  "crossbeam-channel",
  "hwlocality",
- "indicatif",
+ "indicatif 0.17.11",
  "ndarray",
  "obicompactvec",
  "obikpartitionner",
  "obikseq",
  "obilayeredmap",
- "obipipeline",
  "obiread",
- "obiskbuilder",
  "obiskio",
  "obisys",
+ "obitaxonomy",
  "rayon",
  "serde",
  "serde_json",
  "tempfile",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
 name = "obikmer"
-version = "1.1.40"
+version = "1.1.44"
 dependencies = [
  "clap",
  "csv",
- "indicatif",
+ "indicatif 0.18.6",
  "kodama",
  "obidebruinj",
  "obifastwrite",
  "obikindex",
  "obikpartitionner",
+ "obikphylo",
  "obikrope",
  "obikseq",
  "obilayeredmap",
@@ -1736,7 +1805,9 @@ dependencies = [
  "obitaxonomy",
  "pprof",
  "rayon",
+ "serde",
  "serde_json",
+ "serde_yaml",
  "speedytree",
  "tracing",
  "tracing-subscriber",
@@ -1748,7 +1819,7 @@ version = "0.1.0"
 dependencies = [
  "cacheline-ef",
  "epserde",
- "indicatif",
+ "indicatif 0.17.11",
  "memmap2",
  "niffler 3.0.0",
  "obicompactvec",
@@ -1772,6 +1843,29 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "obikphylo"
+version = "0.1.0"
+dependencies = [
+ "memmap2",
+ "ndarray",
+ "obicompactvec",
+ "obikindex",
+ "obikpartitionner",
+ "obikseq",
+ "obilayeredmap",
+ "obipipeline",
+ "obiread",
+ "obiskbuilder",
+ "obiskio",
+ "obisys",
+ "rand 0.8.6",
+ "rayon",
+ "tempfile",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "obikrope"
 version = "0.1.0"
@@ -1852,7 +1946,7 @@ dependencies = [
  "memmap2",
  "niffler 3.0.0",
  "obikseq",
- "rustix",
+ "rustix 1.1.4",
  "serde",
  "serde_json",
  "tempfile",
@@ -1862,7 +1956,8 @@ dependencies = [
 name = "obisys"
 version = "0.1.0"
 dependencies = [
- "indicatif",
+ "fs4",
+ "indicatif 0.17.11",
  "libc",
  "sysinfo",
  "tracing",
@@ -2016,10 +2111,11 @@ dependencies = [
 
 [[package]]
 name = "pprof"
-version = "0.13.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb"
+checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187"
 dependencies = [
+ "aligned-vec",
  "backtrace",
  "cfg-if",
  "findshlibs",
@@ -2027,15 +2123,15 @@ dependencies = [
  "log",
  "nix 0.26.4",
  "once_cell",
- "parking_lot",
  "prost",
  "prost-build",
  "prost-derive",
  "sha2",
  "smallvec",
+ "spin",
  "symbolic-demangle",
  "tempfile",
- "thiserror 1.0.69",
+ "thiserror 2.0.18",
 ]
 
 [[package]]
@@ -2385,6 +2481,19 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
+dependencies = [
+ "bitflags 2.11.1",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.15",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.4"
@@ -2394,7 +2503,7 @@ dependencies = [
  "bitflags 2.11.1",
  "errno",
  "libc",
- "linux-raw-sys",
+ "linux-raw-sys 0.12.1",
  "windows-sys 0.61.2",
 ]
 
@@ -2520,6 +2629,19 @@ dependencies = [
  "zmij",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.34+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha2"
 version = "0.10.9"
@@ -2587,6 +2709,15 @@ dependencies = [
  "rb_tree",
 ]
 
+[[package]]
+name = "spin"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
+dependencies = [
+ "lock_api",
+]
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.1"
@@ -2735,7 +2866,7 @@ dependencies = [
  "fastrand",
  "getrandom 0.3.4",
  "once_cell",
- "rustix",
+ "rustix 1.1.4",
  "windows-sys 0.61.2",
 ]
 
@@ -2919,6 +3050,18 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
+[[package]]
+name = "unit-prefix"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3"
+
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
@@ -3420,7 +3563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
 dependencies = [
  "libc",
- "rustix",
+ "rustix 1.1.4",
 ]
 
 [[package]]

src/Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
 resolver = "3"
-members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy"]
+members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo"]
 [profile.release]
 debug = 1

src/obicompactvec/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+common_traits = "0.11"
 memmap2  = "0.9"
 ndarray  = "0.16"
 rayon    = "1"

src/obicompactvec/src/bitmatrix.rs

@@ -1,578 +0,0 @@
-use std::fs::{self, File};
-use std::io::{self, BufWriter, Read as _, Write as _};
-use std::path::{Path, PathBuf};
-
-use memmap2::Mmap;
-use ndarray::{Array1, Array2};
-use rayon::prelude::*;
-
-use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder};
-use crate::colgroup::{ColGroup, MatrixGroupOps};
-use crate::layer_meta::LayerMeta;
-use crate::meta::MatrixMeta;
-use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
-use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
-use crate::views::BitSliceView;
-
-fn col_path(dir: &Path, col: usize) -> PathBuf {
-    dir.join(format!("col_{col:06}.pbiv"))
-}
-
-// ── ColumnarBitMatrix ─────────────────────────────────────────────────────────
-
-/// Per-column file layout (original format).
-pub struct ColumnarBitMatrix {
-    cols: Vec<PersistentBitVec>,
-    n:    usize,
-}
-
-impl ColumnarBitMatrix {
-    pub(crate) fn open(dir: &Path) -> io::Result<Self> {
-        let meta = MatrixMeta::load(dir)?;
-        let cols = (0..meta.n_cols)
-            .map(|c| PersistentBitVec::open(&col_path(dir, c)))
-            .collect::<io::Result<Vec<_>>>()?;
-        Ok(Self { cols, n: meta.n })
-    }
-
-    pub(crate) fn n(&self)      -> usize { self.n }
-    pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
-    pub(crate) fn col(&self, c: usize) -> &PersistentBitVec { &self.cols[c] }
-
-    pub(crate) fn row(&self, slot: usize) -> Box<[bool]> {
-        self.cols.iter().map(|c| c.get(slot)).collect()
-    }
-
-    pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
-        for (c, col) in self.cols.iter().enumerate() {
-            buf[c] = col.get(slot) as u32;
-        }
-    }
-
-    pub(crate) fn count_ones(&self) -> Array1<u64> {
-        let counts: Vec<u64> = (0..self.n_cols())
-            .into_par_iter()
-            .map(|c| self.col(c).count_ones())
-            .collect();
-        Array1::from_vec(counts)
-    }
-
-    pub(crate) fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
-        pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_jaccard_dist(self.col(j)))
-    }
-
-    pub(crate) fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
-        pairwise_matrix(self.n_cols(), |i, j| self.col(i).hamming_dist(self.col(j)))
-    }
-
-    pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> bool) -> io::Result<()> {
-        let mut meta = MatrixMeta::load(dir)?;
-        let mut b = PersistentBitVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
-        for slot in 0..meta.n {
-            b.set(slot, value_of(slot));
-        }
-        b.close()?;
-        meta.n_cols += 1;
-        meta.save(dir)
-    }
-}
-
-// ── PackedBitMatrix ───────────────────────────────────────────────────────────
-
-const PBMX_MAGIC:  [u8; 4] = *b"PBMX";
-const PBMX_HEADER: usize   = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
-const PBIV_HEADER: usize   = 16; // magic(4) + pad(4) + n(8)
-
-/// Single-file packed layout: all columns concatenated behind a header.
-pub struct PackedBitMatrix {
-    mmap:         Mmap,
-    n_rows:       usize,
-    n_cols:       usize,
-    /// Absolute byte offset to the start of each column's bit data
-    /// (= file offset of the PBIV blob + PBIV_HEADER).
-    data_offsets: Vec<usize>,
-}
-
-impl PackedBitMatrix {
-    pub(crate) fn open(path: &Path) -> io::Result<Self> {
-        let mmap = unsafe { Mmap::map(&File::open(path)?)? };
-        if mmap.len() < PBMX_HEADER {
-            return Err(io::Error::new(io::ErrorKind::InvalidData, "PBMX file too short"));
-        }
-        if &mmap[0..4] != &PBMX_MAGIC {
-            return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PBMX magic"));
-        }
-        let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap())  as usize;
-        let n_cols = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize;
-
-        let mut data_offsets = Vec::with_capacity(n_cols);
-        for c in 0..n_cols {
-            let off_pos = PBMX_HEADER + c * 8;
-            let col_file_off = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
-            data_offsets.push(col_file_off + PBIV_HEADER);
-        }
-
-        Ok(Self { mmap, n_rows, n_cols, data_offsets })
-    }
-
-    #[inline]
-    pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
-        for (c, &data_off) in self.data_offsets.iter().enumerate() {
-            buf[c] = ((self.mmap[data_off + (slot >> 3)] >> (slot & 7)) & 1) as u32;
-        }
-    }
-
-    pub(crate) fn row(&self, slot: usize) -> Box<[bool]> {
-        (0..self.n_cols).map(|c| {
-            (self.mmap[self.data_offsets[c] + (slot >> 3)] >> (slot & 7)) & 1 != 0
-        }).collect()
-    }
-
-    fn col_bytes(&self, c: usize) -> &[u8] {
-        let start = self.data_offsets[c];
-        &self.mmap[start..start + self.n_rows.div_ceil(8)]
-    }
-
-    fn col_words(&self, c: usize) -> &[u64] {
-        let nw = self.n_rows.div_ceil(64);
-        // SAFETY: data_offsets[c] is always 8-byte aligned.
-        // PBMX header = 24 + n_cols×8 (multiple of 8); each PBIV blob =
-        // 16 + nwords×8 (multiple of 8); mmap base is page-aligned.
-        let ptr = self.mmap[self.data_offsets[c]..].as_ptr() as *const u64;
-        unsafe { std::slice::from_raw_parts(ptr, nw) }
-    }
-
-    pub(crate) fn col_slice(&self, c: usize) -> BitSliceView<'_> {
-        BitSliceView::new(self.col_words(c), self.n_rows)
-    }
-
-    pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentBitVecBuilder> {
-        PersistentBitVecBuilder::from_raw_bytes(self.col_bytes(c), self.n_rows, path)
-    }
-
-    pub(crate) fn count_ones(&self) -> Array1<u64> {
-        Array1::from_vec(
-            (0..self.n_cols).into_par_iter()
-                .map(|c| self.col_slice(c).count_ones())
-                .collect()
-        )
-    }
-
-    pub(crate) fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
-        pairwise2_matrix(self.n_cols, |i, j| {
-            self.col_slice(i).partial_jaccard_dist(self.col_slice(j))
-        })
-    }
-
-    pub(crate) fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
-        pairwise_matrix(self.n_cols, |i, j| {
-            self.col_slice(i).hamming_dist(self.col_slice(j))
-        })
-    }
-}
-
-/// Reads just the `n_cols` field from an existing packed matrix's header,
-/// without mapping the file. Used by `pack_bit_matrix` to tell a genuinely
-/// complete pack from a stale one that predates a later column-widening.
-fn packed_bit_matrix_n_cols(path: &Path) -> io::Result<usize> {
-    let mut f = File::open(path)?;
-    let mut header = [0u8; PBMX_HEADER];
-    f.read_exact(&mut header)?;
-    Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
-}
-
-/// Build `presence/matrix.pbmx` from existing `col_*.pbiv` files.
-pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
-    let packed_path = dir.join("matrix.pbmx");
-
-    let meta = match MatrixMeta::load(dir) {
-        Ok(meta) => meta,
-        Err(e) => {
-            // No columnar data pending: either this layer was already
-            // packed and cleaned up (matrix.pbmx complete, nothing left to
-            // do), or genuinely nothing was ever written here.
-            return if packed_path.exists() { Ok(()) } else { Err(e) };
-        }
-    };
-
-    // A `matrix.pbmx` can already exist here even though columnar data is
-    // still pending — e.g. copied verbatim from a merge's base source
-    // before this layer was widened with more genome columns (see
-    // `obikpartitionner::merge_partition`). Only skip (re-)packing if the
-    // existing file already reflects the current column count; otherwise
-    // the columnar files are newer and must be (re-)packed, overwriting the
-    // stale one — never silently discarded as "leftover cleanup".
-    if packed_bit_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
-        for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
-        let _ = fs::remove_file(dir.join("meta.json"));
-        return Ok(());
-    }
-
-    let n_cols = meta.n_cols;
-
-    // Compute offsets from file sizes — no column data loaded into RAM.
-    let col_sizes: Vec<u64> = (0..n_cols)
-        .map(|c| fs::metadata(col_path(dir, c)).map(|m| m.len()))
-        .collect::<io::Result<_>>()?;
-
-    let header_size = (PBMX_HEADER + n_cols * 8) as u64;
-    let mut col_offset = header_size;
-    let mut offsets = Vec::with_capacity(n_cols);
-    for &size in &col_sizes {
-        offsets.push(col_offset);
-        col_offset += size;
-    }
-
-    // Write to a temp file; rename atomically so a killed process never leaves
-    // a truncated matrix.pbmx that would be mistaken for a complete file.
-    let tmp_path = dir.join("matrix.pbmx.tmp");
-    let mut out = BufWriter::new(File::create(&tmp_path)?);
-    out.write_all(&PBMX_MAGIC)?;
-    out.write_all(&[0u8; 4])?;
-    out.write_all(&(meta.n as u64).to_le_bytes())?;
-    out.write_all(&(n_cols as u64).to_le_bytes())?;
-    for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
-    for c in 0..n_cols {
-        io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
-    }
-    out.flush()?;
-    drop(out);
-    fs::rename(&tmp_path, &packed_path)?;
-
-    for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
-    fs::remove_file(dir.join("meta.json"))?;
-    Ok(())
-}
-
-// ── PersistentBitMatrix — public enum ────────────────────────────────────────
-
-/// Bit matrix that transparently handles columnar, packed, and implicit formats.
-///
-/// - `Columnar`: per-column `.pbiv` files (original format, used during build)
-/// - `Packed`:   single `matrix.pbmx` file (optimised for query — one `mmap`)
-/// - `Implicit`: no file — all values are 1 (mono-genome presence/absence)
-pub enum PersistentBitMatrix {
-    Columnar(ColumnarBitMatrix),
-    Packed(PackedBitMatrix),
-    Implicit { n_rows: usize, n_cols: usize },
-}
-
-impl PersistentBitMatrix {
-    /// Open from `layer_dir`, auto-detecting the format.
-    ///
-    /// Checks (in order):
-    /// 1. `layer_dir/presence/matrix.pbmx`  → Packed
-    /// 2. `layer_dir/presence/meta.json`    → Columnar
-    /// 3. `layer_dir/layer_meta.json`        → Implicit (new index)
-    /// 4. `layer_dir/unitigs.bin`            → Implicit with warning (old index)
-    pub fn open(layer_dir: &Path) -> io::Result<Self> {
-        let presence_dir = layer_dir.join("presence");
-
-        if presence_dir.join("matrix.pbmx").exists() {
-            return Ok(Self::Packed(PackedBitMatrix::open(&presence_dir.join("matrix.pbmx"))?));
-        }
-
-        if MatrixMeta::load(&presence_dir).is_ok() {
-            return Ok(Self::Columnar(ColumnarBitMatrix::open(&presence_dir)?));
-        }
-
-        // No presence matrix → Implicit; requires layer_meta.json
-        let meta = LayerMeta::load(layer_dir).map_err(|_| io::Error::new(
-            io::ErrorKind::NotFound,
-            format!(
-                "no presence matrix and no layer_meta.json in {} — run 'obikmer upgrade'",
-                layer_dir.display()
-            ),
-        ))?;
-        Ok(Self::Implicit { n_rows: meta.n, n_cols: 1 })
-    }
-
-    pub fn n(&self) -> usize {
-        match self {
-            Self::Columnar(m)            => m.n(),
-            Self::Packed(m)              => m.n_rows,
-            Self::Implicit { n_rows, .. } => *n_rows,
-        }
-    }
-
-    pub fn n_cols(&self) -> usize {
-        match self {
-            Self::Columnar(m)            => m.n_cols(),
-            Self::Packed(m)              => m.n_cols,
-            Self::Implicit { n_cols, .. } => *n_cols,
-        }
-    }
-
-    pub fn col(&self, c: usize) -> &PersistentBitVec {
-        match self {
-            Self::Columnar(m) => m.col(c),
-            _ => panic!("col() only available on Columnar PersistentBitMatrix"),
-        }
-    }
-
-    pub fn col_view(&self, c: usize) -> BitSliceView<'_> {
-        match self {
-            Self::Columnar(m) => m.col(c).view(),
-            Self::Packed(m)   => m.col_slice(c),
-            Self::Implicit { .. } => panic!("col_view() not available on Implicit PersistentBitMatrix"),
-        }
-    }
-
-    /// Column-major point lookup: value at column `c`, slot `slot`, as 0/1.
-    ///
-    /// Unlike [`col_view`](Self::col_view), this never panics on `Implicit`
-    /// (every column reads as present, per the mono-genome fast path) — safe
-    /// to call for any `c < self.n_cols()`.
-    pub fn get(&self, c: usize, slot: usize) -> u32 {
-        match self {
-            Self::Columnar(m) => m.col(c).get(slot) as u32,
-            Self::Packed(m)   => m.col_slice(c).get(slot) as u32,
-            Self::Implicit { .. } => 1,
-        }
-    }
-
-    pub fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentBitVecBuilder> {
-        match self {
-            Self::Columnar(m) => PersistentBitVecBuilder::build_from(m.col(c), path),
-            Self::Packed(m)   => m.col_persist(c, path),
-            Self::Implicit { n_rows, .. } => {
-                PersistentBitVecBuilder::new_ones(*n_rows, path)
-            }
-        }
-    }
-
-    pub fn row(&self, slot: usize) -> Box<[bool]> {
-        match self {
-            Self::Columnar(m)             => m.row(slot),
-            Self::Packed(m)               => m.row(slot),
-            Self::Implicit { n_cols, .. } => vec![true; *n_cols].into_boxed_slice(),
-        }
-    }
-
-    /// Fill `buf[i]` with `col_i[slot]` as 0/1 u32, without allocating.
-    pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
-        match self {
-            Self::Columnar(m)             => m.fill_row(slot, buf),
-            Self::Packed(m)               => m.fill_row(slot, buf),
-            Self::Implicit { n_cols, .. } => buf[..*n_cols].fill(1),
-        }
-    }
-
-    pub fn count_ones(&self) -> Array1<u64> {
-        match self {
-            Self::Columnar(m)              => m.count_ones(),
-            Self::Packed(m)                => m.count_ones(),
-            Self::Implicit { n_rows, n_cols } => Array1::from_elem(*n_cols, *n_rows as u64),
-        }
-    }
-
-    pub fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
-        match self {
-            Self::Columnar(m)                 => m.partial_jaccard_dist_matrix(),
-            Self::Packed(m)                   => m.partial_jaccard_dist_matrix(),
-            Self::Implicit { n_rows, n_cols } => {
-                let v = *n_rows as u64;
-                let n = *n_cols;
-                let mut inter = Array2::zeros((n, n));
-                let mut union = Array2::zeros((n, n));
-                for i in 0..n { for j in 0..n {
-                    inter[[i, j]] = v; union[[i, j]] = v;
-                }}
-                (inter, union)
-            }
-        }
-    }
-
-    pub fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
-        match self {
-            Self::Columnar(m)              => m.partial_hamming_dist_matrix(),
-            Self::Packed(m)                => m.partial_hamming_dist_matrix(),
-            Self::Implicit { n_cols, .. }  => Array2::zeros((*n_cols, *n_cols)),
-        }
-    }
-
-    /// Append a new column to an on-disk Columnar matrix.
-    pub fn append_column(dir: &Path, value_of: impl Fn(usize) -> bool) -> io::Result<()> {
-        ColumnarBitMatrix::append_column(dir, value_of)
-    }
-}
-
-// ── Trait impls ───────────────────────────────────────────────────────────────
-
-use crate::traits::{BitPartials, ColumnWeights};
-
-impl ColumnWeights for PersistentBitMatrix {
-    fn col_weights(&self) -> Array1<u64> { self.count_ones() }
-}
-
-impl BitPartials for PersistentBitMatrix {
-    fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) {
-        self.partial_jaccard_dist_matrix()
-    }
-    fn partial_hamming(&self) -> Array2<u64> {
-        self.partial_hamming_dist_matrix()
-    }
-}
-
-// ── Builder (unchanged — always builds Columnar) ──────────────────────────────
-
-pub struct PersistentBitMatrixBuilder {
-    dir:    PathBuf,
-    n:      usize,
-    n_cols: usize,
-}
-
-impl PersistentBitMatrixBuilder {
-    pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
-        fs::create_dir_all(dir)?;
-        Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
-    }
-
-    pub fn n(&self)      -> usize { self.n }
-    pub fn n_cols(&self) -> usize { self.n_cols }
-
-    pub fn add_col(&mut self) -> io::Result<PersistentBitVecBuilder> {
-        let path = col_path(&self.dir, self.n_cols);
-        self.n_cols += 1;
-        PersistentBitVecBuilder::new(self.n, &path)
-    }
-
-    pub fn add_col_ones(&mut self) -> io::Result<PersistentBitVecBuilder> {
-        let path = col_path(&self.dir, self.n_cols);
-        self.n_cols += 1;
-        PersistentBitVecBuilder::new_ones(self.n, &path)
-    }
-
-    pub fn add_col_from(&mut self, src: &TempBitVec) -> io::Result<()> {
-        src.make_persistent(&col_path(&self.dir, self.n_cols))?;
-        self.n_cols += 1;
-        Ok(())
-    }
-
-    pub fn add_col_from_int(&mut self, src: &TempCompactIntVec) -> io::Result<()> {
-        let path = col_path(&self.dir, self.n_cols);
-        self.n_cols += 1;
-        let mut b = PersistentBitVecBuilder::new(self.n, &path)?;
-        b.or_where(src.view(), |v| v > 0);
-        b.close()
-    }
-
-    pub fn close(self) -> io::Result<()> {
-        MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
-    }
-}
-
-// ── MatrixGroupOps ────────────────────────────────────────────────────────────
-
-impl MatrixGroupOps for PersistentBitMatrix {
-    fn partial_group_presence_count(&self, g: &ColGroup, _threshold: u32) -> io::Result<TempCompactIntVec> {
-        // Bit matrices store 0/1 — threshold is structurally always 1.
-        let n = self.n();
-        if g.indices.len() < 255 {
-            let mut builder = TempCompactIntVecBuilder::new(n)?;
-            for &c in &g.indices {
-                builder.inc_present_fast(self.col_view(c));
-            }
-            builder.freeze()
-        } else {
-            let mut result = TempCompactIntVecBuilder::new(n)?;
-            for chunk in g.indices.chunks(254) {
-                let mut chunk_b = TempCompactIntVecBuilder::new(n)?;
-                for &c in chunk {
-                    chunk_b.inc_present_fast(self.col_view(c));
-                }
-                let frozen = chunk_b.freeze()?;
-                result.add(frozen.view());
-            }
-            result.freeze()
-        }
-    }
-
-    fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
-        // For bit matrices, sum = count of 1-bits — identical to presence_count.
-        self.partial_group_presence_count(g, 1)
-    }
-
-    fn partial_group_any(&self, g: &ColGroup, _threshold: u32) -> io::Result<TempBitVec> {
-        let n = self.n();
-        let mut result = TempBitVecBuilder::new(n)?;
-        for &c in &g.indices {
-            result.or(self.col_view(c));
-        }
-        result.freeze()
-    }
-
-    fn partial_group_min(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
-        // min of 0/1 values = AND: 1 only if ALL columns are 1
-        let n = self.n();
-        let mut result = TempCompactIntVecBuilder::new(n)?;
-        if let Some((&first, rest)) = g.indices.split_first() {
-            result.inc_present_fast(self.col_view(first));
-            for &c in rest { result.mask_with(self.col_view(c)); }
-        }
-        result.freeze()
-    }
-
-    fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
-        // max of 0/1 values = OR: 1 if any column is 1
-        let any = self.partial_group_any(g, 1)?;
-        let n = any.len();
-        let mut result = TempCompactIntVecBuilder::new(n)?;
-        result.inc_present(any.view());
-        result.freeze()
-    }
-}
-
-// ── Shared matrix helpers (also used by intmatrix.rs) ─────────────────────────
-
-fn upper_pairs(n: usize) -> Vec<(usize, usize)> {
-    (0..n).flat_map(|i| (i + 1..n).map(move |j| (i, j))).collect()
-}
-
-fn fill_symmetric<T>(n: usize, vals: impl Iterator<Item = (usize, usize, T, T)>) -> Array2<T>
-where T: Clone + Default {
-    let mut m = Array2::from_elem((n, n), T::default());
-    for (i, j, vij, vji) in vals { m[[i, j]] = vij; m[[j, i]] = vji; }
-    m
-}
-
-/// Compute a symmetric `n×n` matrix in parallel by evaluating `f(i,j)` for
-/// all upper-triangle pairs, plus `f(i,i)` for the diagonal. `T: Copy` avoids
-/// the `.clone()` needed for the lower-triangle mirror.
-///
-/// The diagonal is *not* generally `T::default()`: for a self-comparison,
-/// `f(i,i)` is often the column's own weight (e.g. intersection-with-self —
-/// see `pairwise2_matrix`), not zero. Distance finalisations that need a
-/// zero diagonal (self-distance) already overwrite it explicitly.
-pub(crate) fn pairwise_matrix<T>(n: usize, f: impl Fn(usize, usize) -> T + Sync) -> Array2<T>
-where T: Copy + Default + Send {
-    let results: Vec<(usize, usize, T)> = upper_pairs(n)
-        .into_par_iter().map(|(i, j)| (i, j, f(i, j))).collect();
-    let mut m = fill_symmetric(n, results.into_iter().map(|(i, j, v)| (i, j, v, v)));
-    for i in 0..n { m[[i, i]] = f(i, i); }
-    m
-}
-
-/// Same as `pairwise_matrix` but `f` returns two values that fill two
-/// symmetric matrices simultaneously (e.g. intersection + union for Jaccard).
-/// The diagonal is `f(i,i)` (e.g. a genome's kmer count intersected with
-/// itself), not `T::default()` — see `pairwise_matrix` for why that matters.
-pub(crate) fn pairwise2_matrix<T>(n: usize, f: impl Fn(usize, usize) -> (T, T) + Sync) -> (Array2<T>, Array2<T>)
-where T: Copy + Default + Send {
-    let results: Vec<(usize, usize, T, T)> = upper_pairs(n)
-        .into_par_iter()
-        .map(|(i, j)| { let (a, b) = f(i, j); (i, j, a, b) })
-        .collect();
-    let mut m0 = Array2::from_elem((n, n), T::default());
-    let mut m1 = Array2::from_elem((n, n), T::default());
-    for (i, j, a, b) in results {
-        m0[[i, j]] = a; m0[[j, i]] = a;
-        m1[[i, j]] = b; m1[[j, i]] = b;
-    }
-    for i in 0..n {
-        let (a, b) = f(i, i);
-        m0[[i, i]] = a;
-        m1[[i, i]] = b;
-    }
-    (m0, m1)
-}

src/obicompactvec/src/bitmatrix/builder.rs

@@ -0,0 +1,58 @@
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+
+use crate::bitvec::PersistentBitVecBuilder;
+use crate::meta::MatrixMeta;
+use crate::tempbitvec::TempBitVec;
+use crate::tempintvec::TempCompactIntVec;
+
+use super::col_path;
+
+// ── Builder (unchanged — always builds Columnar) ──────────────────────────────
+
+pub struct PersistentBitMatrixBuilder {
+    dir:    PathBuf,
+    n:      usize,
+    n_cols: usize,
+}
+
+impl PersistentBitMatrixBuilder {
+    pub fn new(n: usize, dir: &Path) -> io::Result<Self> {
+        fs::create_dir_all(dir)?;
+        Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
+    }
+
+    pub fn n(&self)      -> usize { self.n }
+    pub fn n_cols(&self) -> usize { self.n_cols }
+
+    pub fn add_col(&mut self) -> io::Result<PersistentBitVecBuilder> {
+        let path = col_path(&self.dir, self.n_cols);
+        self.n_cols += 1;
+        PersistentBitVecBuilder::new(self.n, &path)
+    }
+
+    pub fn add_col_ones(&mut self) -> io::Result<PersistentBitVecBuilder> {
+        let path = col_path(&self.dir, self.n_cols);
+        self.n_cols += 1;
+        PersistentBitVecBuilder::new_ones(self.n, &path)
+    }
+
+    pub fn add_col_from(&mut self, src: &TempBitVec) -> io::Result<()> {
+        src.make_persistent(&col_path(&self.dir, self.n_cols))?;
+        self.n_cols += 1;
+        Ok(())
+    }
+
+    pub fn add_col_from_int(&mut self, src: &TempCompactIntVec) -> io::Result<()> {
+        let path = col_path(&self.dir, self.n_cols);
+        self.n_cols += 1;
+        let mut b = PersistentBitVecBuilder::new(self.n, &path)?;
+        b.or_where(src.view(), |v| v > 0);
+        b.close()
+    }
+
+    pub fn close(self) -> io::Result<()> {
+        MatrixMeta { n: self.n, n_cols: self.n_cols }.save(&self.dir)
+    }
+}

src/obicompactvec/src/bitmatrix/columnar.rs

@@ -0,0 +1,75 @@
+use std::io;
+use std::path::Path;
+
+use ndarray::{Array1, Array2};
+use rayon::prelude::*;
+
+use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder};
+use crate::meta::MatrixMeta;
+
+use super::col_path;
+use super::pairwise::{pairwise_matrix, pairwise2_matrix};
+
+// ── ColumnarBitMatrix ─────────────────────────────────────────────────────────
+
+/// Per-column file layout (original format).
+pub struct ColumnarBitMatrix {
+    cols: Vec<PersistentBitVec>,
+    n:    usize,
+}
+
+impl ColumnarBitMatrix {
+    pub(crate) fn open(dir: &Path) -> io::Result<Self> {
+        let meta = MatrixMeta::load(dir)?;
+        let cols = (0..meta.n_cols)
+            .map(|c| PersistentBitVec::open(&col_path(dir, c)))
+            .collect::<io::Result<Vec<_>>>()?;
+        Ok(Self { cols, n: meta.n })
+    }
+
+    #[inline]
+    pub(crate) fn n(&self)      -> usize { self.n }
+    #[inline]
+    pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
+    #[inline]
+    pub(crate) fn col(&self, c: usize) -> &PersistentBitVec { &self.cols[c] }
+
+    #[inline]
+    pub(crate) fn row(&self, slot: usize) -> Box<[bool]> {
+        self.cols.iter().map(|c| c.get(slot)).collect()
+    }
+
+    #[inline]
+    pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
+        for (c, col) in self.cols.iter().enumerate() {
+            buf[c] = col.get(slot) as u32;
+        }
+    }
+
+    pub(crate) fn count_ones(&self) -> Array1<u64> {
+        let counts: Vec<u64> = (0..self.n_cols())
+            .into_par_iter()
+            .map(|c| self.col(c).count_ones())
+            .collect();
+        Array1::from_vec(counts)
+    }
+
+    pub(crate) fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
+        pairwise2_matrix(self.n_cols(), |i, j| self.col(i).partial_jaccard_dist(self.col(j)))
+    }
+
+    pub(crate) fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
+        pairwise_matrix(self.n_cols(), |i, j| self.col(i).hamming_dist(self.col(j)))
+    }
+
+    pub(crate) fn append_column(dir: &Path, value_of: impl Fn(usize) -> bool) -> io::Result<()> {
+        let mut meta = MatrixMeta::load(dir)?;
+        let mut b = PersistentBitVecBuilder::new(meta.n, &col_path(dir, meta.n_cols))?;
+        for slot in 0..meta.n {
+            b.set(slot, value_of(slot));
+        }
+        b.close()?;
+        meta.n_cols += 1;
+        meta.save(dir)
+    }
+}

src/obicompactvec/src/bitmatrix/group_ops.rs

@@ -0,0 +1,68 @@
+use std::io;
+
+use crate::colgroup::{ColGroup, MatrixGroupOps};
+use crate::tempbitvec::{TempBitVec, TempBitVecBuilder};
+use crate::tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
+
+use super::persistent::PersistentBitMatrix;
+
+// ── MatrixGroupOps ────────────────────────────────────────────────────────────
+
+impl MatrixGroupOps for PersistentBitMatrix {
+    fn partial_group_presence_count(&self, g: &ColGroup, _threshold: u32) -> io::Result<TempCompactIntVec> {
+        // Bit matrices store 0/1 — threshold is structurally always 1.
+        let n = self.n();
+        if g.indices.len() < 255 {
+            let mut builder = TempCompactIntVecBuilder::new(n)?;
+            for &c in &g.indices {
+                builder.inc_present_fast(self.col_view(c));
+            }
+            builder.freeze()
+        } else {
+            let mut result = TempCompactIntVecBuilder::new(n)?;
+            for chunk in g.indices.chunks(254) {
+                let mut chunk_b = TempCompactIntVecBuilder::new(n)?;
+                for &c in chunk {
+                    chunk_b.inc_present_fast(self.col_view(c));
+                }
+                let frozen = chunk_b.freeze()?;
+                result.add(frozen.view());
+            }
+            result.freeze()
+        }
+    }
+
+    fn partial_group_sum(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        // For bit matrices, sum = count of 1-bits — identical to presence_count.
+        self.partial_group_presence_count(g, 1)
+    }
+
+    fn partial_group_any(&self, g: &ColGroup, _threshold: u32) -> io::Result<TempBitVec> {
+        let n = self.n();
+        let mut result = TempBitVecBuilder::new(n)?;
+        for &c in &g.indices {
+            result.or(self.col_view(c));
+        }
+        result.freeze()
+    }
+
+    fn partial_group_min(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        // min of 0/1 values = AND: 1 only if ALL columns are 1
+        let n = self.n();
+        let mut result = TempCompactIntVecBuilder::new(n)?;
+        if let Some((&first, rest)) = g.indices.split_first() {
+            result.inc_present_fast(self.col_view(first));
+            for &c in rest { result.mask_with(self.col_view(c)); }
+        }
+        result.freeze()
+    }
+
+    fn partial_group_max(&self, g: &ColGroup) -> io::Result<TempCompactIntVec> {
+        // max of 0/1 values = OR: 1 if any column is 1
+        let any = self.partial_group_any(g, 1)?;
+        let n = any.len();
+        let mut result = TempCompactIntVecBuilder::new(n)?;
+        result.inc_present(any.view());
+        result.freeze()
+    }
+}

src/obicompactvec/src/bitmatrix/mod.rs

@@ -0,0 +1,31 @@
+//! Bit matrices (presence/absence), in three on-disk formats transparently
+//! handled by [`PersistentBitMatrix`]: per-column `Columnar`, single-file
+//! mmap'd `Packed`, and implicit (mono-genome, no file at all).
+//!
+//! Submodules: [`columnar`] (build-time per-column format), [`packed`]
+//! (query-optimised single-mmap format + [`pack_bit_matrix`]),
+//! [`persistent`] (the format-dispatching [`PersistentBitMatrix`] enum),
+//! [`builder`] ([`PersistentBitMatrixBuilder`], always builds Columnar),
+//! [`group_ops`] (`MatrixGroupOps` impl), [`pairwise`] (shared symmetric
+//! pairwise-matrix helpers, also used by `intmatrix.rs`).
+
+use std::path::{Path, PathBuf};
+
+mod builder;
+mod columnar;
+mod group_ops;
+mod packed;
+mod pairwise;
+mod persistent;
+mod sparse;
+
+pub use builder::PersistentBitMatrixBuilder;
+pub use packed::pack_bit_matrix;
+pub use persistent::PersistentBitMatrix;
+pub use sparse::{PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_sparse_bit_matrix};
+
+pub(crate) use pairwise::{pairwise_matrix, pairwise2_matrix};
+
+fn col_path(dir: &Path, col: usize) -> PathBuf {
+    dir.join(format!("col_{col:06}.pbiv"))
+}

src/obicompactvec/src/bitmatrix/packed.rs

@@ -0,0 +1,185 @@
+use std::fs::{self, File};
+use std::io::{self, BufWriter, Read as _, Write as _};
+use std::path::Path;
+
+use memmap2::Mmap;
+use ndarray::{Array1, Array2};
+use rayon::prelude::*;
+
+use crate::bitvec::PersistentBitVecBuilder;
+use crate::meta::MatrixMeta;
+use crate::views::BitSliceView;
+
+use super::col_path;
+use super::pairwise::{pairwise_matrix, pairwise2_matrix};
+
+// ── PackedBitMatrix ───────────────────────────────────────────────────────────
+
+const PBMX_MAGIC:  [u8; 4] = *b"PBMX";
+const PBMX_HEADER: usize   = 24; // magic(4) + pad(4) + n_rows(8) + n_cols(8)
+const PBIV_HEADER: usize   = 16; // magic(4) + pad(4) + n(8)
+
+/// Single-file packed layout: all columns concatenated behind a header.
+pub struct PackedBitMatrix {
+    mmap:         Mmap,
+    pub(super) n_rows: usize,
+    pub(super) n_cols: usize,
+    /// Absolute byte offset to the start of each column's bit data
+    /// (= file offset of the PBIV blob + PBIV_HEADER).
+    data_offsets: Vec<usize>,
+}
+
+impl PackedBitMatrix {
+    pub(crate) fn open(path: &Path) -> io::Result<Self> {
+        let mmap = unsafe { Mmap::map(&File::open(path)?)? };
+        if mmap.len() < PBMX_HEADER {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "PBMX file too short"));
+        }
+        if &mmap[0..4] != &PBMX_MAGIC {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PBMX magic"));
+        }
+        let n_rows = u64::from_le_bytes(mmap[8..16].try_into().unwrap())  as usize;
+        let n_cols = u64::from_le_bytes(mmap[16..24].try_into().unwrap()) as usize;
+
+        let mut data_offsets = Vec::with_capacity(n_cols);
+        for c in 0..n_cols {
+            let off_pos = PBMX_HEADER + c * 8;
+            let col_file_off = u64::from_le_bytes(mmap[off_pos..off_pos+8].try_into().unwrap()) as usize;
+            data_offsets.push(col_file_off + PBIV_HEADER);
+        }
+
+        Ok(Self { mmap, n_rows, n_cols, data_offsets })
+    }
+
+    #[inline]
+    pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
+        for (c, &data_off) in self.data_offsets.iter().enumerate() {
+            buf[c] = ((self.mmap[data_off + (slot >> 3)] >> (slot & 7)) & 1) as u32;
+        }
+    }
+
+    #[inline]
+    pub(crate) fn row(&self, slot: usize) -> Box<[bool]> {
+        (0..self.n_cols).map(|c| {
+            (self.mmap[self.data_offsets[c] + (slot >> 3)] >> (slot & 7)) & 1 != 0
+        }).collect()
+    }
+
+    #[inline]
+    fn col_bytes(&self, c: usize) -> &[u8] {
+        let start = self.data_offsets[c];
+        &self.mmap[start..start + self.n_rows.div_ceil(8)]
+    }
+
+    #[inline]
+    fn col_words(&self, c: usize) -> &[u64] {
+        let nw = self.n_rows.div_ceil(64);
+        // SAFETY: data_offsets[c] is always 8-byte aligned.
+        // PBMX header = 24 + n_cols×8 (multiple of 8); each PBIV blob =
+        // 16 + nwords×8 (multiple of 8); mmap base is page-aligned.
+        let ptr = self.mmap[self.data_offsets[c]..].as_ptr() as *const u64;
+        unsafe { std::slice::from_raw_parts(ptr, nw) }
+    }
+
+    #[inline]
+    pub(crate) fn col_slice(&self, c: usize) -> BitSliceView<'_> {
+        BitSliceView::new(self.col_words(c), self.n_rows)
+    }
+
+    pub(crate) fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentBitVecBuilder> {
+        PersistentBitVecBuilder::from_raw_bytes(self.col_bytes(c), self.n_rows, path)
+    }
+
+    pub(crate) fn count_ones(&self) -> Array1<u64> {
+        Array1::from_vec(
+            (0..self.n_cols).into_par_iter()
+                .map(|c| self.col_slice(c).count_ones())
+                .collect()
+        )
+    }
+
+    pub(crate) fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
+        pairwise2_matrix(self.n_cols, |i, j| {
+            self.col_slice(i).partial_jaccard_dist(self.col_slice(j))
+        })
+    }
+
+    pub(crate) fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
+        pairwise_matrix(self.n_cols, |i, j| {
+            self.col_slice(i).hamming_dist(self.col_slice(j))
+        })
+    }
+}
+
+/// Reads just the `n_cols` field from an existing packed matrix's header,
+/// without mapping the file. Used by `pack_bit_matrix` to tell a genuinely
+/// complete pack from a stale one that predates a later column-widening.
+fn packed_bit_matrix_n_cols(path: &Path) -> io::Result<usize> {
+    let mut f = File::open(path)?;
+    let mut header = [0u8; PBMX_HEADER];
+    f.read_exact(&mut header)?;
+    Ok(u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize)
+}
+
+/// Build `presence/matrix.pbmx` from existing `col_*.pbiv` files.
+pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> {
+    let packed_path = dir.join("matrix.pbmx");
+
+    let meta = match MatrixMeta::load(dir) {
+        Ok(meta) => meta,
+        Err(e) => {
+            // No columnar data pending: either this layer was already
+            // packed and cleaned up (matrix.pbmx complete, nothing left to
+            // do), or genuinely nothing was ever written here.
+            return if packed_path.exists() { Ok(()) } else { Err(e) };
+        }
+    };
+
+    // A `matrix.pbmx` can already exist here even though columnar data is
+    // still pending — e.g. copied verbatim from a merge's base source
+    // before this layer was widened with more genome columns (see
+    // `obikpartitionner::merge_partition`). Only skip (re-)packing if the
+    // existing file already reflects the current column count; otherwise
+    // the columnar files are newer and must be (re-)packed, overwriting the
+    // stale one — never silently discarded as "leftover cleanup".
+    if packed_bit_matrix_n_cols(&packed_path).ok() == Some(meta.n_cols) {
+        for c in 0..meta.n_cols { let _ = fs::remove_file(col_path(dir, c)); }
+        let _ = fs::remove_file(dir.join("meta.json"));
+        return Ok(());
+    }
+
+    let n_cols = meta.n_cols;
+
+    // Compute offsets from file sizes — no column data loaded into RAM.
+    let col_sizes: Vec<u64> = (0..n_cols)
+        .map(|c| fs::metadata(col_path(dir, c)).map(|m| m.len()))
+        .collect::<io::Result<_>>()?;
+
+    let header_size = (PBMX_HEADER + n_cols * 8) as u64;
+    let mut col_offset = header_size;
+    let mut offsets = Vec::with_capacity(n_cols);
+    for &size in &col_sizes {
+        offsets.push(col_offset);
+        col_offset += size;
+    }
+
+    // Write to a temp file; rename atomically so a killed process never leaves
+    // a truncated matrix.pbmx that would be mistaken for a complete file.
+    let tmp_path = dir.join("matrix.pbmx.tmp");
+    let mut out = BufWriter::new(File::create(&tmp_path)?);
+    out.write_all(&PBMX_MAGIC)?;
+    out.write_all(&[0u8; 4])?;
+    out.write_all(&(meta.n as u64).to_le_bytes())?;
+    out.write_all(&(n_cols as u64).to_le_bytes())?;
+    for &off in &offsets { out.write_all(&off.to_le_bytes())?; }
+    for c in 0..n_cols {
+        io::copy(&mut File::open(col_path(dir, c))?, &mut out)?;
+    }
+    out.flush()?;
+    drop(out);
+    fs::rename(&tmp_path, &packed_path)?;
+
+    for c in 0..n_cols { fs::remove_file(col_path(dir, c))?; }
+    fs::remove_file(dir.join("meta.json"))?;
+    Ok(())
+}

src/obicompactvec/src/bitmatrix/pairwise.rs

@@ -0,0 +1,56 @@
+use ndarray::Array2;
+use rayon::prelude::*;
+
+// ── Shared matrix helpers (also used by intmatrix.rs) ─────────────────────────
+
+fn upper_pairs(n: usize) -> Vec<(usize, usize)> {
+    (0..n).flat_map(|i| (i + 1..n).map(move |j| (i, j))).collect()
+}
+
+fn fill_symmetric<T>(n: usize, vals: impl Iterator<Item = (usize, usize, T, T)>) -> Array2<T>
+where T: Clone + Default {
+    let mut m = Array2::from_elem((n, n), T::default());
+    for (i, j, vij, vji) in vals { m[[i, j]] = vij; m[[j, i]] = vji; }
+    m
+}
+
+/// Compute a symmetric `n×n` matrix in parallel by evaluating `f(i,j)` for
+/// all upper-triangle pairs, plus `f(i,i)` for the diagonal. `T: Copy` avoids
+/// the `.clone()` needed for the lower-triangle mirror.
+///
+/// The diagonal is *not* generally `T::default()`: for a self-comparison,
+/// `f(i,i)` is often the column's own weight (e.g. intersection-with-self —
+/// see `pairwise2_matrix`), not zero. Distance finalisations that need a
+/// zero diagonal (self-distance) already overwrite it explicitly.
+pub(crate) fn pairwise_matrix<T>(n: usize, f: impl Fn(usize, usize) -> T + Sync) -> Array2<T>
+where T: Copy + Default + Send {
+    let results: Vec<(usize, usize, T)> = upper_pairs(n)
+        .into_par_iter().map(|(i, j)| (i, j, f(i, j))).collect();
+    let mut m = fill_symmetric(n, results.into_iter().map(|(i, j, v)| (i, j, v, v)));
+    for i in 0..n { m[[i, i]] = f(i, i); }
+    m
+}
+
+/// Same as `pairwise_matrix` but `f` returns two values that fill two
+/// symmetric matrices simultaneously (e.g. intersection + union for Jaccard).
+/// The diagonal is `f(i,i)` (e.g. a genome's kmer count intersected with
+/// itself), not `T::default()` — see `pairwise_matrix` for why that matters.
+pub(crate) fn pairwise2_matrix<T>(n: usize, f: impl Fn(usize, usize) -> (T, T) + Sync) -> (Array2<T>, Array2<T>)
+where T: Copy + Default + Send {
+    let results: Vec<(usize, usize, T, T)> = upper_pairs(n)
+        .into_par_iter()
+        .map(|(i, j)| { let (a, b) = f(i, j); (i, j, a, b) })
+        .collect();
+    let mut m0 = Array2::from_elem((n, n), T::default());
+    let mut m1 = Array2::from_elem((n, n), T::default());
+    for (i, j, a, b) in results {
+        m0[[i, j]] = a; m0[[j, i]] = a;
+        m1[[i, j]] = b; m1[[j, i]] = b;
+    }
+    for i in 0..n {
+        let (a, b) = f(i, i);
+        m0[[i, i]] = a;
+        m1[[i, i]] = b;
+    }
+    (m0, m1)
+}

src/obicompactvec/src/bitmatrix/persistent.rs

@@ -0,0 +1,258 @@
+use std::io;
+use std::path::Path;
+
+use ndarray::{Array1, Array2};
+
+use crate::bitvec::{PersistentBitVec, PersistentBitVecBuilder};
+use crate::layer_meta::LayerMeta;
+use crate::meta::MatrixMeta;
+use crate::traits::{BitPartials, ColumnWeights};
+use crate::views::BitSliceView;
+
+use super::columnar::ColumnarBitMatrix;
+use super::packed::PackedBitMatrix;
+
+// ── PersistentBitMatrix — public enum ────────────────────────────────────────
+
+/// Bit matrix that transparently handles columnar, packed, and implicit formats.
+///
+/// - `Columnar`: per-column `.pbiv` files (original format, used during build)
+/// - `Packed`:   single `matrix.pbmx` file (optimised for query — one `mmap`)
+/// - `Implicit`: no file — all values are 1 (mono-genome presence/absence)
+pub enum PersistentBitMatrix {
+    Columnar(ColumnarBitMatrix),
+    Packed(PackedBitMatrix),
+    Implicit { n_rows: usize, n_cols: usize },
+}
+
+impl PersistentBitMatrix {
+    /// Open from `layer_dir`, auto-detecting the format.
+    ///
+    /// Checks (in order):
+    /// 1. `layer_dir/presence/matrix.pbmx`  → Packed
+    /// 2. `layer_dir/presence/meta.json`    → Columnar
+    /// 3. `layer_dir/layer_meta.json`        → Implicit (new index)
+    /// 4. `layer_dir/unitigs.bin`            → Implicit with warning (old index)
+    pub fn open(layer_dir: &Path) -> io::Result<Self> {
+        let presence_dir = layer_dir.join("presence");
+
+        if presence_dir.join("matrix.pbmx").exists() {
+            return Ok(Self::Packed(PackedBitMatrix::open(&presence_dir.join("matrix.pbmx"))?));
+        }
+
+        if MatrixMeta::load(&presence_dir).is_ok() {
+            return Ok(Self::Columnar(ColumnarBitMatrix::open(&presence_dir)?));
+        }
+
+        // No presence matrix → Implicit; requires layer_meta.json
+        let meta = LayerMeta::load(layer_dir).map_err(|_| io::Error::new(
+            io::ErrorKind::NotFound,
+            format!(
+                "no presence matrix and no layer_meta.json in {} — run 'obikmer upgrade'",
+                layer_dir.display()
+            ),
+        ))?;
+        Ok(Self::Implicit { n_rows: meta.n, n_cols: 1 })
+    }
+
+    #[inline]
+    pub fn n(&self) -> usize {
+        match self {
+            Self::Columnar(m)            => m.n(),
+            Self::Packed(m)              => m.n_rows,
+            Self::Implicit { n_rows, .. } => *n_rows,
+        }
+    }
+
+    #[inline]
+    pub fn n_cols(&self) -> usize {
+        match self {
+            Self::Columnar(m)            => m.n_cols(),
+            Self::Packed(m)              => m.n_cols,
+            Self::Implicit { n_cols, .. } => *n_cols,
+        }
+    }
+
+    #[inline]
+    pub fn col(&self, c: usize) -> &PersistentBitVec {
+        match self {
+            Self::Columnar(m) => m.col(c),
+            _ => panic!("col() only available on Columnar PersistentBitMatrix"),
+        }
+    }
+
+    #[inline]
+    pub fn col_view(&self, c: usize) -> BitSliceView<'_> {
+        match self {
+            Self::Columnar(m) => m.col(c).view(),
+            Self::Packed(m)   => m.col_slice(c),
+            Self::Implicit { .. } => panic!("col_view() not available on Implicit PersistentBitMatrix"),
+        }
+    }
+
+    /// Column-major point lookup: value at column `c`, slot `slot`, as 0/1.
+    ///
+    /// Unlike [`col_view`](Self::col_view), this never panics on `Implicit`
+    /// (every column reads as present, per the mono-genome fast path) — safe
+    /// to call for any `c < self.n_cols()`.
+    #[inline]
+    pub fn get(&self, c: usize, slot: usize) -> u32 {
+        match self {
+            Self::Columnar(m) => m.col(c).get(slot) as u32,
+            Self::Packed(m)   => m.col_slice(c).get(slot) as u32,
+            Self::Implicit { .. } => 1,
+        }
+    }
+
+    pub fn col_persist(&self, c: usize, path: &Path) -> io::Result<PersistentBitVecBuilder> {
+        match self {
+            Self::Columnar(m) => PersistentBitVecBuilder::build_from(m.col(c), path),
+            Self::Packed(m)   => m.col_persist(c, path),
+            Self::Implicit { n_rows, .. } => {
+                PersistentBitVecBuilder::new_ones(*n_rows, path)
+            }
+        }
+    }
+
+    #[inline]
+    pub fn row(&self, slot: usize) -> Box<[bool]> {
+        match self {
+            Self::Columnar(m)             => m.row(slot),
+            Self::Packed(m)               => m.row(slot),
+            Self::Implicit { n_cols, .. } => vec![true; *n_cols].into_boxed_slice(),
+        }
+    }
+
+    /// Fill `buf[i]` with `col_i[slot]` as 0/1 u32, without allocating.
+    #[inline]
+    pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
+        match self {
+            Self::Columnar(m)             => m.fill_row(slot, buf),
+            Self::Packed(m)               => m.fill_row(slot, buf),
+            Self::Implicit { n_cols, .. } => buf[..*n_cols].fill(1),
+        }
+    }
+
+    /// Extract a sub-matrix containing only the rows at `slots`.
+    ///
+    /// Returns a column-first `Vec<Vec<bool>>`: the outer `Vec` has one entry per
+    /// column, and each inner `Vec` contains the values for the requested rows
+    /// in the same order as `slots`.  Column access is sequential to maximize
+    /// cache efficiency on the underlying mmap.
+    pub fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<bool>> {
+        let n_cols = self.n_cols();
+        let mut out: Vec<Vec<bool>> = Vec::with_capacity(n_cols);
+        for c in 0..n_cols {
+            let mut col_buf = vec![false; slots.len()];
+            match self {
+                Self::Columnar(m) => m.col(c).view().fill_batch(slots, &mut col_buf),
+                Self::Packed(m)   => m.col_slice(c).fill_batch(slots, &mut col_buf),
+                Self::Implicit { .. } => col_buf.iter_mut().for_each(|b| *b = true),
+            }
+            out.push(col_buf);
+        }
+        out
+    }
+
+    /// Like [`sub_matrix`](Self::sub_matrix), but fills caller-provided column
+    /// buffers to avoid allocating the outer `Vec`.
+    ///
+    /// `out` must have length `self.n_cols()`.  Each `out[c]` is cleared,
+    /// resized to `slots.len()`, and filled with the values for column `c`
+    /// in the same order as `slots`.
+    pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]) {
+        assert_eq!(out.len(), self.n_cols());
+        let n = slots.len();
+        if n == 0 {
+            for col in out.iter_mut() { col.clear(); }
+            return;
+        }
+        let mut perm: Vec<usize> = (0..n).collect();
+        perm.sort_by_key(|&i| slots[i]);
+        let sorted_slots: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
+        for (c, col) in out.iter_mut().enumerate() {
+            col.resize(n, false);
+            let mut tmp = vec![false; n];
+            match self {
+                Self::Columnar(m) => m.col(c).view().fill_batch_sorted(&sorted_slots, &mut tmp),
+                Self::Packed(m)   => m.col_slice(c).fill_batch_sorted(&sorted_slots, &mut tmp),
+                Self::Implicit { .. } => tmp.iter_mut().for_each(|b| *b = true),
+            }
+            for (i, &orig_idx) in perm.iter().enumerate() {
+                col[orig_idx] = tmp[i];
+            }
+        }
+    }
+
+    #[inline]
+    pub fn count_ones(&self) -> Array1<u64> {
+        match self {
+            Self::Columnar(m)              => m.count_ones(),
+            Self::Packed(m)                => m.count_ones(),
+            Self::Implicit { n_rows, n_cols } => Array1::from_elem(*n_cols, *n_rows as u64),
+        }
+    }
+
+    pub fn partial_jaccard_dist_matrix(&self) -> (Array2<u64>, Array2<u64>) {
+        match self {
+            Self::Columnar(m)                 => m.partial_jaccard_dist_matrix(),
+            Self::Packed(m)                   => m.partial_jaccard_dist_matrix(),
+            Self::Implicit { n_rows, n_cols } => {
+                let v = *n_rows as u64;
+                let n = *n_cols;
+                let mut inter = Array2::zeros((n, n));
+                let mut union = Array2::zeros((n, n));
+                for i in 0..n { for j in 0..n {
+                    inter[[i, j]] = v; union[[i, j]] = v;
+                }}
+                (inter, union)
+            }
+        }
+    }
+
+    pub fn partial_hamming_dist_matrix(&self) -> Array2<u64> {
+        match self {
+            Self::Columnar(m)              => m.partial_hamming_dist_matrix(),
+            Self::Packed(m)                => m.partial_hamming_dist_matrix(),
+            Self::Implicit { n_cols, .. }  => Array2::zeros((*n_cols, *n_cols)),
+        }
+    }
+
+    /// Append a new column to an on-disk Columnar matrix.
+    pub fn append_column(dir: &Path, value_of: impl Fn(usize) -> bool) -> io::Result<()> {
+        ColumnarBitMatrix::append_column(dir, value_of)
+    }
+}
+
+// ── Trait impls ───────────────────────────────────────────────────────────────
+
+impl ColumnWeights for PersistentBitMatrix {
+    #[inline]
+    fn col_weights(&self) -> Array1<u64> { self.count_ones() }
+}
+
+impl BitPartials for PersistentBitMatrix {
+    #[inline]
+    fn partial_jaccard(&self) -> (Array2<u64>, Array2<u64>) {
+        self.partial_jaccard_dist_matrix()
+    }
+    #[inline]
+    fn partial_hamming(&self) -> Array2<u64> {
+        self.partial_hamming_dist_matrix()
+    }
+}
+
+impl crate::traits::BinaryMatrix for PersistentBitMatrix {
+    #[inline]
+    fn n(&self) -> usize { self.n() }
+    #[inline]
+    fn n_cols(&self) -> usize { self.n_cols() }
+    #[inline]
+    fn row(&self, slot: usize) -> Box<[bool]> { self.row(slot) }
+    #[inline]
+    fn fill_row(&self, slot: usize, buf: &mut [u32]) { self.fill_row(slot, buf) }
+    #[inline]
+    fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]) { self.fill_sub_matrix(slots, out) }
+    #[inline]
+    fn count_ones(&self) -> Array1<u64> { self.count_ones() }
+}

src/obicompactvec/src/bitmatrix/sparse.rs

@@ -0,0 +1,400 @@
+//! `PersistentSparseBitMatrix` — row-major (k-mer-major), deduplicated
+//! sparse alternative to [`super::PersistentBitMatrix`]. See
+//! `docmd/architecture/siblings.md` and the sparse-matrix design plan for
+//! the full rationale (measured sparsity/duplication on real data). Not
+//! used by any production code path yet — a new type, not a replacement.
+//!
+//! On-disk layout (a directory, mirroring [`super::PersistentBitMatrix`]'s
+//! own directory-of-files convention): `sparse_meta.json` plus four components —
+//! `is_multi.prsb` (rank-capable flag: singleton row vs. multi-genome
+//! row), `singleton.pfiv` (genome index, one entry per singleton row,
+//! `ceil(log2(n_cols))` bits each), `multi.pfiv` (`dict_id`, one entry per
+//! multi-genome row, `ceil(log2(n_distinct_multi))` bits each),
+//! `dict_offsets` (Elias-Fano — `.efl`/`.efh` — one entry per *distinct*
+//! multi-genome set, byte offset into `dict_values.bin`), `dict_values.bin`
+//! (varint-encoded sorted genome-index list per distinct set).
+//!
+//! A row's genome set is read by: check `is_multi[slot]`; if singleton,
+//! `singleton[rank0(is_multi, slot)]` is the genome index directly; if
+//! multi, `multi[rank1(is_multi, slot)]` is a `dict_id`, and
+//! `dict_values[dict_offsets[dict_id]..dict_offsets[dict_id+1]]` is its
+//! varint-encoded genome list.
+
+use std::collections::HashMap;
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+
+use ndarray::Array1;
+
+use crate::eliasfano::{EliasFano, EliasFanoBuilder};
+use crate::fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
+use crate::meta::field;
+use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
+
+use super::PersistentBitMatrix;
+use super::packed::PackedBitMatrix;
+
+fn is_multi_path(dir: &Path) -> PathBuf { dir.join("is_multi.prsb") }
+fn singleton_path(dir: &Path) -> PathBuf { dir.join("singleton.pfiv") }
+fn multi_path(dir: &Path) -> PathBuf { dir.join("multi.pfiv") }
+fn dict_offsets_base(dir: &Path) -> PathBuf { dir.join("dict_offsets") }
+fn dict_values_path(dir: &Path) -> PathBuf { dir.join("dict_values.bin") }
+fn meta_path(dir: &Path) -> PathBuf { dir.join("sparse_meta.json") }
+
+struct SparseMeta {
+    n: usize,
+    n_cols: usize,
+    n_singleton: usize,
+    n_multi: usize,
+    n_distinct_multi: usize,
+}
+
+impl SparseMeta {
+    fn load(dir: &Path) -> io::Result<Self> {
+        let s = fs::read_to_string(meta_path(dir))?;
+        let get = |name: &str| {
+            field(&s, name).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("bad sparse_meta.json: missing {name}")))
+        };
+        Ok(Self {
+            n: get("n")?,
+            n_cols: get("n_cols")?,
+            n_singleton: get("n_singleton")?,
+            n_multi: get("n_multi")?,
+            n_distinct_multi: get("n_distinct_multi")?,
+        })
+    }
+
+    fn save(&self, dir: &Path) -> io::Result<()> {
+        fs::write(
+            meta_path(dir),
+            format!(
+                "{{\"n\":{},\"n_cols\":{},\"n_singleton\":{},\"n_multi\":{},\"n_distinct_multi\":{}}}\n",
+                self.n, self.n_cols, self.n_singleton, self.n_multi, self.n_distinct_multi,
+            ),
+        )
+    }
+}
+
+// ── varint (LEB128-style) ────────────────────────────────────────────────────
+
+fn write_varint(buf: &mut Vec<u8>, mut v: u32) {
+    loop {
+        let byte = (v & 0x7F) as u8;
+        v >>= 7;
+        if v == 0 {
+            buf.push(byte);
+            break;
+        }
+        buf.push(byte | 0x80);
+    }
+}
+
+fn read_varint(data: &[u8], pos: &mut usize) -> u32 {
+    let mut result = 0u32;
+    let mut shift = 0u32;
+    loop {
+        let byte = data[*pos];
+        *pos += 1;
+        result |= ((byte & 0x7F) as u32) << shift;
+        if byte & 0x80 == 0 {
+            break;
+        }
+        shift += 7;
+    }
+    result
+}
+
+// ── PersistentSparseBitMatrix ───────────────────────────────────────────────
+
+pub struct PersistentSparseBitMatrix {
+    is_multi: PersistentRankSelectBitVec,
+    singleton: PersistentFixedIntVec,
+    multi: PersistentFixedIntVec,
+    dict_offsets: EliasFano,
+    dict_values: Vec<u8>,
+    n: usize,
+    n_cols: usize,
+    n_distinct_multi: usize,
+}
+
+impl PersistentSparseBitMatrix {
+    pub fn open(dir: &Path) -> io::Result<Self> {
+        let meta = SparseMeta::load(dir)?;
+        let is_multi = PersistentRankSelectBitVec::open(&is_multi_path(dir))?;
+        let singleton = PersistentFixedIntVec::open(&singleton_path(dir))?;
+        let multi = PersistentFixedIntVec::open(&multi_path(dir))?;
+        let dict_offsets = EliasFano::open(&dict_offsets_base(dir))?;
+        let dict_values = fs::read(dict_values_path(dir))?;
+        Ok(Self {
+            is_multi, singleton, multi, dict_offsets, dict_values,
+            n: meta.n, n_cols: meta.n_cols, n_distinct_multi: meta.n_distinct_multi,
+        })
+    }
+
+    #[inline]
+    pub fn n(&self) -> usize { self.n }
+    #[inline]
+    pub fn n_cols(&self) -> usize { self.n_cols }
+
+    /// Byte range of `dict_id`'s varint-encoded genome list within
+    /// `dict_values`.
+    #[inline]
+    fn dict_entry_range(&self, dict_id: usize) -> (usize, usize) {
+        let start = self.dict_offsets.get(dict_id) as usize;
+        let end = if dict_id + 1 < self.n_distinct_multi {
+            self.dict_offsets.get(dict_id + 1) as usize
+        } else {
+            self.dict_values.len()
+        };
+        (start, end)
+    }
+
+    pub fn row(&self, slot: usize) -> Box<[bool]> {
+        let mut out = vec![false; self.n_cols];
+        self.fill_row_bool(slot, &mut out);
+        out.into_boxed_slice()
+    }
+
+    /// Fill `buf[i]` with `1` iff genome `i` is present at `slot`, else `0`
+    /// — mirrors [`super::PersistentBitMatrix::fill_row`]'s signature.
+    pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
+        buf[..self.n_cols].fill(0);
+        if self.is_multi.get(slot) {
+            let pos = self.is_multi.rank1(slot) as usize;
+            let dict_id = self.multi.get(pos) as usize;
+            let (start, end) = self.dict_entry_range(dict_id);
+            let mut p = start;
+            while p < end {
+                buf[read_varint(&self.dict_values, &mut p) as usize] = 1;
+            }
+        } else {
+            let pos = self.is_multi.rank0(slot) as usize;
+            buf[self.singleton.get(pos) as usize] = 1;
+        }
+    }
+
+    fn fill_row_bool(&self, slot: usize, buf: &mut [bool]) {
+        buf.fill(false);
+        if self.is_multi.get(slot) {
+            let pos = self.is_multi.rank1(slot) as usize;
+            let dict_id = self.multi.get(pos) as usize;
+            let (start, end) = self.dict_entry_range(dict_id);
+            let mut p = start;
+            while p < end {
+                buf[read_varint(&self.dict_values, &mut p) as usize] = true;
+            }
+        } else {
+            let pos = self.is_multi.rank0(slot) as usize;
+            buf[self.singleton.get(pos) as usize] = true;
+        }
+    }
+
+    /// Column-oriented per-genome k-mer totals — a naive row-by-row scan,
+    /// not the O(1)-per-column reduction the dense matrix's `count_ones`
+    /// is. Deliberately not optimised: see `docmd/architecture/siblings.md`
+    /// and the sparse-matrix plan's "Explicitly deferred" — column-side
+    /// access stays correct but slow on this type for now.
+    pub fn count_ones(&self) -> Array1<u64> {
+        let mut counts = vec![0u64; self.n_cols];
+        let mut buf = vec![false; self.n_cols];
+        for slot in 0..self.n {
+            self.fill_row_bool(slot, &mut buf);
+            for (c, &present) in buf.iter().enumerate() {
+                if present {
+                    counts[c] += 1;
+                }
+            }
+        }
+        Array1::from(counts)
+    }
+
+    /// Like [`super::PersistentBitMatrix::fill_sub_matrix`]: `out` has one
+    /// entry per genome column, each filled with that column's values at
+    /// `slots`, in `slots` order. Naive (row-major decode, scattered into
+    /// column buffers) — see [`count_ones`](Self::count_ones)'s doc comment.
+    pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]) {
+        assert_eq!(out.len(), self.n_cols);
+        for col in out.iter_mut() {
+            col.clear();
+            col.resize(slots.len(), false);
+        }
+        let mut buf = vec![false; self.n_cols];
+        for (i, &slot) in slots.iter().enumerate() {
+            self.fill_row_bool(slot, &mut buf);
+            for (c, &present) in buf.iter().enumerate() {
+                out[c][i] = present;
+            }
+        }
+    }
+}
+
+impl crate::traits::BinaryMatrix for PersistentSparseBitMatrix {
+    #[inline]
+    fn n(&self) -> usize { self.n() }
+    #[inline]
+    fn n_cols(&self) -> usize { self.n_cols() }
+    #[inline]
+    fn row(&self, slot: usize) -> Box<[bool]> { self.row(slot) }
+    #[inline]
+    fn fill_row(&self, slot: usize, buf: &mut [u32]) { self.fill_row(slot, buf) }
+    #[inline]
+    fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]) { self.fill_sub_matrix(slots, out) }
+    #[inline]
+    fn count_ones(&self) -> Array1<u64> { self.count_ones() }
+}
+
+// ── PersistentSparseBitMatrixBuilder ────────────────────────────────────────
+
+pub struct PersistentSparseBitMatrixBuilder {
+    dir: PathBuf,
+    n_cols: usize,
+    is_multi: Vec<bool>,
+    singleton_values: Vec<u32>,
+    multi_dict_ids: Vec<u32>,
+    dedup: HashMap<Vec<u32>, u32>,
+    dict_sets_in_order: Vec<Vec<u32>>,
+}
+
+impl PersistentSparseBitMatrixBuilder {
+    pub fn new(n: usize, n_cols: usize, dir: &Path) -> io::Result<Self> {
+        fs::create_dir_all(dir)?;
+        Ok(Self {
+            dir: dir.to_path_buf(),
+            n_cols,
+            is_multi: Vec::with_capacity(n),
+            singleton_values: Vec::new(),
+            multi_dict_ids: Vec::new(),
+            dedup: HashMap::new(),
+            dict_sets_in_order: Vec::new(),
+        })
+    }
+
+    /// Appends one row's genome set — `genomes` sorted ascending, each
+    /// `< n_cols`. Rows must be pushed in slot order (0, 1, 2, ...), one
+    /// call per k-mer slot of the layer being converted/built.
+    ///
+    /// Cardinality 0 (no genome present — not expected on a real built
+    /// index, every slot has at least one, but handled correctly rather
+    /// than assumed away) routes through the dictionary path as a genuine
+    /// empty entry, *not* the singleton shortcut: a placeholder singleton
+    /// value would be indistinguishable from a real singleton at that same
+    /// genome index on read-back.
+    pub fn push_row(&mut self, genomes: &[u32]) {
+        match genomes.len() {
+            1 => {
+                self.is_multi.push(false);
+                self.singleton_values.push(genomes[0]);
+            }
+            _ => {
+                self.is_multi.push(true);
+                let id = if let Some(&id) = self.dedup.get(genomes) {
+                    id
+                } else {
+                    let id = self.dict_sets_in_order.len() as u32;
+                    self.dict_sets_in_order.push(genomes.to_vec());
+                    self.dedup.insert(genomes.to_vec(), id);
+                    id
+                };
+                self.multi_dict_ids.push(id);
+            }
+        }
+    }
+
+    pub fn close(self) -> io::Result<()> {
+        let n = self.is_multi.len();
+        let n_singleton = self.singleton_values.len();
+        let n_multi = self.multi_dict_ids.len();
+        let n_distinct_multi = self.dict_sets_in_order.len();
+
+        let mut is_multi_b = PersistentRankSelectBitVecBuilder::new(n, &is_multi_path(&self.dir))?;
+        for (i, &v) in self.is_multi.iter().enumerate() {
+            is_multi_b.set(i, v);
+        }
+        is_multi_b.close()?;
+
+        let singleton_width = bit_width_for_range(self.n_cols as u64);
+        let mut singleton_b = PersistentFixedIntVecBuilder::new(n_singleton, singleton_width, &singleton_path(&self.dir))?;
+        for (i, &v) in self.singleton_values.iter().enumerate() {
+            singleton_b.set(i, v as u64);
+        }
+        singleton_b.close()?;
+
+        let multi_width = bit_width_for_range(n_distinct_multi as u64);
+        let mut multi_b = PersistentFixedIntVecBuilder::new(n_multi, multi_width, &multi_path(&self.dir))?;
+        for (i, &v) in self.multi_dict_ids.iter().enumerate() {
+            multi_b.set(i, v as u64);
+        }
+        multi_b.close()?;
+
+        let mut dict_values = Vec::new();
+        let mut offsets: Vec<u64> = Vec::with_capacity(n_distinct_multi);
+        for set in &self.dict_sets_in_order {
+            offsets.push(dict_values.len() as u64);
+            for &g in set {
+                write_varint(&mut dict_values, g);
+            }
+        }
+        let universe = dict_values.len() as u64 + 1;
+        let mut offsets_b = EliasFanoBuilder::new(offsets.len(), universe, &dict_offsets_base(&self.dir))?;
+        for &o in &offsets {
+            offsets_b.push(o);
+        }
+        offsets_b.close()?;
+        fs::write(dict_values_path(&self.dir), &dict_values)?;
+
+        SparseMeta { n, n_cols: self.n_cols, n_singleton, n_multi, n_distinct_multi }.save(&self.dir)?;
+        Ok(())
+    }
+
+    pub fn finish(self) -> io::Result<PersistentSparseBitMatrix> {
+        let dir = self.dir.clone();
+        self.close()?;
+        PersistentSparseBitMatrix::open(&dir)
+    }
+
+    /// Builds a sparse matrix from an already-built dense
+    /// [`PersistentBitMatrix`] — row-by-row transpose via
+    /// [`PersistentBitMatrix::fill_row`], for migrating an existing index.
+    pub fn build_from_dense(dense: &PersistentBitMatrix, dir: &Path) -> io::Result<Self> {
+        let n = dense.n();
+        let n_cols = dense.n_cols();
+        let mut builder = Self::new(n, n_cols, dir)?;
+        let mut buf = vec![0u32; n_cols];
+        let mut genomes: Vec<u32> = Vec::new();
+        for slot in 0..n {
+            dense.fill_row(slot, &mut buf);
+            genomes.clear();
+            genomes.extend((0..n_cols).filter(|&c| buf[c] != 0).map(|c| c as u32));
+            builder.push_row(&genomes);
+        }
+        Ok(builder)
+    }
+}
+
+/// `pack --sparse`'s entry point: converts a presence directory into the
+/// sparse on-disk format in place, mirroring [`super::pack_bit_matrix`]'s
+/// convention (old-format files removed only after the new format is
+/// fully written, so a crash mid-conversion leaves the previous, still
+/// valid format rather than a half-written one). Idempotent — does
+/// nothing if `is_multi.prsb` already exists. Packs to dense
+/// (`matrix.pbmx`) first via [`super::pack_bit_matrix`] if that hasn't
+/// happened yet, since the dense→sparse transpose (`build_from_dense`)
+/// needs random row access, which only the packed/columnar dense forms
+/// give — not a further reason to keep the dense file around afterward.
+pub fn pack_sparse_bit_matrix(dir: &Path) -> io::Result<()> {
+    if is_multi_path(dir).exists() {
+        return Ok(());
+    }
+
+    let packed_path = dir.join("matrix.pbmx");
+    if !packed_path.exists() {
+        super::pack_bit_matrix(dir)?;
+    }
+
+    let dense = PersistentBitMatrix::Packed(PackedBitMatrix::open(&packed_path)?);
+    PersistentSparseBitMatrixBuilder::build_from_dense(&dense, dir)?.close()?;
+    drop(dense);
+    fs::remove_file(&packed_path)?;
+    Ok(())
+}

src/obicompactvec/src/bitvec.rs

@@ -14,16 +14,20 @@ const MAGIC: [u8; 4] = *b"PBIV";
 const HEADER_SIZE: usize = 16;
 
 #[inline]
-pub(crate) fn n_words(n: usize) -> usize { n.div_ceil(64) }
+pub(crate) fn n_words(n: usize) -> usize {
+    n.div_ceil(64)
+}
 
 #[inline]
-fn n_bytes_for_words(n: usize) -> usize { n_words(n) * 8 }
+fn n_bytes_for_words(n: usize) -> usize {
+    n_words(n) * 8
+}
 
 // ── PersistentBitVec ──────────────────────────────────────────────────────────
 
 pub struct PersistentBitVec {
     mmap: Mmap,
-    n:    usize,
+    n: usize,
     path: PathBuf,
 }
 
@@ -31,78 +35,159 @@ impl PersistentBitVec {
     pub fn open(path: &Path) -> io::Result<Self> {
         let mmap = unsafe { Mmap::map(&File::open(path)?)? };
         if mmap.len() < HEADER_SIZE {
-            return Err(io::Error::new(io::ErrorKind::InvalidData, "PBIV file too short"));
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "PBIV file too short",
+            ));
         }
         if &mmap[0..4] != &MAGIC {
             return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PBIV magic"));
         }
         let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
-        Ok(Self { mmap, n, path: path.to_path_buf() })
+        Ok(Self {
+            mmap,
+            n,
+            path: path.to_path_buf(),
+        })
     }
 
-    pub fn path(&self) -> &Path { &self.path }
-    pub fn len(&self)      -> usize { self.n }
-    pub fn is_empty(&self) -> bool  { self.n == 0 }
+    #[inline]
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.n
+    }
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.n == 0
+    }
 
+    #[inline]
     pub fn get(&self, slot: usize) -> bool {
         (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
     }
 
+    /// Batch lookup: read bits at `slots` in an order that minimizes
+    /// cache misses.
+    ///
+    /// The slots are sorted internally before reading so that accesses to
+    /// the underlying mmap are as sequential as possible, then the results
+    /// are reordered to match the input order.
+    pub fn get_batch(&self, slots: &[usize]) -> Vec<bool> {
+        let mut out = vec![false; slots.len()];
+        self.fill_batch(slots, &mut out);
+        out
+    }
+
+    /// Like [`get_batch`](Self::get_batch), but fills a caller-provided buffer.
+    pub fn fill_batch(&self, slots: &[usize], out: &mut [bool]) {
+        assert_eq!(slots.len(), out.len());
+        let n = slots.len();
+        if n == 0 {
+            return;
+        }
+        let mut perm: Vec<usize> = (0..n).collect();
+        perm.sort_by_key(|&i| slots[i]);
+        let sorted: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
+        let mut tmp = vec![false; n];
+        self.fill_batch_sorted(&sorted, &mut tmp);
+        for (i, &orig_idx) in perm.iter().enumerate() {
+            out[orig_idx] = tmp[i];
+        }
+    }
+
+    /// Fill `out` assuming `sorted_slots` is already in ascending order.
+    /// Results are written in `sorted_slots` order (no reordering).
+    pub(crate) fn fill_batch_sorted(&self, sorted_slots: &[usize], out: &mut [bool]) {
+        assert_eq!(sorted_slots.len(), out.len());
+        for (i, &slot) in sorted_slots.iter().enumerate() {
+            out[i] = self.get(slot);
+        }
+    }
+
     // SAFETY: mmap is page-aligned, HEADER_SIZE=16 divisible by 8 → u64-aligned.
+    #[inline]
     fn data_words(&self) -> &[u64] {
-        let nw  = n_words(self.n);
+        let nw = n_words(self.n);
         let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
         unsafe { std::slice::from_raw_parts(ptr, nw) }
     }
 
+    #[inline]
     pub fn view(&self) -> BitSliceView<'_> {
         BitSliceView::new(self.data_words(), self.n)
     }
 
-    pub fn words(&self) -> &[u64] { self.data_words() }
+    #[inline]
+    pub fn words(&self) -> &[u64] {
+        self.data_words()
+    }
 
-    pub fn count_ones(&self)  -> u64 { self.view().count_ones() }
-    pub fn count_zeros(&self) -> u64 { self.view().count_zeros() }
+    #[inline]
+    pub fn count_ones(&self) -> u64 {
+        self.view().count_ones()
+    }
+    #[inline]
+    pub fn count_zeros(&self) -> u64 {
+        self.view().count_zeros()
+    }
 
+    #[inline]
     pub fn partial_jaccard_dist(&self, other: &PersistentBitVec) -> (u64, u64) {
         self.view().partial_jaccard_dist(other.view())
     }
+    #[inline]
     pub fn jaccard_dist(&self, other: &PersistentBitVec) -> f64 {
         self.view().jaccard_dist(other.view())
     }
+    #[inline]
     pub fn hamming_dist(&self, other: &PersistentBitVec) -> u64 {
         self.view().hamming_dist(other.view())
     }
 
+    #[inline]
     pub fn iter(&self) -> BitIter<'_> {
-        BitIter { words: self.data_words(), slot: 0, n: self.n }
+        BitIter {
+            words: self.data_words(),
+            slot: 0,
+            n: self.n,
+        }
     }
 }
 
 impl<'a> IntoIterator for &'a PersistentBitVec {
     type Item = bool;
     type IntoIter = BitIter<'a>;
-    fn into_iter(self) -> BitIter<'a> { self.iter() }
+    #[inline]
+    fn into_iter(self) -> BitIter<'a> {
+        self.iter()
+    }
 }
 
 // ── BitIter ───────────────────────────────────────────────────────────────────
 
 pub struct BitIter<'a> {
     words: &'a [u64],
-    slot:  usize,
-    n:     usize,
+    slot: usize,
+    n: usize,
 }
 
 impl ExactSizeIterator for BitIter<'_> {}
 
 impl Iterator for BitIter<'_> {
     type Item = bool;
+    #[inline]
     fn next(&mut self) -> Option<bool> {
-        if self.slot >= self.n { return None; }
+        if self.slot >= self.n {
+            return None;
+        }
         let v = (self.words[self.slot >> 6] >> (self.slot & 63)) & 1 != 0;
         self.slot += 1;
         Some(v)
     }
+    #[inline]
     fn size_hint(&self) -> (usize, Option<usize>) {
         let rem = self.n - self.slot;
         (rem, Some(rem))
@@ -113,7 +198,7 @@ impl Iterator for BitIter<'_> {
 
 pub struct PersistentBitVecBuilder {
     mmap: MmapMut,
-    n:    usize,
+    n: usize,
     path: PathBuf,
 }
 
@@ -121,7 +206,10 @@ impl PersistentBitVecBuilder {
     pub fn new(n: usize, path: &Path) -> io::Result<Self> {
         let file_size = HEADER_SIZE + n_bytes_for_words(n);
         let mut file = OpenOptions::new()
-            .read(true).write(true).create(true).truncate(true)
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(true)
             .open(path)?;
         file.write_all(&MAGIC)?;
         file.write_all(&[0u8; 4])?;
@@ -129,20 +217,31 @@ impl PersistentBitVecBuilder {
         file.seek(SeekFrom::Start(0))?;
         file.set_len(file_size as u64)?;
         let mmap = unsafe { MmapMut::map_mut(&file)? };
-        Ok(Self { mmap, n, path: path.to_path_buf() })
+        Ok(Self {
+            mmap,
+            n,
+            path: path.to_path_buf(),
+        })
     }
 
     pub fn from_raw_bytes(bytes: &[u8], n: usize, path: &Path) -> io::Result<Self> {
         let file_size = HEADER_SIZE + n_bytes_for_words(n);
         let file = OpenOptions::new()
-            .read(true).write(true).create(true).truncate(true)
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(true)
             .open(path)?;
         file.set_len(file_size as u64)?;
         let mut mmap = unsafe { MmapMut::map_mut(&file)? };
         mmap[0..4].copy_from_slice(&MAGIC);
         mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
         mmap[HEADER_SIZE..HEADER_SIZE + bytes.len()].copy_from_slice(bytes);
-        Ok(Self { mmap, n, path: path.to_path_buf() })
+        Ok(Self {
+            mmap,
+            n,
+            path: path.to_path_buf(),
+        })
     }
 
     /// Create an all-ones bit vector of length `n` at `path`.
@@ -150,10 +249,13 @@ impl PersistentBitVecBuilder {
     /// More efficient than `new(n, path)` + `not()`: the data is written as
     /// 0xFF bytes in a single sequential pass, with no intermediate all-zeros state.
     pub fn new_ones(n: usize, path: &Path) -> io::Result<Self> {
-        let nw        = n_words(n);
+        let nw = n_words(n);
         let file_size = HEADER_SIZE + nw * 8;
-        let mut file  = OpenOptions::new()
-            .read(true).write(true).create(true).truncate(true)
+        let mut file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(true)
             .open(path)?;
         file.write_all(&MAGIC)?;
         file.write_all(&[0u8; 4])?;
@@ -165,11 +267,15 @@ impl PersistentBitVecBuilder {
         // Clear padding bits in the last word so trailing bits are always 0.
         let rem = n % 64;
         if rem != 0 {
-            let ptr   = mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
+            let ptr = mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
             let words = unsafe { std::slice::from_raw_parts_mut(ptr, nw) };
             words[nw - 1] &= (1u64 << rem) - 1;
         }
-        Ok(Self { mmap, n, path: path.to_path_buf() })
+        Ok(Self {
+            mmap,
+            n,
+            path: path.to_path_buf(),
+        })
     }
 
     pub fn build_from(source: &PersistentBitVec, path: &Path) -> io::Result<Self> {
@@ -177,14 +283,25 @@ impl PersistentBitVecBuilder {
         let file = OpenOptions::new().read(true).write(true).open(path)?;
         let mmap = unsafe { MmapMut::map_mut(&file)? };
         let n = source.len();
-        Ok(Self { mmap, n, path: path.to_path_buf() })
+        Ok(Self {
+            mmap,
+            n,
+            path: path.to_path_buf(),
+        })
     }
 
-    pub fn build_from_counts(source: &PersistentCompactIntVec, threshold: u32, path: &Path) -> io::Result<Self> {
+    pub fn build_from_counts(
+        source: &PersistentCompactIntVec,
+        threshold: u32,
+        path: &Path,
+    ) -> io::Result<Self> {
         let n = source.len();
         let file_size = HEADER_SIZE + n_bytes_for_words(n);
         let mut file = OpenOptions::new()
-            .read(true).write(true).create(true).truncate(true)
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(true)
             .open(path)?;
         file.write_all(&MAGIC)?;
         file.write_all(&[0u8; 4])?;
@@ -193,51 +310,74 @@ impl PersistentBitVecBuilder {
         file.set_len(file_size as u64)?;
         let mut mmap = unsafe { MmapMut::map_mut(&file)? };
         {
-            let nw  = n_words(n);
+            let nw = n_words(n);
             let ptr = mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
             let words = unsafe { std::slice::from_raw_parts_mut(ptr, nw) };
             for (slot, count) in source.iter().enumerate() {
-                if count >= threshold { words[slot >> 6] |= 1u64 << (slot & 63); }
+                if count >= threshold {
+                    words[slot >> 6] |= 1u64 << (slot & 63);
+                }
             }
         }
-        Ok(Self { mmap, n, path: path.to_path_buf() })
+        Ok(Self {
+            mmap,
+            n,
+            path: path.to_path_buf(),
+        })
     }
 
     pub fn build_from_presence(source: &PersistentCompactIntVec, path: &Path) -> io::Result<Self> {
         Self::build_from_counts(source, 1, path)
     }
 
-    pub fn len(&self)      -> usize { self.n }
-    pub fn is_empty(&self) -> bool  { self.n == 0 }
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.n
+    }
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.n == 0
+    }
 
+    #[inline]
     pub fn get(&self, slot: usize) -> bool {
         (self.mmap[HEADER_SIZE + (slot >> 3)] >> (slot & 7)) & 1 != 0
     }
 
+    #[inline]
     pub fn set(&mut self, slot: usize, value: bool) {
         let bit = 1u64 << (slot & 63);
-        if value { self.data_words_mut()[slot >> 6] |=  bit; }
-        else     { self.data_words_mut()[slot >> 6] &= !bit; }
+        if value {
+            self.data_words_mut()[slot >> 6] |= bit;
+        } else {
+            self.data_words_mut()[slot >> 6] &= !bit;
+        }
     }
 
+    #[inline]
     fn data_words(&self) -> &[u64] {
-        let nw  = n_words(self.n);
+        let nw = n_words(self.n);
         let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
         unsafe { std::slice::from_raw_parts(ptr, nw) }
     }
 
     // SAFETY: same alignment argument as PersistentBitVec::data_words.
+    #[inline]
     fn data_words_mut(&mut self) -> &mut [u64] {
-        let nw  = n_words(self.n);
+        let nw = n_words(self.n);
         let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
         unsafe { std::slice::from_raw_parts_mut(ptr, nw) }
     }
 
+    #[inline]
     pub fn view(&self) -> BitSliceView<'_> {
         BitSliceView::new(self.data_words(), self.n)
     }
 
-    pub fn words(&self) -> &[u64] { self.data_words() }
+    #[inline]
+    pub fn words(&self) -> &[u64] {
+        self.data_words()
+    }
 
     pub fn copy_from(&mut self, src: BitSliceView<'_>) {
         assert_eq!(self.n, src.len(), "BitSliceView length mismatch");
@@ -246,25 +386,35 @@ impl PersistentBitVecBuilder {
 
     pub fn and(&mut self, other: BitSliceView<'_>) {
         assert_eq!(self.n, other.len(), "BitSliceView length mismatch");
-        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) { *w &= o; }
+        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) {
+            *w &= o;
+        }
     }
 
     pub fn or(&mut self, other: BitSliceView<'_>) {
         assert_eq!(self.n, other.len(), "BitSliceView length mismatch");
-        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) { *w |= o; }
+        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) {
+            *w |= o;
+        }
     }
 
     pub fn xor(&mut self, other: BitSliceView<'_>) {
         assert_eq!(self.n, other.len(), "BitSliceView length mismatch");
-        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) { *w ^= o; }
+        for (w, &o) in self.data_words_mut().iter_mut().zip(other.words()) {
+            *w ^= o;
+        }
     }
 
     pub fn not(&mut self) {
-        let rem   = self.n % 64;
+        let rem = self.n % 64;
         let words = self.data_words_mut();
-        for w in words.iter_mut() { *w ^= u64::MAX; }
+        for w in words.iter_mut() {
+            *w ^= u64::MAX;
+        }
         if rem != 0 {
-            if let Some(last) = words.last_mut() { *last &= (1u64 << rem) - 1; }
+            if let Some(last) = words.last_mut() {
+                *last &= (1u64 << rem) - 1;
+            }
         }
     }
 
@@ -276,17 +426,21 @@ impl PersistentBitVecBuilder {
         let words = self.data_words_mut();
         let nw = n_words(n);
         for wi in 0..nw {
-            let base  = wi * 64;
+            let base = wi * 64;
             let limit = (base + 64).min(n);
             let mut mask = 0u64;
             for bit in 0..(limit - base) {
                 let b = primary[base + bit];
-                if b < 255 && pred(b as u32) { mask |= 1u64 << bit; }
+                if b < 255 && pred(b as u32) {
+                    mask |= 1u64 << bit;
+                }
             }
             words[wi] |= mask;
         }
         for (slot, val) in col.overflow_entries() {
-            if pred(val) { words[slot >> 6] |= 1u64 << (slot & 63); }
+            if pred(val) {
+                words[slot >> 6] |= 1u64 << (slot & 63);
+            }
         }
     }
 
@@ -298,17 +452,21 @@ impl PersistentBitVecBuilder {
         let words = self.data_words_mut();
         let nw = n_words(n);
         for wi in 0..nw {
-            let base  = wi * 64;
+            let base = wi * 64;
             let limit = (base + 64).min(n);
             let mut mask = 0u64;
             for bit in 0..(limit - base) {
                 let b = primary[base + bit];
-                if b < 255 && !pred(b as u32) { mask |= 1u64 << bit; }
+                if b < 255 && !pred(b as u32) {
+                    mask |= 1u64 << bit;
+                }
             }
             words[wi] &= !mask;
         }
         for (slot, val) in col.overflow_entries() {
-            if !pred(val) { words[slot >> 6] &= !(1u64 << (slot & 63)); }
+            if !pred(val) {
+                words[slot >> 6] &= !(1u64 << (slot & 63));
+            }
         }
     }
 
@@ -320,25 +478,32 @@ impl PersistentBitVecBuilder {
         let words = self.data_words_mut();
         let nw = n_words(n);
         for wi in 0..nw {
-            let base  = wi * 64;
+            let base = wi * 64;
             let limit = (base + 64).min(n);
             let mut mask = 0u64;
             for bit in 0..(limit - base) {
                 let b = primary[base + bit];
-                if b < 255 && pred(b as u32) { mask |= 1u64 << bit; }
+                if b < 255 && pred(b as u32) {
+                    mask |= 1u64 << bit;
+                }
             }
             words[wi] ^= mask;
         }
         for (slot, val) in col.overflow_entries() {
-            if pred(val) { words[slot >> 6] ^= 1u64 << (slot & 63); }
+            if pred(val) {
+                words[slot >> 6] ^= 1u64 << (slot & 63);
+            }
         }
     }
 
+    #[inline]
     pub fn iter(&self) -> BitSliceIter<'_> {
         self.view().iter()
     }
 
-    pub fn close(self) -> io::Result<()> { self.mmap.flush() }
+    pub fn close(self) -> io::Result<()> {
+        self.mmap.flush()
+    }
 
     pub fn finish(self) -> io::Result<PersistentBitVec> {
         let path = self.path.clone();

src/obicompactvec/src/builder.rs

@@ -52,6 +52,7 @@ impl PersistentCompactIntVecBuilder {
         Ok(Self { path: path.to_path_buf(), mmap, n, overflow })
     }
 
+    #[inline]
     pub fn get(&self, slot: usize) -> u32 {
         match self.mmap[HEADER_SIZE + slot] {
             255 => *self.overflow.get(&slot).expect("sentinel without overflow entry"),
@@ -59,6 +60,7 @@ impl PersistentCompactIntVecBuilder {
         }
     }
 
+    #[inline]
     pub fn set(&mut self, slot: usize, value: u32) {
         if value < 255 {
             self.mmap[HEADER_SIZE + slot] = value as u8;
@@ -69,20 +71,28 @@ impl PersistentCompactIntVecBuilder {
         }
     }
 
+    #[inline]
     pub fn len(&self)      -> usize { self.n }
+    #[inline]
     pub fn is_empty(&self) -> bool  { self.n == 0 }
 
+    #[inline]
     pub fn primary_bytes(&self)     -> &[u8]      { &self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
+    #[inline]
     pub fn primary_bytes_mut(&mut self) -> &mut [u8] { &mut self.mmap[HEADER_SIZE..HEADER_SIZE + self.n] }
+    #[inline]
     pub fn clear_overflow(&mut self) { self.overflow.clear(); }
 
+    #[inline]
     pub fn sum(&self) -> u64 {
         byte_sum(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n], self.overflow.values().copied())
     }
+    #[inline]
     pub fn count_nonzero(&self) -> u64 {
         byte_count_nonzero(&self.mmap[HEADER_SIZE..HEADER_SIZE + self.n])
     }
 
+    #[inline]
     pub fn view(&self) -> IntSliceView<'_> {
         // Builder overflow is a HashMap, not sorted raw bytes — convert on the fly
         // by collecting into a sorted vec and storing in a thread-local buffer.
@@ -94,10 +104,12 @@ impl PersistentCompactIntVecBuilder {
         IntSliceView::new(primary, &[], 0, self.n)
     }
 
+    #[inline]
     pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + '_ {
         self.overflow.iter().map(|(&k, &v)| (k, v))
     }
 
+    #[inline]
     pub fn inc(&mut self, slot: usize) {
         let v = self.get(slot);
         self.set(slot, v.saturating_add(1));

src/obicompactvec/src/eliasfano.rs

@@ -0,0 +1,133 @@
+//! Elias-Fano encoding of a monotone (non-decreasing) `u64` sequence —
+//! used only for the sparse presence matrix's dictionary offsets (see
+//! `docmd/architecture/siblings.md` and the sparse-matrix plan): a
+//! monotone, unbounded-magnitude sequence, the one component in that
+//! design that genuinely needs this.
+//!
+//! Classic two-array construction: each value's low `l` bits are packed
+//! at fixed width ([`crate::fixedintvec::PersistentFixedIntVec`]), its
+//! high bits (`value >> l`) are unary-coded into a bitvector
+//! ([`crate::rankselect::PersistentRankSelectBitVec`]) — a 1 bit at
+//! position `(value >> l) + i` for the `i`-th value, which is itself
+//! monotone non-decreasing whenever the input is, so it can be built by a
+//! single forward pass with no lookahead. Decoding value `i`:
+//! `((select1(i) - i) << l) | low[i]`.
+//!
+//! `l` is chosen as `floor(log2(universe / n))` (0 if `n >= universe`),
+//! the standard choice that keeps the high bitvector's length — `n +
+//! (universe >> l) + 1` — within a small constant factor of `n`.
+
+use std::io;
+use std::path::{Path, PathBuf};
+
+use crate::fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder};
+use crate::rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
+
+fn low_bits_width(n: usize, universe: u64) -> u32 {
+    if n == 0 || universe <= n as u64 {
+        0
+    } else {
+        // floor(log2(universe / n))
+        (universe / n as u64).max(1).ilog2()
+    }
+}
+
+fn low_path(base: &Path) -> PathBuf {
+    let mut p = base.as_os_str().to_owned();
+    p.push(".efl");
+    PathBuf::from(p)
+}
+
+fn high_path(base: &Path) -> PathBuf {
+    let mut p = base.as_os_str().to_owned();
+    p.push(".efh");
+    PathBuf::from(p)
+}
+
+// ── EliasFano ────────────────────────────────────────────────────────────────
+
+pub struct EliasFano {
+    low: PersistentFixedIntVec,
+    high: PersistentRankSelectBitVec,
+    n: usize,
+    low_width: u32,
+}
+
+impl EliasFano {
+    /// Opens a structure previously built at `base` (i.e. `{base}.efl` and
+    /// `{base}.efh`).
+    pub fn open(base: &Path) -> io::Result<Self> {
+        let low = PersistentFixedIntVec::open(&low_path(base))?;
+        let high = PersistentRankSelectBitVec::open(&high_path(base))?;
+        let n = low.len();
+        let low_width = low.width();
+        Ok(Self { low, high, n, low_width })
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize { self.n }
+    #[inline]
+    pub fn is_empty(&self) -> bool { self.n == 0 }
+
+    /// The `i`-th value of the encoded sequence.
+    pub fn get(&self, i: usize) -> u64 {
+        debug_assert!(i < self.n);
+        let pos = self.high.select1(i as u64);
+        let high_part = pos as u64 - i as u64;
+        (high_part << self.low_width) | self.low.get(i)
+    }
+}
+
+// ── EliasFanoBuilder ─────────────────────────────────────────────────────────
+
+pub struct EliasFanoBuilder {
+    low: PersistentFixedIntVecBuilder,
+    high: PersistentRankSelectBitVecBuilder,
+    low_width: u32,
+    n: usize,
+    next: usize,
+    last_value: u64,
+}
+
+impl EliasFanoBuilder {
+    /// `n` values will be [`push`](Self::push)ed, in non-decreasing order,
+    /// each `< universe`. Writes `{base}.efl` and `{base}.efh`.
+    pub fn new(n: usize, universe: u64, base: &Path) -> io::Result<Self> {
+        let low_width = low_bits_width(n, universe);
+        let high_len = n + (universe >> low_width) as usize + 1;
+        // `PersistentFixedIntVec` genuinely supports width 0 (every value
+        // 0, no storage) — the persisted width byte is then the source of
+        // truth on reopen, no separate bookkeeping of `low_width` needed.
+        let low = PersistentFixedIntVecBuilder::new(n, low_width, &low_path(base))?;
+        let high = PersistentRankSelectBitVecBuilder::new(high_len, &high_path(base))?;
+        Ok(Self { low, high, low_width, n, next: 0, last_value: 0 })
+    }
+
+    /// Appends the next value — must be `>= ` every previously pushed
+    /// value (monotone non-decreasing), and `< universe` as given to
+    /// [`new`](Self::new).
+    pub fn push(&mut self, value: u64) {
+        assert!(self.next < self.n, "push() called more than n={} times", self.n);
+        assert!(
+            self.next == 0 || value >= self.last_value,
+            "push({value}) breaks monotonicity: last value was {}", self.last_value
+        );
+        let low_mask = if self.low_width >= 64 { u64::MAX } else { (1u64 << self.low_width) - 1 };
+        self.low.set(self.next, value & low_mask);
+        let high_part = value >> self.low_width;
+        self.high.set(high_part as usize + self.next, true);
+        self.last_value = value;
+        self.next += 1;
+    }
+
+    pub fn close(self) -> io::Result<()> {
+        assert_eq!(self.next, self.n, "push() called {} times, expected n={}", self.next, self.n);
+        self.low.close()?;
+        self.high.close()
+    }
+
+    pub fn finish(self, base: &Path) -> io::Result<EliasFano> {
+        self.close()?;
+        EliasFano::open(base)
+    }
+}

src/obicompactvec/src/fixedintvec.rs

@@ -0,0 +1,202 @@
+//! Fixed-bit-width packed integer vector — `n` values, each exactly `width`
+//! bits (1..=64, chosen once at construction from the actual data being
+//! stored), no overflow scheme. Mirrors [`crate::bitvec::PersistentBitVec`]'s
+//! mmap-backed reader/builder split, generalised from 1 bit/value to an
+//! arbitrary width.
+//!
+//! Deliberately distinct from [`crate::reader::PersistentCompactIntVec`]
+//! (`PCIV`): that format is "1 primary byte + overflow map for values ≥
+//! 255", tuned for mostly-small-value distributions. This format is for
+//! values roughly uniform over a known range (e.g. genome indices, or
+//! `dict_id`s) — every value costs the same, `width` bits, by construction.
+
+use std::fs::{File, OpenOptions};
+use std::io::{self, Seek, SeekFrom, Write as _};
+use std::path::{Path, PathBuf};
+
+use memmap2::{Mmap, MmapMut};
+
+const MAGIC: [u8; 4] = *b"PFIV";
+
+// Header: magic(4) + width(1) + _pad(3) + n(8) = 16 bytes.
+// Data starts at offset 16, u64-aligned (mmap base is page-aligned, 16 % 8 == 0).
+const HEADER_SIZE: usize = 16;
+
+/// Smallest bit width that can hold every value in `0..range` (`range`
+/// itself excluded, i.e. the number of distinct values needed) — the
+/// standard `ceil(log2(range))` sizing used throughout the sparse matrix
+/// design, with the same `.max(1)` floor (a single-value range still needs
+/// 1 bit, not 0, since `0..1` is a real distinct value).
+pub fn bit_width_for_range(range: u64) -> u32 {
+    if range <= 1 {
+        1
+    } else {
+        (u64::BITS - (range - 1).leading_zeros()).max(1)
+    }
+}
+
+#[inline]
+fn n_words_for(n: usize, width: u32) -> usize {
+    // +1 guard word: a value straddling the last word boundary reads/writes
+    // one bit into a word past the nominal bit count otherwise.
+    (n as u64 * width as u64).div_ceil(64) as usize + 1
+}
+
+#[inline]
+fn mask(width: u32) -> u64 {
+    if width >= 64 { u64::MAX } else { (1u64 << width) - 1 }
+}
+
+// ── PersistentFixedIntVec ───────────────────────────────────────────────────
+
+pub struct PersistentFixedIntVec {
+    mmap: Mmap,
+    n: usize,
+    width: u32,
+    path: PathBuf,
+}
+
+impl PersistentFixedIntVec {
+    pub fn open(path: &Path) -> io::Result<Self> {
+        let mmap = unsafe { Mmap::map(&File::open(path)?)? };
+        if mmap.len() < HEADER_SIZE {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "PFIV file too short"));
+        }
+        if mmap[0..4] != MAGIC {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PFIV magic"));
+        }
+        let width = mmap[4] as u32;
+        let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
+        Ok(Self { mmap, n, width, path: path.to_path_buf() })
+    }
+
+    #[inline]
+    pub fn path(&self) -> &Path { &self.path }
+    #[inline]
+    pub fn len(&self) -> usize { self.n }
+    #[inline]
+    pub fn is_empty(&self) -> bool { self.n == 0 }
+    #[inline]
+    pub fn width(&self) -> u32 { self.width }
+
+    // SAFETY: mmap is page-aligned, HEADER_SIZE=16 divisible by 8 → u64-aligned.
+    #[inline]
+    fn data_words(&self) -> &[u64] {
+        let nw = n_words_for(self.n, self.width);
+        let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
+        unsafe { std::slice::from_raw_parts(ptr, nw) }
+    }
+
+    #[inline]
+    pub fn get(&self, slot: usize) -> u64 {
+        debug_assert!(slot < self.n);
+        get_packed(self.data_words(), slot, self.width)
+    }
+}
+
+#[inline]
+fn get_packed(words: &[u64], slot: usize, width: u32) -> u64 {
+    let bit_offset = slot as u64 * width as u64;
+    let word_idx = (bit_offset / 64) as usize;
+    let bit_in_word = (bit_offset % 64) as u32;
+    let m = mask(width);
+    let lo = words[word_idx] >> bit_in_word;
+    if bit_in_word + width <= 64 {
+        lo & m
+    } else {
+        let hi = words[word_idx + 1] << (64 - bit_in_word);
+        (lo | hi) & m
+    }
+}
+
+#[inline]
+fn set_packed(words: &mut [u64], slot: usize, width: u32, value: u64) {
+    let m = mask(width);
+    debug_assert!(value & !m == 0, "value {value} does not fit in {width} bits");
+    let bit_offset = slot as u64 * width as u64;
+    let word_idx = (bit_offset / 64) as usize;
+    let bit_in_word = (bit_offset % 64) as u32;
+    words[word_idx] &= !(m << bit_in_word);
+    words[word_idx] |= (value & m) << bit_in_word;
+    if bit_in_word + width > 64 {
+        let hi_bits = bit_in_word + width - 64;
+        let hi_mask = (1u64 << hi_bits) - 1;
+        words[word_idx + 1] &= !hi_mask;
+        words[word_idx + 1] |= value >> (64 - bit_in_word);
+    }
+}
+
+// ── PersistentFixedIntVecBuilder ────────────────────────────────────────────
+
+pub struct PersistentFixedIntVecBuilder {
+    mmap: MmapMut,
+    n: usize,
+    width: u32,
+    path: PathBuf,
+}
+
+impl PersistentFixedIntVecBuilder {
+    /// `width` (1..=64) is chosen by the caller from the actual data being
+    /// stored — see [`bit_width_for_range`] — never hardcoded. `width == 0`
+    /// is allowed (every value is 0, e.g. a whole `EliasFano` sequence
+    /// packed entirely into its high bits) — `get` always returns 0,
+    /// `set` is a no-op, no storage beyond the header.
+    pub fn new(n: usize, width: u32, path: &Path) -> io::Result<Self> {
+        assert!(width <= 64, "width must be 0..=64, got {width}");
+        let file_size = HEADER_SIZE + n_words_for(n, width) * 8;
+        let mut file = OpenOptions::new()
+            .read(true).write(true).create(true).truncate(true)
+            .open(path)?;
+        file.write_all(&MAGIC)?;
+        file.write_all(&[width as u8, 0, 0, 0])?;
+        file.write_all(&(n as u64).to_le_bytes())?;
+        file.seek(SeekFrom::Start(0))?;
+        file.set_len(file_size as u64)?;
+        let mmap = unsafe { MmapMut::map_mut(&file)? };
+        Ok(Self { mmap, n, width, path: path.to_path_buf() })
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize { self.n }
+    #[inline]
+    pub fn is_empty(&self) -> bool { self.n == 0 }
+    #[inline]
+    pub fn width(&self) -> u32 { self.width }
+
+    #[inline]
+    fn data_words_mut(&mut self) -> &mut [u64] {
+        let nw = n_words_for(self.n, self.width);
+        let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
+        unsafe { std::slice::from_raw_parts_mut(ptr, nw) }
+    }
+
+    #[inline]
+    fn data_words(&self) -> &[u64] {
+        let nw = n_words_for(self.n, self.width);
+        let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
+        unsafe { std::slice::from_raw_parts(ptr, nw) }
+    }
+
+    #[inline]
+    pub fn get(&self, slot: usize) -> u64 {
+        debug_assert!(slot < self.n);
+        get_packed(self.data_words(), slot, self.width)
+    }
+
+    #[inline]
+    pub fn set(&mut self, slot: usize, value: u64) {
+        debug_assert!(slot < self.n);
+        let width = self.width;
+        set_packed(self.data_words_mut(), slot, width, value);
+    }
+
+    pub fn close(self) -> io::Result<()> {
+        self.mmap.flush()
+    }
+
+    pub fn finish(self) -> io::Result<PersistentFixedIntVec> {
+        let path = self.path.clone();
+        self.close()?;
+        PersistentFixedIntVec::open(&path)
+    }
+}

src/obicompactvec/src/intmatrix.rs

@@ -36,14 +36,19 @@ impl ColumnarCompactIntMatrix {
         Ok(Self { cols, n: meta.n })
     }
 
+    #[inline]
     pub(crate) fn n(&self)      -> usize { self.n }
+    #[inline]
     pub(crate) fn n_cols(&self) -> usize { self.cols.len() }
+    #[inline]
     pub(crate) fn col(&self, c: usize) -> &PersistentCompactIntVec { &self.cols[c] }
 
+    #[inline]
     pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
         self.cols.iter().map(|c| c.get(slot)).collect()
     }
 
+    #[inline]
     pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
         for (c, col) in self.cols.iter().enumerate() { buf[c] = col.get(slot); }
     }
@@ -142,6 +147,7 @@ impl PackedCompactIntMatrix {
         Ok(Self { mmap, n_rows, n_cols, columns })
     }
 
+    #[inline]
     pub(crate) fn col_view(&self, c: usize) -> IntSliceView<'_> {
         let ci = &self.columns[c];
         let primary     = &self.mmap[ci.primary_start..ci.primary_start + self.n_rows];
@@ -158,10 +164,12 @@ impl PackedCompactIntMatrix {
     #[inline]
     pub(crate) fn get(&self, col: usize, slot: usize) -> u32 { self.col_view(col).get(slot) }
 
+    #[inline]
     pub(crate) fn fill_row(&self, slot: usize, buf: &mut [u32]) {
         for c in 0..self.n_cols { buf[c] = self.get(c, slot); }
     }
 
+    #[inline]
     pub(crate) fn row(&self, slot: usize) -> Box<[u32]> {
         (0..self.n_cols).map(|c| self.get(c, slot)).collect()
     }
@@ -312,13 +320,16 @@ impl PersistentCompactIntMatrix {
         ))
     }
 
+    #[inline]
     pub fn n(&self) -> usize {
         match self { Self::Columnar(m) => m.n(), Self::Packed(m) => m.n_rows }
     }
+    #[inline]
     pub fn n_cols(&self) -> usize {
         match self { Self::Columnar(m) => m.n_cols(), Self::Packed(m) => m.n_cols }
     }
 
+    #[inline]
     pub fn col(&self, c: usize) -> &PersistentCompactIntVec {
         match self {
             Self::Columnar(m) => m.col(c),
@@ -326,6 +337,7 @@ impl PersistentCompactIntMatrix {
         }
     }
 
+    #[inline]
     pub fn col_view(&self, c: usize) -> IntSliceView<'_> {
         match self {
             Self::Columnar(m) => m.col(c).view(),
@@ -340,36 +352,91 @@ impl PersistentCompactIntMatrix {
         }
     }
 
+    #[inline]
     pub fn row(&self, slot: usize) -> Box<[u32]> {
         match self { Self::Columnar(m) => m.row(slot), Self::Packed(m) => m.row(slot) }
     }
+    #[inline]
     pub fn fill_row(&self, slot: usize, buf: &mut [u32]) {
         match self { Self::Columnar(m) => m.fill_row(slot, buf), Self::Packed(m) => m.fill_row(slot, buf) }
     }
+
+    /// Extract a sub-matrix containing only the rows at `slots`.
+    ///
+    /// Returns a column-first `Vec<Vec<u32>>`: the outer `Vec` has one entry per
+    /// column, and each inner `Vec` contains the values for the requested rows
+    /// in the same order as `slots`.  Column access is sequential to maximize
+    /// cache efficiency on the underlying mmap.
+    pub fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<u32>> {
+        let n_cols = self.n_cols();
+        let mut out: Vec<Vec<u32>> = Vec::with_capacity(n_cols);
+        for c in 0..n_cols {
+            let mut col_buf = vec![0u32; slots.len()];
+            self.col_view(c).fill_batch(slots, &mut col_buf);
+            out.push(col_buf);
+        }
+        out
+    }
+
+    /// Like [`sub_matrix`](Self::sub_matrix), but fills caller-provided column
+    /// buffers to avoid allocating the outer `Vec`.
+    ///
+    /// `out` must have length `self.n_cols()`.  Each `out[c]` is cleared,
+    /// resized to `slots.len()`, and filled with the values for column `c`
+    /// in the same order as `slots`.
+    pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<u32>]) {
+        assert_eq!(out.len(), self.n_cols());
+        let n = slots.len();
+        if n == 0 {
+            for col in out.iter_mut() { col.clear(); }
+            return;
+        }
+        let mut perm: Vec<usize> = (0..n).collect();
+        perm.sort_by_key(|&i| slots[i]);
+        let sorted_slots: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
+        for (c, col) in out.iter_mut().enumerate() {
+            col.resize(n, 0);
+            let mut tmp = vec![0u32; n];
+            self.col_view(c).fill_batch_sorted(&sorted_slots, &mut tmp);
+            for (i, &orig_idx) in perm.iter().enumerate() {
+                col[orig_idx] = tmp[i];
+            }
+        }
+    }
+
+    #[inline]
     pub fn sum(&self) -> Array1<u64> {
         match self { Self::Columnar(m) => m.sum(), Self::Packed(m) => m.sum() }
     }
+    #[inline]
     pub fn count_nonzero(&self) -> Array1<u64> {
         match self { Self::Columnar(m) => m.count_nonzero(), Self::Packed(m) => m.count_nonzero() }
     }
+    #[inline]
     pub fn partial_bray_dist_matrix(&self) -> Array2<u64> {
         match self { Self::Columnar(m) => m.partial_bray_dist_matrix(), Self::Packed(m) => m.partial_bray_dist_matrix() }
     }
+    #[inline]
     pub fn partial_euclidean_dist_matrix(&self) -> Array2<f64> {
         match self { Self::Columnar(m) => m.partial_euclidean_dist_matrix(), Self::Packed(m) => m.partial_euclidean_dist_matrix() }
     }
+    #[inline]
     pub fn partial_threshold_jaccard_dist_matrix(&self, threshold: u32) -> (Array2<u64>, Array2<u64>) {
         match self { Self::Columnar(m) => m.partial_threshold_jaccard_dist_matrix(threshold), Self::Packed(m) => m.partial_threshold_jaccard_dist_matrix(threshold) }
     }
+    #[inline]
     pub fn partial_relfreq_bray_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         match self { Self::Columnar(m) => m.partial_relfreq_bray_dist_matrix(col_sums), Self::Packed(m) => m.partial_relfreq_bray_dist_matrix(col_sums) }
     }
+    #[inline]
     pub fn partial_relfreq_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         match self { Self::Columnar(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_relfreq_euclidean_dist_matrix(col_sums) }
     }
+    #[inline]
     pub fn partial_hellinger_euclidean_dist_matrix(&self, col_sums: &Array1<u64>) -> Array2<f64> {
         match self { Self::Columnar(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums), Self::Packed(m) => m.partial_hellinger_euclidean_dist_matrix(col_sums) }
     }
+    #[inline]
     pub fn append_column(dir: &Path, value_of: impl Fn(usize) -> u32) -> io::Result<()> {
         ColumnarCompactIntMatrix::append_column(dir, value_of)
     }
@@ -380,16 +447,24 @@ impl PersistentCompactIntMatrix {
 use crate::traits::{ColumnWeights, CountPartials};
 
 impl ColumnWeights for PersistentCompactIntMatrix {
+    #[inline]
     fn col_weights(&self) -> Array1<u64> { self.sum() }
+    #[inline]
     fn partial_kmer_counts(&self) -> Array1<u64> { self.count_nonzero() }
 }
 
 impl CountPartials for PersistentCompactIntMatrix {
+    #[inline]
     fn partial_bray(&self) -> Array2<u64>                                 { self.partial_bray_dist_matrix() }
+    #[inline]
     fn partial_euclidean(&self) -> Array2<f64>                            { self.partial_euclidean_dist_matrix() }
+    #[inline]
     fn partial_threshold_jaccard(&self, t: u32) -> (Array2<u64>, Array2<u64>) { self.partial_threshold_jaccard_dist_matrix(t) }
+    #[inline]
     fn partial_relfreq_bray(&self, g: &Array1<u64>) -> Array2<f64>        { self.partial_relfreq_bray_dist_matrix(g) }
+    #[inline]
     fn partial_relfreq_euclidean(&self, g: &Array1<u64>) -> Array2<f64>   { self.partial_relfreq_euclidean_dist_matrix(g) }
+    #[inline]
     fn partial_hellinger(&self, g: &Array1<u64>) -> Array2<f64>           { self.partial_hellinger_euclidean_dist_matrix(g) }
 }
 
@@ -406,7 +481,9 @@ impl PersistentCompactIntMatrixBuilder {
         fs::create_dir_all(dir)?;
         Ok(Self { dir: dir.to_path_buf(), n, n_cols: 0 })
     }
+    #[inline]
     pub fn n(&self)      -> usize { self.n }
+    #[inline]
     pub fn n_cols(&self) -> usize { self.n_cols }
     pub fn add_col(&mut self) -> io::Result<PersistentCompactIntVecBuilder> {
         let path = col_path(&self.dir, self.n_cols);

src/obicompactvec/src/lib.rs

@@ -2,28 +2,32 @@ mod bitvec;
 mod bitmatrix;
 mod builder;
 mod colgroup;
+mod eliasfano;
+mod fixedintvec;
 mod format;
+mod rankselect;
 mod intmatrix;
 mod layer_meta;
 mod meta;
 mod reader;
-mod siblingannex;
 mod tempbitvec;
 mod tempintvec;
 mod views;
 pub mod traits;
 
 pub use bitvec::{BitIter, PersistentBitVec, PersistentBitVecBuilder};
-pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, pack_bit_matrix};
+pub use fixedintvec::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
+pub use rankselect::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
+pub use eliasfano::{EliasFano, EliasFanoBuilder};
+pub use bitmatrix::{PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder, pack_bit_matrix, pack_sparse_bit_matrix};
 pub use builder::PersistentCompactIntVecBuilder;
 pub use colgroup::{ColGroup, FilterMask, MatrixGroupOps, eval_filter_mask};
 pub use intmatrix::{PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, pack_compact_int_matrix};
 pub use layer_meta::LayerMeta;
-pub use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder};
 pub use reader::{PersistentCompactIntVec, Iter as CompactIntVecIter};
 pub use tempbitvec::{TempBitVec, TempBitVecBuilder};
 pub use tempintvec::{TempCompactIntVec, TempCompactIntVecBuilder};
-pub use traits::{BitPartials, ColumnWeights, CountPartials};
+pub use traits::{BinaryMatrix, BitPartials, ColumnWeights, CountPartials};
 pub use views::{BitSliceView, BitSliceIter, IntSliceView, IntSliceViewIter};
 
 #[cfg(test)]

src/obicompactvec/src/rankselect.rs

@@ -0,0 +1,265 @@
+//! Rank/select-capable bitvector — extends the crate's existing bit-count
+//! (`count_ones`, a global reduction) with `rank1`/`rank0` (count of 1s/0s
+//! in a prefix) and `select1` (position of the k-th 1 bit). Needed by two
+//! consumers in the sparse presence-matrix design (see
+//! `docmd/architecture/siblings.md` and the sparse-matrix plan): the
+//! `is_multi` row-kind flag (needs rank, to locate a row's position within
+//! whichever of the two split arrays it belongs to) and the Elias-Fano high
+//! bits (needs select).
+//!
+//! Two-level structure, the standard succinct-bitvector approach: a
+//! cumulative rank sampled every [`BLOCK_WORDS`] words, plus a linear scan
+//! within the block (at most [`BLOCK_WORDS`] popcounts) for the remainder.
+//! `select1` binary-searches the block samples, then
+//! `common_traits::SelectInWord` locates the exact bit within the winning
+//! word.
+
+use std::fs::{File, OpenOptions};
+use std::io::{self, Seek, SeekFrom, Write as _};
+use std::path::{Path, PathBuf};
+
+use common_traits::SelectInWord;
+use memmap2::{Mmap, MmapMut};
+
+const MAGIC: [u8; 4] = *b"PRSB";
+
+/// Words per rank-sample block — 8 words = 512 bits, one cache line's
+/// worth of linear-scan work for the within-block remainder.
+const BLOCK_WORDS: usize = 8;
+
+// Header: magic(4) + _pad(4) + n(8) + total_ones(8) = 24 bytes (8-aligned).
+const HEADER_SIZE: usize = 24;
+
+#[inline]
+fn n_words(n: usize) -> usize {
+    n.div_ceil(64)
+}
+
+#[inline]
+fn n_blocks(n: usize) -> usize {
+    n_words(n).div_ceil(BLOCK_WORDS)
+}
+
+#[inline]
+fn bits_bytes(n: usize) -> usize {
+    n_words(n) * 8
+}
+
+#[inline]
+fn blocks_bytes(n: usize) -> usize {
+    n_blocks(n) * 8
+}
+
+// ── PersistentRankSelectBitVec ──────────────────────────────────────────────
+
+pub struct PersistentRankSelectBitVec {
+    mmap: Mmap,
+    n: usize,
+    total_ones: u64,
+    path: PathBuf,
+}
+
+impl PersistentRankSelectBitVec {
+    pub fn open(path: &Path) -> io::Result<Self> {
+        let mmap = unsafe { Mmap::map(&File::open(path)?)? };
+        if mmap.len() < HEADER_SIZE {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "PRSB file too short"));
+        }
+        if mmap[0..4] != MAGIC {
+            return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PRSB magic"));
+        }
+        let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
+        let total_ones = u64::from_le_bytes(mmap[16..24].try_into().unwrap());
+        Ok(Self { mmap, n, total_ones, path: path.to_path_buf() })
+    }
+
+    #[inline]
+    pub fn path(&self) -> &Path { &self.path }
+    #[inline]
+    pub fn len(&self) -> usize { self.n }
+    #[inline]
+    pub fn is_empty(&self) -> bool { self.n == 0 }
+    #[inline]
+    pub fn count_ones(&self) -> u64 { self.total_ones }
+    #[inline]
+    pub fn count_zeros(&self) -> u64 { self.n as u64 - self.total_ones }
+
+    // SAFETY: mmap is page-aligned, HEADER_SIZE=24 divisible by 8 → u64-aligned.
+    #[inline]
+    fn bit_words(&self) -> &[u64] {
+        let nw = n_words(self.n);
+        let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
+        unsafe { std::slice::from_raw_parts(ptr, nw) }
+    }
+
+    #[inline]
+    fn block_ranks(&self) -> &[u64] {
+        let nb = n_blocks(self.n);
+        let off = HEADER_SIZE + bits_bytes(self.n);
+        let ptr = self.mmap[off..].as_ptr() as *const u64;
+        unsafe { std::slice::from_raw_parts(ptr, nb) }
+    }
+
+    #[inline]
+    pub fn get(&self, pos: usize) -> bool {
+        debug_assert!(pos < self.n);
+        (self.bit_words()[pos >> 6] >> (pos & 63)) & 1 != 0
+    }
+
+    /// Number of 1 bits in `[0, pos)`.
+    pub fn rank1(&self, pos: usize) -> u64 {
+        debug_assert!(pos <= self.n);
+        if pos == 0 {
+            return 0;
+        }
+        let words = self.bit_words();
+        let block_ranks = self.block_ranks();
+        let word_idx = (pos - 1) / 64;
+        let block_idx = word_idx / BLOCK_WORDS;
+        let mut rank = block_ranks[block_idx];
+        let block_start_word = block_idx * BLOCK_WORDS;
+        for w in &words[block_start_word..word_idx] {
+            rank += w.count_ones() as u64;
+        }
+        let bit_in_word = pos - word_idx * 64;
+        let last = words[word_idx];
+        let masked = if bit_in_word >= 64 { last } else { last & ((1u64 << bit_in_word) - 1) };
+        rank += masked.count_ones() as u64;
+        rank
+    }
+
+    /// Number of 0 bits in `[0, pos)`.
+    #[inline]
+    pub fn rank0(&self, pos: usize) -> u64 {
+        pos as u64 - self.rank1(pos)
+    }
+
+    /// Position of the `k`-th (0-indexed) 1 bit. Panics if fewer than
+    /// `k + 1` ones exist.
+    pub fn select1(&self, k: u64) -> usize {
+        assert!(k < self.total_ones, "select1({k}) out of range: only {} ones", self.total_ones);
+        let block_ranks = self.block_ranks();
+        let words = self.bit_words();
+
+        // Binary search: last block whose cumulative rank is <= k.
+        let mut lo = 0usize;
+        let mut hi = block_ranks.len();
+        while lo + 1 < hi {
+            let mid = lo + (hi - lo) / 2;
+            if block_ranks[mid] <= k { lo = mid; } else { hi = mid; }
+        }
+        let block_idx = lo;
+        let mut remaining = k - block_ranks[block_idx];
+        let start_word = block_idx * BLOCK_WORDS;
+        let end_word = (start_word + BLOCK_WORDS).min(words.len());
+        for (i, &w) in words[start_word..end_word].iter().enumerate() {
+            let c = w.count_ones() as u64;
+            if remaining < c {
+                return (start_word + i) * 64 + w.select_in_word(remaining as usize);
+            }
+            remaining -= c;
+        }
+        unreachable!("select1({k}): ran off the end of its own block — rank/select index inconsistent");
+    }
+}
+
+// ── PersistentRankSelectBitVecBuilder ───────────────────────────────────────
+
+pub struct PersistentRankSelectBitVecBuilder {
+    mmap: MmapMut,
+    n: usize,
+    path: PathBuf,
+}
+
+impl PersistentRankSelectBitVecBuilder {
+    pub fn new(n: usize, path: &Path) -> io::Result<Self> {
+        let file_size = HEADER_SIZE + bits_bytes(n) + blocks_bytes(n);
+        let mut file = OpenOptions::new()
+            .read(true).write(true).create(true).truncate(true)
+            .open(path)?;
+        file.write_all(&MAGIC)?;
+        file.write_all(&[0u8; 4])?;
+        file.write_all(&(n as u64).to_le_bytes())?;
+        file.write_all(&0u64.to_le_bytes())?; // total_ones, patched in close()
+        file.seek(SeekFrom::Start(0))?;
+        file.set_len(file_size as u64)?;
+        let mmap = unsafe { MmapMut::map_mut(&file)? };
+        Ok(Self { mmap, n, path: path.to_path_buf() })
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize { self.n }
+    #[inline]
+    pub fn is_empty(&self) -> bool { self.n == 0 }
+
+    #[inline]
+    fn bit_words_mut(&mut self) -> &mut [u64] {
+        let nw = n_words(self.n);
+        let ptr = self.mmap[HEADER_SIZE..].as_mut_ptr() as *mut u64;
+        unsafe { std::slice::from_raw_parts_mut(ptr, nw) }
+    }
+
+    #[inline]
+    pub fn get(&self, pos: usize) -> bool {
+        debug_assert!(pos < self.n);
+        let nw = n_words(self.n);
+        let ptr = self.mmap[HEADER_SIZE..].as_ptr() as *const u64;
+        let words = unsafe { std::slice::from_raw_parts(ptr, nw) };
+        (words[pos >> 6] >> (pos & 63)) & 1 != 0
+    }
+
+    #[inline]
+    pub fn set(&mut self, pos: usize, value: bool) {
+        debug_assert!(pos < self.n);
+        let bit = 1u64 << (pos & 63);
+        let words = self.bit_words_mut();
+        if value {
+            words[pos >> 6] |= bit;
+        } else {
+            words[pos >> 6] &= !bit;
+        }
+    }
+
+    /// Computes the block-rank index and total-ones count from the bits
+    /// written so far, then flushes. Must run after every `set()` call —
+    /// the index is a function of the final bit pattern, not maintainable
+    /// incrementally through arbitrary overwrites.
+    pub fn close(mut self) -> io::Result<()> {
+        let nb = n_blocks(self.n);
+        let nw = n_words(self.n);
+        let mut block_ranks = vec![0u64; nb];
+        let mut running = 0u64;
+        {
+            let words = self.bit_words_mut();
+            for (block_idx, block_rank) in block_ranks.iter_mut().enumerate() {
+                *block_rank = running;
+                let start = block_idx * BLOCK_WORDS;
+                let end = (start + BLOCK_WORDS).min(nw);
+                for &w in &words[start..end] {
+                    running += w.count_ones() as u64;
+                }
+            }
+        }
+        self.mmap[16..24].copy_from_slice(&running.to_le_bytes());
+        let block_off = HEADER_SIZE + bits_bytes(self.n);
+        let block_bytes = u64_slice_to_le_bytes(&block_ranks);
+        self.mmap[block_off..block_off + block_bytes.len()].copy_from_slice(&block_bytes);
+        self.mmap.flush()
+    }
+
+    pub fn finish(self) -> io::Result<PersistentRankSelectBitVec> {
+        let path = self.path.clone();
+        self.close()?;
+        PersistentRankSelectBitVec::open(&path)
+    }
+}
+
+/// Minimal, local `u64` slice -> byte vec conversion (little-endian,
+/// matching every other on-disk format in this crate).
+fn u64_slice_to_le_bytes(values: &[u64]) -> Vec<u8> {
+    let mut out = Vec::with_capacity(values.len() * 8);
+    for &v in values {
+        out.extend_from_slice(&v.to_le_bytes());
+    }
+    out
+}

src/obicompactvec/src/reader.rs

@@ -46,10 +46,14 @@ impl PersistentCompactIntVec {
         Ok(Self { mmap, n, n_overflow, step, index, primary_offset, data_offset, path: path.to_path_buf() })
     }
 
+    #[inline]
     pub fn path(&self) -> &Path { &self.path }
+    #[inline]
     pub fn len(&self)      -> usize { self.n }
+    #[inline]
     pub fn is_empty(&self) -> bool  { self.n == 0 }
 
+    #[inline]
     pub fn get(&self, slot: usize) -> u32 {
         match self.mmap[self.primary_offset + slot] {
             255 => self.overflow_get(slot),
@@ -57,6 +61,42 @@ impl PersistentCompactIntVec {
         }
     }
 
+    /// Batch lookup: read values at `slots` in an order that minimizes
+    /// cache misses.
+    ///
+    /// The slots are sorted internally before reading so that accesses to
+    /// the underlying mmap are as sequential as possible, then the results
+    /// are reordered to match the input order.
+    pub fn get_batch(&self, slots: &[usize]) -> Vec<u32> {
+        let mut out = vec![0u32; slots.len()];
+        self.fill_batch(slots, &mut out);
+        out
+    }
+
+    /// Like [`get_batch`](Self::get_batch), but fills a caller-provided buffer.
+    pub fn fill_batch(&self, slots: &[usize], out: &mut [u32]) {
+        assert_eq!(slots.len(), out.len());
+        let n = slots.len();
+        if n == 0 { return; }
+        let mut perm: Vec<usize> = (0..n).collect();
+        perm.sort_by_key(|&i| slots[i]);
+        let sorted: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
+        let mut tmp = vec![0u32; n];
+        self.fill_batch_sorted(&sorted, &mut tmp);
+        for (i, &orig_idx) in perm.iter().enumerate() {
+            out[orig_idx] = tmp[i];
+        }
+    }
+
+    /// Fill `out` assuming `sorted_slots` is already in ascending order.
+    /// Results are written in `sorted_slots` order (no reordering).
+    pub(crate) fn fill_batch_sorted(&self, sorted_slots: &[usize], out: &mut [u32]) {
+        assert_eq!(sorted_slots.len(), out.len());
+        for (i, &slot) in sorted_slots.iter().enumerate() {
+            out[i] = self.get(slot);
+        }
+    }
+
     fn overflow_get(&self, slot: usize) -> u32 {
         let (pos_start, pos_end) = if self.step == 0 {
             (0, self.n_overflow)
@@ -91,23 +131,27 @@ impl PersistentCompactIntVec {
         u32::from_le_bytes(self.mmap[off..off + 4].try_into().unwrap())
     }
 
+    #[inline]
     pub fn sum(&self) -> u64 {
         let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
         byte_sum(primary, (0..self.n_overflow).map(|i| self.data_value(i)))
     }
 
+    #[inline]
     pub fn count_nonzero(&self) -> u64 {
         let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
         byte_count_nonzero(primary)
     }
 
     /// Lightweight zero-copy view — primary and overflow point into the mmap.
+    #[inline]
     pub fn view(&self) -> IntSliceView<'_> {
         let primary = &self.mmap[self.primary_offset..self.primary_offset + self.n];
         let overflow_raw = &self.mmap[self.data_offset..self.data_offset + self.n_overflow * OVERFLOW_ENTRY_SIZE];
         IntSliceView::new(primary, overflow_raw, self.n_overflow, self.n)
     }
 
+    #[inline]
     pub fn iter(&self) -> Iter<'_> {
         Iter { pciv: self, slot: 0, overflow_pos: 0 }
     }
@@ -220,6 +264,7 @@ impl PersistentCompactIntVec {
 impl<'a> IntoIterator for &'a PersistentCompactIntVec {
     type Item = u32;
     type IntoIter = Iter<'a>;
+    #[inline]
     fn into_iter(self) -> Iter<'a> { self.iter() }
 }
 
@@ -234,6 +279,7 @@ impl ExactSizeIterator for Iter<'_> {}
 impl Iterator for Iter<'_> {
     type Item = u32;
 
+    #[inline]
     fn next(&mut self) -> Option<u32> {
         if self.slot >= self.pciv.n { return None; }
         let v = self.pciv.mmap[self.pciv.primary_offset + self.slot];
@@ -247,6 +293,7 @@ impl Iterator for Iter<'_> {
         }
     }
 
+    #[inline]
     fn size_hint(&self) -> (usize, Option<usize>) {
         let remaining = self.pciv.n - self.slot;
         (remaining, Some(remaining))

src/obicompactvec/src/siblingannex.rs

@@ -1,245 +0,0 @@
-//! Family presence-mask annex: a compact, read-only-after-build, per-slot
-//! derived value used by the central-position SNP distance estimator (see
-//! `docmd/theory/evolutionary_distances.md`, "Step 2b" and "Definitions:
-//! family, and the canonical form of a family").
-//!
-//! One byte is stored per MPHF slot of a partition/layer, its low 4 bits
-//! encoding a **presence mask** for the slot's k-mer's "family" (the up to 4
-//! k-mers sharing the same flanks, differing only at the central base):
-//! bit `b` (`b` = 0..3, in the fixed A/C/G/T = 0/1/2/3 encoding already used
-//! for a single nucleotide) is set iff the family member whose *own* central
-//! base — in its own canonical orientation — is `b`, is observed anywhere in
-//! the current multi-genome index. This is a property of the whole index,
-//! not of any one genome.
-//!
-//! Both facts the earlier (superseded) 3-bit design stored explicitly are
-//! derived from the mask instead, not stored:
-//! - sibling count = `popcount(mask) - 1`;
-//! - minorant = regenerate the family's 4 canonical forms from the slot's
-//!   own k-mer (`CanonicalKmerOf::central_canonical_neighbors`, cheap, no
-//!   lookup), compare the raw encodings of whichever are set in the mask,
-//!   take the smallest — see `obikindex::siblings`.
-//!
-//! Mask value 0 is logically unreachable as a real result (a slot's own base
-//! is always present in its own family) and is reused as the "not yet
-//! computed" sentinel: annex files are pre-initialised to all-zero, and a
-//! real value is only ever written once, by the computation pass.
-//!
-//! Deliberately simpler than a true 4-bit pack (1 byte/slot instead of 4
-//! bits/slot): correctness and simplicity first, for a first implementation.
-//! Packing to 4 bits/slot is a pure storage-density follow-up, not a
-//! behavioural change, left for later.
-
-use std::fs::{File, OpenOptions};
-use std::io;
-use std::path::{Path, PathBuf};
-
-use memmap2::{Mmap, MmapMut};
-
-const MAGIC: [u8; 4] = *b"PSIB";
-
-// Header: magic(4) + _pad(4) + n(8) = 16 bytes. Data (1 byte/slot) follows.
-const HEADER_SIZE: usize = 16;
-
-/// A family presence mask: bit `b` set iff the member whose own canonical
-/// central base is `b` (0=A, 1=C, 2=G, 3=T) is observed in the index.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct FamilyMask(u8);
-
-impl FamilyMask {
-    /// The empty mask — never a valid *computed* result (a slot's own base
-    /// is always present in its own family) — used only to build up a mask
-    /// via repeated [`with`](Self::with) calls before storing it.
-    pub const EMPTY: FamilyMask = FamilyMask(0);
-
-    /// Set bit `base` (0=A, 1=C, 2=G, 3=T).
-    #[inline]
-    pub fn with(self, base: u8) -> Self {
-        debug_assert!(base < 4, "base out of range: {base}");
-        FamilyMask(self.0 | (1 << base))
-    }
-
-    /// Is the member with central base `base` (0..3) present?
-    #[inline]
-    pub fn has(self, base: u8) -> bool {
-        debug_assert!(base < 4, "base out of range: {base}");
-        self.0 & (1 << base) != 0
-    }
-
-    /// Number of family members observed anywhere in the index (1..=4).
-    #[inline]
-    pub fn family_size(self) -> u32 {
-        self.0.count_ones()
-    }
-
-    /// Number of *other* members observed (0..=3) — `family_size() - 1`.
-    #[inline]
-    pub fn siblings(self) -> u32 {
-        self.family_size() - 1
-    }
-
-    /// Raw bitmask (bit `b` = base `b` present) — for callers that build up
-    /// a mask via their own bit operations (e.g. concurrently, via an
-    /// `AtomicU8`) and only need the `FamilyMask` wrapper at the end.
-    #[inline]
-    pub fn bits(self) -> u8 {
-        self.0
-    }
-
-    /// Construct from a raw bitmask (only the low 4 bits are kept).
-    #[inline]
-    pub fn from_bits(bits: u8) -> Self {
-        FamilyMask(bits & 0b1111)
-    }
-
-    #[inline]
-    fn encode(self) -> u8 {
-        self.0
-    }
-
-    #[inline]
-    fn decode(byte: u8) -> Option<Self> {
-        if byte == 0 {
-            // Unreachable for a real result — reserved as the "not yet
-            // computed" sentinel.
-            return None;
-        }
-        Some(FamilyMask(byte & 0b1111))
-    }
-}
-
-// ── SiblingAnnex (reader) ───────────────────────────────────────────────────
-
-pub struct SiblingAnnex {
-    mmap: Mmap,
-    n: usize,
-    path: PathBuf,
-}
-
-impl SiblingAnnex {
-    pub fn open(path: &Path) -> io::Result<Self> {
-        let mmap = unsafe { Mmap::map(&File::open(path)?)? };
-        if mmap.len() < HEADER_SIZE {
-            return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file too short"));
-        }
-        if mmap[0..4] != MAGIC {
-            return Err(io::Error::new(io::ErrorKind::InvalidData, "bad PSIB magic"));
-        }
-        let n = u64::from_le_bytes(mmap[8..16].try_into().unwrap()) as usize;
-        if mmap.len() < HEADER_SIZE + n {
-            return Err(io::Error::new(io::ErrorKind::InvalidData, "PSIB file truncated"));
-        }
-        Ok(Self { mmap, n, path: path.to_path_buf() })
-    }
-
-    pub fn path(&self) -> &Path { &self.path }
-    pub fn len(&self) -> usize { self.n }
-    pub fn is_empty(&self) -> bool { self.n == 0 }
-
-    /// `None` means the slot has not (yet) been computed — see module docs.
-    pub fn get(&self, slot: usize) -> Option<FamilyMask> {
-        FamilyMask::decode(self.mmap[HEADER_SIZE + slot])
-    }
-}
-
-// ── SiblingAnnexBuilder (writer) ────────────────────────────────────────────
-
-pub struct SiblingAnnexBuilder {
-    mmap: MmapMut,
-    n: usize,
-    path: PathBuf,
-}
-
-impl SiblingAnnexBuilder {
-    /// Create a new annex of `n` slots at `path`, pre-initialised to the
-    /// "not yet computed" sentinel (all-zero).
-    pub fn new(n: usize, path: &Path) -> io::Result<Self> {
-        let file_size = HEADER_SIZE + n;
-        let file = OpenOptions::new()
-            .read(true).write(true).create(true).truncate(true)
-            .open(path)?;
-        file.set_len(file_size as u64)?;
-        let mut mmap = unsafe { MmapMut::map_mut(&file)? };
-        mmap[0..4].copy_from_slice(&MAGIC);
-        mmap[4..8].copy_from_slice(&[0u8; 4]);
-        mmap[8..16].copy_from_slice(&(n as u64).to_le_bytes());
-        // Data region left at 0 by `set_len`/mmap — the sentinel value.
-        Ok(Self { mmap, n, path: path.to_path_buf() })
-    }
-
-    pub fn len(&self) -> usize { self.n }
-    pub fn is_empty(&self) -> bool { self.n == 0 }
-
-    pub fn get(&self, slot: usize) -> Option<FamilyMask> {
-        FamilyMask::decode(self.mmap[HEADER_SIZE + slot])
-    }
-
-    pub fn set(&mut self, slot: usize, mask: FamilyMask) {
-        // Redundant concurrent writes from independent recomputation paths
-        // converge to the same encoded byte for a given slot, so a plain
-        // store here is safe even without external synchronisation, as long
-        // as the byte write itself is atomic (true for a single aligned
-        // byte on every platform this project targets).
-        self.mmap[HEADER_SIZE + slot] = mask.encode();
-    }
-
-    pub fn close(self) -> io::Result<()> { self.mmap.flush() }
-
-    pub fn finish(self) -> io::Result<SiblingAnnex> {
-        let path = self.path.clone();
-        self.close()?;
-        SiblingAnnex::open(&path)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tempfile::tempdir;
-
-    #[test]
-    fn sentinel_is_zero_and_unset_slots_read_as_uncomputed() {
-        let dir = tempdir().unwrap();
-        let path = dir.path().join("test.psib");
-        let builder = SiblingAnnexBuilder::new(4, &path).unwrap();
-        for slot in 0..4 {
-            assert_eq!(builder.get(slot), None);
-        }
-        builder.close().unwrap();
-    }
-
-    #[test]
-    fn roundtrip_all_valid_masks() {
-        let dir = tempdir().unwrap();
-        let path = dir.path().join("test.psib");
-        let mut builder = SiblingAnnexBuilder::new(4, &path).unwrap();
-
-        let masks = [
-            FamilyMask::EMPTY.with(0),                   // just A: family size 1
-            FamilyMask::EMPTY.with(0).with(3),            // A + T: size 2
-            FamilyMask::EMPTY.with(1).with(2).with(3),    // C+G+T: size 3
-            FamilyMask::EMPTY.with(0).with(1).with(2).with(3), // all 4
-        ];
-        for (slot, mask) in masks.iter().enumerate() {
-            builder.set(slot, *mask);
-        }
-        let annex = builder.finish().unwrap();
-        for (slot, mask) in masks.iter().enumerate() {
-            assert_eq!(annex.get(slot), Some(*mask));
-        }
-        assert_eq!(annex.get(0).unwrap().siblings(), 0);
-        assert_eq!(annex.get(1).unwrap().siblings(), 1);
-        assert_eq!(annex.get(2).unwrap().siblings(), 2);
-        assert_eq!(annex.get(3).unwrap().siblings(), 3);
-        assert_eq!(annex.get(3).unwrap().family_size(), 4);
-    }
-
-    #[test]
-    fn has_reflects_individual_bits() {
-        let mask = FamilyMask::EMPTY.with(0).with(2);
-        assert!(mask.has(0));
-        assert!(!mask.has(1));
-        assert!(mask.has(2));
-        assert!(!mask.has(3));
-    }
-}

src/obicompactvec/src/tempbitvec.rs

@@ -21,21 +21,27 @@ impl TempBitVec {
         PersistentBitVec::open(path)
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         self.vec.len()
     }
+    #[inline]
     pub fn is_empty(&self) -> bool {
         self.vec.is_empty()
     }
+    #[inline]
     pub fn get(&self, slot: usize) -> bool {
         self.vec.get(slot)
     }
+    #[inline]
     pub fn count_ones(&self) -> u64 {
         self.vec.count_ones()
     }
+    #[inline]
     pub fn view(&self) -> BitSliceView<'_> {
         self.vec.view()
     }
+    #[inline]
     pub fn iter(&self) -> BitSliceIter<'_> {
         self.view().iter()
     }
@@ -69,42 +75,52 @@ impl TempBitVecBuilder {
         Ok(TempBitVec { vec, _temp: temp })
     }
 
+    #[inline]
     pub fn set(&mut self, slot: usize, value: bool) {
         self.builder.set(slot, value);
     }
 
+    #[inline]
     pub fn view(&self) -> BitSliceView<'_> {
         self.builder.view()
     }
 
+    #[inline]
     pub fn or(&mut self, other: BitSliceView<'_>) {
         self.builder.or(other);
     }
 
+    #[inline]
     pub fn and(&mut self, other: BitSliceView<'_>) {
         self.builder.and(other);
     }
 
+    #[inline]
     pub fn xor(&mut self, other: BitSliceView<'_>) {
         self.builder.xor(other);
     }
 
+    #[inline]
     pub fn not(&mut self) {
         self.builder.not();
     }
 
+    #[inline]
     pub fn copy_from(&mut self, src: BitSliceView<'_>) {
         self.builder.copy_from(src);
     }
 
+    #[inline]
     pub fn or_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
         self.builder.or_where(col, pred);
     }
 
+    #[inline]
     pub fn and_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
         self.builder.and_where(col, pred);
     }
 
+    #[inline]
     pub fn xor_where(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
         self.builder.xor_where(col, pred);
     }

src/obicompactvec/src/tempintvec.rs

@@ -22,11 +22,17 @@ impl TempCompactIntVec {
         PersistentCompactIntVec::open(path)
     }
 
+    #[inline]
     pub fn len(&self)      -> usize { self.vec.len() }
+    #[inline]
     pub fn is_empty(&self) -> bool  { self.vec.is_empty() }
+    #[inline]
     pub fn get(&self, slot: usize) -> u32  { self.vec.get(slot) }
+    #[inline]
     pub fn sum(&self)      -> u64   { self.vec.sum() }
+    #[inline]
     pub fn view(&self)     -> IntSliceView<'_> { self.vec.view() }
+    #[inline]
     pub fn iter(&self) -> crate::reader::Iter<'_> { self.vec.iter() }
 }
 
@@ -51,39 +57,53 @@ impl TempCompactIntVecBuilder {
         Ok(TempCompactIntVec { vec, _temp: temp })
     }
 
+    #[inline]
     pub fn n(&self) -> usize { self.builder.len() }
 
+    #[inline]
     pub fn set(&mut self, slot: usize, value: u32) { self.builder.set(slot, value); }
+    #[inline]
     pub fn get(&self, slot: usize) -> u32           { self.builder.get(slot) }
 
+    #[inline]
     pub fn primary_bytes(&self)         -> &[u8]      { self.builder.primary_bytes() }
+    #[inline]
     pub fn primary_bytes_mut(&mut self) -> &mut [u8]  { self.builder.primary_bytes_mut() }
 
+    #[inline]
     pub fn inc_present(&mut self, col: BitSliceView<'_>) {
         self.builder.inc_present(col);
     }
 
+    #[inline]
     pub fn inc_present_fast(&mut self, col: BitSliceView<'_>) {
         self.builder.inc_present_fast(col);
     }
 
+    #[inline]
     pub fn inc_predicate(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
         self.builder.inc_predicate(col, pred);
     }
 
+    #[inline]
     pub fn inc_predicate_fast(&mut self, col: IntSliceView<'_>, pred: impl Fn(u32) -> bool) {
         self.builder.inc_predicate_fast(col, pred);
     }
 
+    #[inline]
     pub fn add(&mut self, other: IntSliceView<'_>) {
         self.builder.add(other);
     }
 
+    #[inline]
     pub fn mask_with(&mut self, mask: BitSliceView<'_>) {
         self.builder.mask_with(mask);
     }
 
+    #[inline]
     pub fn min(&mut self, other: IntSliceView<'_>)  { self.builder.min(other); }
+    #[inline]
     pub fn max(&mut self, other: IntSliceView<'_>)  { self.builder.max(other); }
+    #[inline]
     pub fn diff(&mut self, other: IntSliceView<'_>) { self.builder.diff(other); }
 }

src/obicompactvec/src/tests/bitvec.rs

@@ -214,3 +214,66 @@ fn hamming_dist_basic() {
     let (_db, rb) = make_bv(&[true,  false, true,  true]);
     assert_eq!(ra.hamming_dist(&rb), 2);
 }
+
+// ── get_batch tests ────────────────────────────────────────────────────────────
+
+#[test]
+fn bitvec_get_batch_in_order() {
+    let bits = vec![true, false, true, false, true];
+    let (_dir, r) = make_bv(&bits);
+    let got = r.get_batch(&[0, 1, 2, 3, 4]);
+    assert_eq!(got, bits);
+}
+
+#[test]
+fn bitvec_get_batch_out_of_order() {
+    let bits = vec![true, false, true, false, true];
+    let (_dir, r) = make_bv(&bits);
+    let got = r.get_batch(&[3, 0, 4, 1]);
+    assert_eq!(got, vec![false, true, true, false]);
+}
+
+#[test]
+fn bitvec_get_batch_with_duplicates() {
+    let bits = vec![true, false, true];
+    let (_dir, r) = make_bv(&bits);
+    let got = r.get_batch(&[0, 2, 0, 1]);
+    assert_eq!(got, vec![true, true, true, false]);
+}
+
+#[test]
+fn bitvec_get_batch_empty() {
+    let (_dir, r) = make_bv(&[true, false]);
+    let got = r.get_batch(&[]);
+    assert!(got.is_empty());
+}
+
+#[test]
+fn bitvec_get_batch_out_of_bounds_returns_false() {
+    // PersistentBitVec::get does NOT bounds-check — out-of-range slots read
+    // whatever bit happens to be in the mmap (zero-initialised, so false).
+    // This is a known gap; get_batch inherits it.
+    let (_dir, r) = make_bv(&[true, false]);
+    let got = r.get_batch(&[0, 2]);
+    assert_eq!(got, vec![true, false]);
+}
+
+// BitSliceView get_batch (same logic, exercised through the view)
+#[test]
+fn bitslice_view_get_batch() {
+    let bits = vec![true, false, true, false, true];
+    let (_dir, r) = make_bv(&bits);
+    let view = r.view();
+    assert_eq!(view.get_batch(&[0, 1, 2]), vec![true, false, true]);
+    assert_eq!(view.get_batch(&[4, 3]), vec![true, false]);
+}
+
+#[test]
+fn bitslice_view_get_batch_out_of_bounds_returns_false() {
+    // BitSliceView::get does NOT bounds-check either.
+    let bits = vec![true, false];
+    let (_dir, r) = make_bv(&bits);
+    let view = r.view();
+    let got = view.get_batch(&[0, 2]);
+    assert_eq!(got, vec![true, false]);
+}

src/obicompactvec/src/tests/eliasfano.rs

@@ -0,0 +1,114 @@
+use tempfile::tempdir;
+
+use crate::{EliasFano, EliasFanoBuilder};
+
+fn build(values: &[u64], universe: u64) -> (tempfile::TempDir, EliasFano) {
+    let dir = tempdir().unwrap();
+    let base = dir.path().join("test.ef");
+    let mut b = EliasFanoBuilder::new(values.len(), universe, &base).unwrap();
+    for &v in values {
+        b.push(v);
+    }
+    let ef = b.finish(&base).unwrap();
+    (dir, ef)
+}
+
+#[test]
+fn empty() {
+    let (_dir, ef) = build(&[], 1000);
+    assert_eq!(ef.len(), 0);
+    assert!(ef.is_empty());
+}
+
+#[test]
+fn small_monotone_sequence() {
+    let values = [0u64, 3, 3, 7, 42, 42, 42, 100];
+    let (_dir, ef) = build(&values, 1000);
+    assert_eq!(ef.len(), values.len());
+    for (i, &expected) in values.iter().enumerate() {
+        assert_eq!(ef.get(i), expected, "index {i}");
+    }
+}
+
+#[test]
+fn all_equal_values() {
+    let values = [5u64; 20];
+    let (_dir, ef) = build(&values, 100);
+    for i in 0..values.len() {
+        assert_eq!(ef.get(i), 5);
+    }
+}
+
+#[test]
+fn strictly_increasing() {
+    let values: Vec<u64> = (0..500).map(|i| i * 3).collect();
+    let universe = values.last().unwrap() + 1;
+    let (_dir, ef) = build(&values, universe);
+    for (i, &expected) in values.iter().enumerate() {
+        assert_eq!(ef.get(i), expected, "index {i}");
+    }
+}
+
+#[test]
+fn realistic_byte_offsets() {
+    // Simulates the actual use case: cumulative byte offsets into a
+    // varint blob, one entry per distinct multi-genome set, strictly
+    // increasing by a small, variable amount each time (1-2 bytes/index,
+    // several indices per set).
+    let mut offset = 0u64;
+    let mut values = Vec::new();
+    let mut rng_state = 12345u64;
+    for _ in 0..10_000 {
+        values.push(offset);
+        // simple xorshift for a deterministic, dependency-free "random" gap
+        rng_state ^= rng_state << 13;
+        rng_state ^= rng_state >> 7;
+        rng_state ^= rng_state << 17;
+        offset += 1 + (rng_state % 8); // 1..=8 bytes per set, realistic for n_cols=91
+    }
+    let universe = offset + 1;
+    let (_dir, ef) = build(&values, universe);
+    for (i, &expected) in values.iter().enumerate() {
+        assert_eq!(ef.get(i), expected, "index {i}");
+    }
+}
+
+#[test]
+fn universe_smaller_than_n_gives_zero_width_low() {
+    // n=100 values drawn from a universe of only 10 — low_bits_width
+    // should come out 0 (falls back to pure unary/high-bits encoding).
+    let values: Vec<u64> = (0..100).map(|i| i / 10).collect(); // 0,0,..,0,1,1,..,9,9
+    let (_dir, ef) = build(&values, 10);
+    for (i, &expected) in values.iter().enumerate() {
+        assert_eq!(ef.get(i), expected, "index {i}");
+    }
+}
+
+#[test]
+#[should_panic(expected = "monotonicity")]
+fn push_out_of_order_panics() {
+    let dir = tempdir().unwrap();
+    let base = dir.path().join("test.ef");
+    let mut b = EliasFanoBuilder::new(3, 100, &base).unwrap();
+    b.push(10);
+    b.push(5); // decreasing — must panic
+}
+
+#[test]
+fn reopen_after_close_matches_original() {
+    let values: Vec<u64> = (0..2000).map(|i| i * 5 + (i % 3)).collect();
+    let universe = *values.last().unwrap() + 1;
+    let dir = tempdir().unwrap();
+    let base = dir.path().join("test.ef");
+    {
+        let mut b = EliasFanoBuilder::new(values.len(), universe, &base).unwrap();
+        for &v in &values {
+            b.push(v);
+        }
+        b.close().unwrap();
+    } // builder + mmaps fully dropped here
+    let ef = EliasFano::open(&base).unwrap();
+    for (i, &expected) in values.iter().enumerate() {
+        assert_eq!(ef.get(i), expected, "index {i}");
+    }
+}

src/obicompactvec/src/tests/fixedintvec.rs

@@ -0,0 +1,112 @@
+use tempfile::tempdir;
+
+use crate::{PersistentFixedIntVec, PersistentFixedIntVecBuilder, bit_width_for_range};
+
+#[test]
+fn bit_width_for_range_matches_expectations() {
+    assert_eq!(bit_width_for_range(0), 1);
+    assert_eq!(bit_width_for_range(1), 1);
+    assert_eq!(bit_width_for_range(2), 1);
+    assert_eq!(bit_width_for_range(3), 2);
+    assert_eq!(bit_width_for_range(4), 2);
+    assert_eq!(bit_width_for_range(91), 7);
+    assert_eq!(bit_width_for_range(128), 7);
+    assert_eq!(bit_width_for_range(129), 8);
+    assert_eq!(bit_width_for_range(460_591), 19);
+}
+
+fn roundtrip(width: u32, values: &[u64]) -> Vec<u64> {
+    let dir = tempdir().unwrap();
+    let path = dir.path().join("test.pfiv");
+    let mut b = PersistentFixedIntVecBuilder::new(values.len(), width, &path).unwrap();
+    for (i, &v) in values.iter().enumerate() {
+        b.set(i, v);
+    }
+    b.close().unwrap();
+    let r = PersistentFixedIntVec::open(&path).unwrap();
+    assert_eq!(r.len(), values.len());
+    assert_eq!(r.width(), width);
+    (0..values.len()).map(|s| r.get(s)).collect()
+}
+
+#[test]
+fn width_1_roundtrip() {
+    let values = [0u64, 1, 1, 0, 1];
+    assert_eq!(roundtrip(1, &values), values);
+}
+
+#[test]
+fn width_7_roundtrip_genome_indices() {
+    // 91-genome-scale values, width=7 (matches bit_width_for_range(91)).
+    let values: Vec<u64> = (0..91).collect();
+    assert_eq!(roundtrip(7, &values), values);
+}
+
+#[test]
+fn width_19_roundtrip_dict_ids() {
+    // Values crossing many word boundaries at a non-power-of-two width —
+    // exactly the case that breaks a naive byte/word-aligned packer.
+    let values: Vec<u64> = (0..2000).map(|i| (i * 37) % 460_591).collect();
+    assert_eq!(roundtrip(19, &values), values);
+}
+
+#[test]
+fn width_64_roundtrip() {
+    let values = [0u64, u64::MAX, 1, u64::MAX - 1, 1 << 40];
+    assert_eq!(roundtrip(64, &values), values);
+}
+
+#[test]
+fn values_spanning_word_boundary() {
+    // width=19: slot 3's bits start at bit 57, spans into the next word.
+    let values: Vec<u64> = vec![524_287, 0, 0, 500_000, 1];
+    assert_eq!(roundtrip(19, &values), values);
+}
+
+#[test]
+fn mutation_via_set_overwrites() {
+    let dir = tempdir().unwrap();
+    let path = dir.path().join("test.pfiv");
+    let mut b = PersistentFixedIntVecBuilder::new(3, 10, &path).unwrap();
+    b.set(0, 5);
+    b.set(1, 1000);
+    b.set(2, 3);
+    b.set(1, 42); // overwrite
+    b.close().unwrap();
+    let r = PersistentFixedIntVec::open(&path).unwrap();
+    assert_eq!(r.get(0), 5);
+    assert_eq!(r.get(1), 42);
+    assert_eq!(r.get(2), 3);
+}
+
+#[test]
+fn all_zero_by_default() {
+    let dir = tempdir().unwrap();
+    let path = dir.path().join("test.pfiv");
+    let b = PersistentFixedIntVecBuilder::new(50, 13, &path).unwrap();
+    b.close().unwrap();
+    let r = PersistentFixedIntVec::open(&path).unwrap();
+    for i in 0..50 {
+        assert_eq!(r.get(i), 0, "slot {i}");
+    }
+}
+
+#[test]
+fn reopen_after_close_matches_original() {
+    // Explicit disk round-trip: drop the builder/mmap entirely, reopen
+    // from a fresh `Mmap::map` call, not the same in-memory handle.
+    let values: Vec<u64> = (0..5000).map(|i| (i * 97) % 8192).collect();
+    let dir = tempdir().unwrap();
+    let path = dir.path().join("test.pfiv");
+    {
+        let mut b = PersistentFixedIntVecBuilder::new(values.len(), 13, &path).unwrap();
+        for (i, &v) in values.iter().enumerate() {
+            b.set(i, v);
+        }
+        b.close().unwrap();
+    } // builder + mmap fully dropped here
+    let r = PersistentFixedIntVec::open(&path).unwrap();
+    for (i, &expected) in values.iter().enumerate() {
+        assert_eq!(r.get(i), expected, "slot {i}");
+    }
+}

src/obicompactvec/src/tests/intmatrix.rs

@@ -1,6 +1,6 @@
 use tempfile::tempdir;
 
-use crate::{pack_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder};
+use crate::{pack_compact_int_matrix, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, PersistentCompactIntVec, PersistentCompactIntVecBuilder, IntSliceView};
 use crate::traits::CountPartials;
 
 fn make_matrix(cols: &[&[u32]]) -> (tempfile::TempDir, PersistentCompactIntMatrix) {
@@ -320,3 +320,63 @@ fn partial_relfreq_bray_additive_across_split() {
         }
     }
 }
+
+// ── get_batch tests ────────────────────────────────────────────────────────────
+
+fn make_pciv(counts: &[u32]) -> (tempfile::TempDir, PersistentCompactIntVec) {
+    let dir  = tempdir().unwrap();
+    let path = dir.path().join("c.pciv");
+    let mut b = PersistentCompactIntVecBuilder::new(counts.len(), &path).unwrap();
+    for (i, &v) in counts.iter().enumerate() { b.set(i, v); }
+    b.close().unwrap();
+    let r = PersistentCompactIntVec::open(&path).unwrap();
+    (dir, r)
+}
+
+#[test]
+fn pciv_get_batch_in_order() {
+    let counts = vec![10u32, 255, 300, 1000];
+    let (_dir, v) = make_pciv(&counts);
+    let got = v.get_batch(&[0, 1, 2, 3]);
+    assert_eq!(got, counts);
+}
+
+#[test]
+fn pciv_get_batch_out_of_order() {
+    let counts = vec![10u32, 255, 300, 1000];
+    let (_dir, v) = make_pciv(&counts);
+    let got = v.get_batch(&[3, 0, 2, 1]);
+    assert_eq!(got, vec![1000, 10, 300, 255]);
+}
+
+#[test]
+fn pciv_get_batch_with_duplicates() {
+    let counts = vec![10u32, 255, 300];
+    let (_dir, v) = make_pciv(&counts);
+    let got = v.get_batch(&[0, 2, 0, 1]);
+    assert_eq!(got, vec![10, 300, 10, 255]);
+}
+
+#[test]
+fn pciv_get_batch_empty() {
+    let (_dir, v) = make_pciv(&[10u32, 20]);
+    let got: Vec<u32> = v.get_batch(&[]);
+    assert!(got.is_empty());
+}
+
+#[test]
+fn pciv_get_batch_out_of_bounds_panics() {
+    let (_dir, v) = make_pciv(&[10u32, 20]);
+    let result = std::panic::catch_unwind(|| v.get_batch(&[0, 2]));
+    assert!(result.is_err(), "get_batch should panic on out-of-bounds slot");
+}
+
+// IntSliceView get_batch (same logic, exercised through the view)
+#[test]
+fn intslice_view_get_batch() {
+    let counts = vec![10u32, 255, 300, 1000];
+    let (_dir, v) = make_pciv(&counts);
+    let view = v.view();
+    assert_eq!(view.get_batch(&[0, 1, 2]), vec![10, 255, 300]);
+    assert_eq!(view.get_batch(&[3, 1]), vec![1000, 255]);
+}

src/obicompactvec/src/tests/mod.rs

@@ -1,7 +1,11 @@
 mod bitmatrix;
 mod bitvec;
 mod colgroup;
+mod eliasfano;
+mod fixedintvec;
 mod intmatrix;
+mod rankselect;
+mod sparse;
 
 use tempfile::tempdir;
 

src/obicompactvec/src/tests/rankselect.rs

@@ -0,0 +1,167 @@
+use tempfile::tempdir;
+
+use crate::{PersistentRankSelectBitVec, PersistentRankSelectBitVecBuilder};
+
+fn build(bits: &[bool]) -> (tempfile::TempDir, PersistentRankSelectBitVec) {
+    let dir = tempdir().unwrap();
+    let path = dir.path().join("test.prsb");
+    let mut b = PersistentRankSelectBitVecBuilder::new(bits.len(), &path).unwrap();
+    for (i, &v) in bits.iter().enumerate() {
+        b.set(i, v);
+    }
+    b.close().unwrap();
+    let r = PersistentRankSelectBitVec::open(&path).unwrap();
+    (dir, r)
+}
+
+fn naive_rank1(bits: &[bool], pos: usize) -> u64 {
+    bits[..pos].iter().filter(|&&b| b).count() as u64
+}
+
+fn naive_select1(bits: &[bool], k: u64) -> usize {
+    bits.iter().enumerate().filter(|&(_, &b)| b).nth(k as usize).unwrap().0
+}
+
+#[test]
+fn get_matches_input() {
+    let bits = [true, false, true, true, false, false, true];
+    let (_dir, r) = build(&bits);
+    for (i, &expected) in bits.iter().enumerate() {
+        assert_eq!(r.get(i), expected, "bit {i}");
+    }
+}
+
+#[test]
+fn count_ones_matches_naive() {
+    let bits: Vec<bool> = (0..1000).map(|i| i % 3 == 0).collect();
+    let (_dir, r) = build(&bits);
+    assert_eq!(r.count_ones(), bits.iter().filter(|&&b| b).count() as u64);
+    assert_eq!(r.count_zeros(), bits.iter().filter(|&&b| !b).count() as u64);
+}
+
+#[test]
+fn rank1_matches_naive_small() {
+    let bits = [true, false, true, true, false, false, true, true, false, true];
+    let (_dir, r) = build(&bits);
+    for pos in 0..=bits.len() {
+        assert_eq!(r.rank1(pos), naive_rank1(&bits, pos), "rank1({pos})");
+    }
+}
+
+#[test]
+fn rank1_matches_naive_multi_block() {
+    // BLOCK_WORDS=8 words=512 bits — exercise several blocks, an odd
+    // total length, and a non-uniform bit pattern.
+    let n = 3000;
+    let bits: Vec<bool> = (0..n).map(|i| (i * 7 + 3) % 11 == 0).collect();
+    let (_dir, r) = build(&bits);
+    for pos in (0..=n).step_by(37) {
+        assert_eq!(r.rank1(pos), naive_rank1(&bits, pos), "rank1({pos})");
+    }
+    assert_eq!(r.rank1(n), naive_rank1(&bits, n));
+}
+
+#[test]
+fn rank0_is_complement_of_rank1() {
+    let bits: Vec<bool> = (0..777).map(|i| i % 5 < 2).collect();
+    let (_dir, r) = build(&bits);
+    for pos in (0..=bits.len()).step_by(13) {
+        assert_eq!(r.rank0(pos), pos as u64 - r.rank1(pos));
+    }
+}
+
+#[test]
+fn select1_matches_naive_small() {
+    let bits = [true, false, true, true, false, false, true, true, false, true];
+    let (_dir, r) = build(&bits);
+    let n_ones = bits.iter().filter(|&&b| b).count() as u64;
+    for k in 0..n_ones {
+        assert_eq!(r.select1(k), naive_select1(&bits, k), "select1({k})");
+    }
+}
+
+#[test]
+fn select1_matches_naive_multi_block() {
+    let n = 3000;
+    let bits: Vec<bool> = (0..n).map(|i| (i * 13 + 5) % 17 == 0).collect();
+    let (_dir, r) = build(&bits);
+    let n_ones = bits.iter().filter(|&&b| b).count() as u64;
+    for k in (0..n_ones).step_by(23) {
+        assert_eq!(r.select1(k), naive_select1(&bits, k), "select1({k})");
+    }
+}
+
+#[test]
+fn rank_select_round_trip() {
+    // For every one-bit's position p, select1(rank1(p)) == p.
+    let n = 2000;
+    let bits: Vec<bool> = (0..n).map(|i| (i * 31 + 1) % 9 == 0).collect();
+    let (_dir, r) = build(&bits);
+    for (p, &is_one) in bits.iter().enumerate() {
+        if is_one {
+            let k = r.rank1(p);
+            assert_eq!(r.select1(k), p, "position {p}, rank {k}");
+        }
+    }
+}
+
+#[test]
+#[should_panic]
+fn select1_out_of_range_panics() {
+    let bits = [true, false, false];
+    let (_dir, r) = build(&bits);
+    r.select1(1); // only one 1-bit (k=0 valid), k=1 must panic
+}
+
+#[test]
+fn all_zeros() {
+    let bits = vec![false; 200];
+    let (_dir, r) = build(&bits);
+    assert_eq!(r.count_ones(), 0);
+    assert_eq!(r.rank1(200), 0);
+    assert_eq!(r.rank0(200), 200);
+}
+
+#[test]
+fn all_ones() {
+    let bits = vec![true; 200];
+    let (_dir, r) = build(&bits);
+    assert_eq!(r.count_ones(), 200);
+    assert_eq!(r.rank1(200), 200);
+    for k in 0..200 {
+        assert_eq!(r.select1(k), k as usize);
+    }
+}
+
+#[test]
+fn reopen_after_close_matches_original() {
+    let n = 5000;
+    let bits: Vec<bool> = (0..n).map(|i| (i * 41 + 7) % 13 == 0).collect();
+    let dir = tempdir().unwrap();
+    let path = dir.path().join("test.prsb");
+    {
+        let mut b = PersistentRankSelectBitVecBuilder::new(n, &path).unwrap();
+        for (i, &v) in bits.iter().enumerate() {
+            b.set(i, v);
+        }
+        b.close().unwrap();
+    } // builder + mmap fully dropped here
+    let r = PersistentRankSelectBitVec::open(&path).unwrap();
+    assert_eq!(r.count_ones(), bits.iter().filter(|&&b| b).count() as u64);
+    for pos in (0..=n).step_by(17) {
+        assert_eq!(r.rank1(pos), naive_rank1(&bits, pos), "rank1({pos})");
+    }
+}
+
+#[test]
+fn select1_run_of_ten_then_gap_then_run() {
+    // Mirrors the exact bit pattern from the failing EliasFano case:
+    // positions 0..=9 set, 10 clear, 11..=20 set.
+    let mut bits = vec![false; 30];
+    for p in 0..=9 { bits[p] = true; }
+    for p in 11..=20 { bits[p] = true; }
+    let (_dir, r) = build(&bits);
+    assert_eq!(r.select1(9), 9, "select1(9)");
+    assert_eq!(r.select1(10), 11, "select1(10)");
+    assert_eq!(r.select1(11), 12, "select1(11)");
+}

src/obicompactvec/src/tests/sparse.rs

@@ -0,0 +1,251 @@
+use tempfile::tempdir;
+
+use crate::{BinaryMatrix, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentSparseBitMatrix, PersistentSparseBitMatrixBuilder};
+
+/// Builds a dense `PersistentBitMatrix` from column-major `bool` data —
+/// mirrors `tests/bitmatrix.rs`'s own `make_matrix` helper.
+fn make_dense(cols: &[&[bool]]) -> (tempfile::TempDir, PersistentBitMatrix) {
+    let n = cols.first().map_or(0, |c| c.len());
+    let dir = tempdir().unwrap();
+    let presence = dir.path().join("presence");
+    let mut b = PersistentBitMatrixBuilder::new(n, &presence).unwrap();
+    for &col in cols {
+        let mut cb = b.add_col().unwrap();
+        for (slot, &v) in col.iter().enumerate() {
+            cb.set(slot, v);
+        }
+        cb.close().unwrap();
+    }
+    b.close().unwrap();
+    let m = PersistentBitMatrix::open(dir.path()).unwrap();
+    (dir, m)
+}
+
+/// Builds a sparse matrix directly from row-major `bool` data (one slice
+/// per row, `n_cols` bools each) — the natural input shape for this type.
+fn make_sparse(rows: &[&[bool]], n_cols: usize) -> (tempfile::TempDir, PersistentSparseBitMatrix) {
+    let dir = tempdir().unwrap();
+    let sparse_dir = dir.path().join("sparse");
+    let mut b = PersistentSparseBitMatrixBuilder::new(rows.len(), n_cols, &sparse_dir).unwrap();
+    let mut genomes = Vec::new();
+    for row in rows {
+        genomes.clear();
+        genomes.extend((0..n_cols).filter(|&c| row[c]).map(|c| c as u32));
+        b.push_row(&genomes);
+    }
+    let m = b.finish().unwrap();
+    (dir, m)
+}
+
+fn row_as_bool(m: &PersistentSparseBitMatrix, slot: usize) -> Vec<bool> {
+    m.row(slot).to_vec()
+}
+
+#[test]
+fn basic_roundtrip_singletons_and_multi() {
+    // 5 rows, 4 genomes: mix of singleton, multi-genome, and one repeated
+    // multi-genome set (dedup should collapse it to one dictionary entry).
+    let rows: Vec<&[bool]> = vec![
+        &[true, false, false, false],  // singleton: genome 0
+        &[false, false, true, false],  // singleton: genome 2
+        &[true, true, false, false],   // multi: {0,1}
+        &[false, false, true, true],   // multi: {2,3}
+        &[true, true, false, false],   // multi: {0,1} again — should dedup
+    ];
+    let (_dir, m) = make_sparse(&rows, 4);
+    assert_eq!(m.n(), 5);
+    assert_eq!(m.n_cols(), 4);
+    for (slot, &expected) in rows.iter().enumerate() {
+        assert_eq!(row_as_bool(&m, slot), expected, "row {slot}");
+    }
+}
+
+#[test]
+fn fill_row_matches_row() {
+    let rows: Vec<&[bool]> = vec![
+        &[true, false, true],
+        &[false, true, false],
+        &[true, true, true],
+    ];
+    let (_dir, m) = make_sparse(&rows, 3);
+    let mut buf = vec![0u32; 3];
+    for slot in 0..3 {
+        m.fill_row(slot, &mut buf);
+        let via_fill: Vec<bool> = buf.iter().map(|&v| v != 0).collect();
+        assert_eq!(via_fill, row_as_bool(&m, slot), "slot {slot}");
+    }
+}
+
+#[test]
+fn dense_to_sparse_matches_dense_on_real_shaped_data() {
+    // Column-major dense fixture: 4 genomes (columns), 6 k-mer slots (rows),
+    // deliberately including singletons, a repeated multi-genome set, and
+    // one all-genomes row.
+    let col0 = [true, false, true, false, true, true];
+    let col1 = [false, false, true, false, true, false];
+    let col2 = [false, true, false, false, false, true];
+    let col3 = [false, false, false, true, false, true];
+    let (_dense_dir, dense) = make_dense(&[&col0, &col1, &col2, &col3]);
+
+    let sparse_root = tempdir().unwrap();
+    let sparse_dir = sparse_root.path().join("sparse");
+    let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, &sparse_dir)
+        .unwrap()
+        .finish()
+        .unwrap();
+
+    assert_eq!(sparse.n(), dense.n());
+    assert_eq!(sparse.n_cols(), dense.n_cols());
+    for slot in 0..dense.n() {
+        assert_eq!(
+            row_as_bool(&sparse, slot), &*dense.row(slot),
+            "slot {slot}: dense vs sparse disagree"
+        );
+    }
+}
+
+#[test]
+fn count_ones_matches_dense() {
+    let col0 = [true, false, true, true];
+    let col1 = [false, true, true, false];
+    let (_dense_dir, dense) = make_dense(&[&col0, &col1]);
+    let sparse_root = tempdir().unwrap();
+    let sparse_dir = sparse_root.path().join("sparse");
+    let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, &sparse_dir)
+        .unwrap()
+        .finish()
+        .unwrap();
+
+    let dense_counts = dense.count_ones();
+    let sparse_counts = sparse.count_ones();
+    assert_eq!(sparse_counts.to_vec(), dense_counts.to_vec());
+}
+
+#[test]
+fn fill_sub_matrix_matches_dense() {
+    let col0 = [true, false, true, false, true];
+    let col1 = [false, true, true, true, false];
+    let col2 = [true, true, false, false, true];
+    let (_dense_dir, dense) = make_dense(&[&col0, &col1, &col2]);
+    let sparse_root = tempdir().unwrap();
+    let sparse_dir = sparse_root.path().join("sparse");
+    let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, &sparse_dir)
+        .unwrap()
+        .finish()
+        .unwrap();
+
+    let slots = [4usize, 0, 2];
+    let dense_sub = dense.sub_matrix(&slots);
+    let mut sparse_sub: Vec<Vec<bool>> = vec![Vec::new(); dense.n_cols()];
+    sparse.fill_sub_matrix(&slots, &mut sparse_sub);
+    assert_eq!(sparse_sub, dense_sub);
+}
+
+#[test]
+fn single_genome_matrix() {
+    // n_cols=1 — every row is necessarily a singleton (cardinality 1 or
+    // 0), the dictionary/multi-array stay entirely empty.
+    let rows: Vec<&[bool]> = vec![&[true], &[false], &[true]];
+    let (_dir, m) = make_sparse(&rows, 1);
+    for (slot, &expected) in rows.iter().enumerate() {
+        assert_eq!(row_as_bool(&m, slot), expected, "row {slot}");
+    }
+}
+
+#[test]
+fn reopen_after_close_matches_original() {
+    // Explicit disk round-trip: drop the builder and every mmap it holds
+    // entirely, reopen fresh from `PersistentSparseBitMatrix::open`.
+    let rows: Vec<Vec<bool>> = (0..500)
+        .map(|i| (0..37).map(|c| (i * 7 + c * 3) % 11 == 0).collect())
+        .collect();
+    let dir = tempdir().unwrap();
+    let sparse_dir = dir.path().join("sparse");
+    {
+        let mut b = PersistentSparseBitMatrixBuilder::new(rows.len(), 37, &sparse_dir).unwrap();
+        let mut genomes = Vec::new();
+        for row in &rows {
+            genomes.clear();
+            genomes.extend((0..37).filter(|&c| row[c]).map(|c| c as u32));
+            b.push_row(&genomes);
+        }
+        b.close().unwrap();
+    } // builder + every mmap fully dropped here
+
+    let m = PersistentSparseBitMatrix::open(&sparse_dir).unwrap();
+    assert_eq!(m.n(), 500);
+    assert_eq!(m.n_cols(), 37);
+    for (slot, expected) in rows.iter().enumerate() {
+        assert_eq!(&row_as_bool(&m, slot), expected, "row {slot}");
+    }
+}
+
+/// Exercises `BinaryMatrix` generically over both concrete types — proves
+/// they're actually interchangeable at that call site, not just
+/// individually correct.
+fn sum_via_trait(m: &dyn BinaryMatrix, slot: usize) -> usize {
+    m.row(slot).iter().filter(|&&b| b).count()
+}
+
+#[test]
+fn binary_matrix_trait_interchangeable_dense_and_sparse() {
+    let col0 = [true, false, true, true];
+    let col1 = [false, true, true, false];
+    let col2 = [true, true, false, true];
+    let (_dense_dir, dense) = make_dense(&[&col0, &col1, &col2]);
+
+    let sparse_root = tempdir().unwrap();
+    let sparse_dir = sparse_root.path().join("sparse");
+    let sparse = PersistentSparseBitMatrixBuilder::build_from_dense(&dense, &sparse_dir)
+        .unwrap()
+        .finish()
+        .unwrap();
+
+    assert_eq!(BinaryMatrix::n(&dense), BinaryMatrix::n(&sparse));
+    assert_eq!(BinaryMatrix::n_cols(&dense), BinaryMatrix::n_cols(&sparse));
+    for slot in 0..BinaryMatrix::n(&dense) {
+        assert_eq!(sum_via_trait(&dense, slot), sum_via_trait(&sparse, slot), "slot {slot}");
+    }
+    let dense_counts = BinaryMatrix::count_ones(&dense);
+    let sparse_counts = BinaryMatrix::count_ones(&sparse);
+    assert_eq!(dense_counts.to_vec(), sparse_counts.to_vec());
+}
+
+#[test]
+fn reopen_large_real_shaped_data_with_heavy_dedup() {
+    // Larger, more realistic case: 91-genome scale, plenty of repeated
+    // multi-genome sets (dedup exercised for real), reopened from disk.
+    let n_cols = 91;
+    let n_rows = 5000;
+    let rows: Vec<Vec<bool>> = (0..n_rows)
+        .map(|i| {
+            let mut row = vec![false; n_cols];
+            match i % 5 {
+                0 => { row[i % n_cols] = true; } // singleton, varies per row
+                1 => { row[3] = true; row[7] = true; } // repeated multi-set A
+                2 => { row[3] = true; row[7] = true; } // same set A again
+                3 => { row[10] = true; row[20] = true; row[30] = true; } // repeated multi-set B
+                _ => { row[(i * 13) % n_cols] = true; row[(i * 29) % n_cols] = true; } // varying pairs
+            }
+            row
+        })
+        .collect();
+
+    let dir = tempdir().unwrap();
+    let sparse_dir = dir.path().join("sparse");
+    {
+        let mut b = PersistentSparseBitMatrixBuilder::new(n_rows, n_cols, &sparse_dir).unwrap();
+        let mut genomes = Vec::new();
+        for row in &rows {
+            genomes.clear();
+            genomes.extend((0..n_cols).filter(|&c| row[c]).map(|c| c as u32));
+            b.push_row(&genomes);
+        }
+        b.close().unwrap();
+    }
+
+    let m = PersistentSparseBitMatrix::open(&sparse_dir).unwrap();
+    for (slot, expected) in rows.iter().enumerate() {
+        assert_eq!(&row_as_bool(&m, slot), expected, "row {slot}");
+    }
+}

src/obicompactvec/src/traits.rs

@@ -1,5 +1,41 @@
 use ndarray::{Array1, Array2};
 
+/// Minimal shared surface between `PersistentBitMatrix` (dense) and
+/// `PersistentSparseBitMatrix` (row-major, deduplicated) — exactly what
+/// real consumers use today (`obikphylo::siblings::cache::Mat`), not the
+/// two types' full individual APIs. Column-oriented operations
+/// (`col`/`col_view`, the `BitPartials`/`ColumnWeights` distance-matrix
+/// traits above) are *not* part of this trait — `PersistentSparseBitMatrix`
+/// only offers a naive, row-scanning `count_ones` for now (see
+/// `docmd/architecture/siblings.md` and the sparse-matrix design plan,
+/// "Explicitly deferred": a row-major co-occurrence rewrite of the
+/// pairwise distance matrices is future work, not part of this trait).
+pub trait BinaryMatrix {
+    /// Number of rows (k-mer slots).
+    fn n(&self) -> usize;
+    /// Number of columns (genomes).
+    fn n_cols(&self) -> usize;
+    /// One row's presence values, one `bool` per genome.
+    fn row(&self, slot: usize) -> Box<[bool]>;
+    /// Like [`row`](Self::row), filling a caller-provided `0`/`1` buffer
+    /// instead of allocating.
+    fn fill_row(&self, slot: usize, buf: &mut [u32]);
+    /// Extracts a sub-matrix at `slots`, column-first: `out[c]` holds
+    /// column `c`'s values at `slots`, in `slots` order. `out.len()` must
+    /// equal `n_cols()`.
+    fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>]);
+    /// Per-genome k-mer totals.
+    fn count_ones(&self) -> Array1<u64>;
+
+    /// Allocating variant of [`fill_sub_matrix`](Self::fill_sub_matrix) —
+    /// provided once here so implementers only need the fill-in-place form.
+    fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<bool>> {
+        let mut out: Vec<Vec<bool>> = (0..self.n_cols()).map(|_| Vec::new()).collect();
+        self.fill_sub_matrix(slots, &mut out);
+        out
+    }
+}
+
 /// Convert a Jaccard distance matrix (`1 - J`) into a Mash distance matrix, per
 /// https://mash.readthedocs.io/en/latest/distances.html:
 /// `D = -1/k * ln(2J / (1+J))`.

src/obicompactvec/src/views.rs

@@ -14,8 +14,11 @@ impl<'a> BitSliceView<'a> {
     #[inline]
     pub fn new(words: &'a [u64], n: usize) -> Self { Self { words, n } }
 
+    #[inline]
     pub fn len(&self)      -> usize  { self.n }
+    #[inline]
     pub fn is_empty(&self) -> bool   { self.n == 0 }
+    #[inline]
     pub fn words(&self)    -> &'a [u64] { self.words }
 
     #[inline]
@@ -23,11 +26,44 @@ impl<'a> BitSliceView<'a> {
         (self.words[slot >> 6] >> (slot & 63)) & 1 != 0
     }
 
+    /// Batch lookup: read bits at `slots` with cache-friendly access pattern.
+    ///
+    /// Slots are sorted internally before reading, then results are reordered
+    /// to match the input order.
+    pub fn get_batch(&self, slots: &[usize]) -> Vec<bool> {
+        let mut out = vec![false; slots.len()];
+        self.fill_batch(slots, &mut out);
+        out
+    }
+    /// Like [`get_batch`](Self::get_batch), but fills a caller-provided buffer.
+    pub fn fill_batch(&self, slots: &[usize], out: &mut [bool]) {
+        assert_eq!(slots.len(), out.len());
+        let n = slots.len();
+        if n == 0 { return; }
+        let mut perm: Vec<usize> = (0..n).collect();
+        perm.sort_by_key(|&i| slots[i]);
+        let sorted: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
+        let mut tmp = vec![false; n];
+        self.fill_batch_sorted(&sorted, &mut tmp);
+        for (i, &orig_idx) in perm.iter().enumerate() {
+            out[orig_idx] = tmp[i];
+        }
+    }
+
+    /// Fill `out` assuming `sorted_slots` is already in ascending order.
+    /// Results are written in `sorted_slots` order (no reordering).
+    pub(crate) fn fill_batch_sorted(&self, sorted_slots: &[usize], out: &mut [bool]) {
+        assert_eq!(sorted_slots.len(), out.len());
+        for (i, &slot) in sorted_slots.iter().enumerate() {
+            out[i] = self.get(slot);
+        }
+    }
     pub fn count_ones(&self) -> u64 {
         self.words.iter().map(|w| w.count_ones() as u64).sum()
     }
     pub fn count_zeros(&self) -> u64 { self.n as u64 - self.count_ones() }
 
+    #[inline]
     pub fn iter(&self) -> BitSliceIter<'a> {
         BitSliceIter { words: self.words, slot: 0, n: self.n }
     }
@@ -63,12 +99,14 @@ pub struct BitSliceIter<'a> {
 
 impl Iterator for BitSliceIter<'_> {
     type Item = bool;
+    #[inline]
     fn next(&mut self) -> Option<bool> {
         if self.slot >= self.n { return None; }
         let v = (self.words[self.slot >> 6] >> (self.slot & 63)) & 1 != 0;
         self.slot += 1;
         Some(v)
     }
+    #[inline]
     fn size_hint(&self) -> (usize, Option<usize>) {
         let rem = self.n - self.slot;
         (rem, Some(rem))
@@ -94,11 +132,16 @@ impl<'a> IntSliceView<'a> {
         Self { primary, overflow_raw, n_overflow, n }
     }
 
+    #[inline]
     pub fn len(&self)        -> usize    { self.n }
+    #[inline]
     pub fn is_empty(&self)   -> bool     { self.n == 0 }
+    #[inline]
     pub fn primary_bytes(&self) -> &'a [u8] { self.primary }
+    #[inline]
     pub fn n_overflow(&self) -> usize    { self.n_overflow }
 
+    #[inline]
     pub fn overflow_entries(&self) -> impl Iterator<Item = (usize, u32)> + 'a {
         let raw  = self.overflow_raw;
         let n_ov = self.n_overflow;
@@ -123,7 +166,42 @@ impl<'a> IntSliceView<'a> {
         panic!("slot {slot} marked overflow but not found")
     }
 
+    /// Batch lookup: read values at `slots` with cache-friendly access pattern.
+    ///
+    /// Slots are sorted internally before reading, then results are reordered
+    /// to match the input order.
+    pub fn get_batch(&self, slots: &[usize]) -> Vec<u32> {
+        let mut out = vec![0u32; slots.len()];
+        self.fill_batch(slots, &mut out);
+        out
+    }
+
+    /// Like [`get_batch`](Self::get_batch), but fills a caller-provided buffer.
+    pub fn fill_batch(&self, slots: &[usize], out: &mut [u32]) {
+        assert_eq!(slots.len(), out.len());
+        let n = slots.len();
+        if n == 0 { return; }
+        let mut perm: Vec<usize> = (0..n).collect();
+        perm.sort_by_key(|&i| slots[i]);
+        let sorted: Vec<usize> = perm.iter().map(|&i| slots[i]).collect();
+        let mut tmp = vec![0u32; n];
+        self.fill_batch_sorted(&sorted, &mut tmp);
+        for (i, &orig_idx) in perm.iter().enumerate() {
+            out[orig_idx] = tmp[i];
+        }
+    }
+
+    /// Fill `out` assuming `sorted_slots` is already in ascending order.
+    /// Results are written in `sorted_slots` order (no reordering).
+    pub(crate) fn fill_batch_sorted(&self, sorted_slots: &[usize], out: &mut [u32]) {
+        assert_eq!(sorted_slots.len(), out.len());
+        for (i, &slot) in sorted_slots.iter().enumerate() {
+            out[i] = self.get(slot);
+        }
+    }
+
     /// Sequential merge scan: yields all n values in slot order.
+    #[inline]
     pub fn iter(&self) -> IntSliceViewIter<'a> {
         IntSliceViewIter {
             primary:      self.primary,
@@ -258,6 +336,7 @@ pub struct IntSliceViewIter<'a> {
 
 impl Iterator for IntSliceViewIter<'_> {
     type Item = u32;
+    #[inline]
     fn next(&mut self) -> Option<u32> {
         if self.slot >= self.n { return None; }
         let v = self.primary[self.slot];
@@ -270,6 +349,7 @@ impl Iterator for IntSliceViewIter<'_> {
             Some(val)
         }
     }
+    #[inline]
     fn size_hint(&self) -> (usize, Option<usize>) {
         let rem = self.n - self.slot;
         (rem, Some(rem))

src/obidebruinj/src/debruijn/graph.rs

@@ -1,250 +1,24 @@
-//use ahash::RandomState;
-use crossbeam_channel;
 use hashbrown::HashMap;
 use obikseq::k;
-use obikseq::{CanonicalKmer, Sequence, Unitig};
+use obikseq::{CanonicalKmer, Unitig};
 #[cfg(not(any(test, feature = "test-utils")))]
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use std::cell::RefCell;
-use std::fmt;
 use std::sync::atomic::{AtomicU8, Ordering};
 use xxhash_rust::xxh3::Xxh3Builder;
 
+use super::node::{IS_VISITED_MASK, Node};
+use super::unitig_iter::UnitigNucIter;
+use super::walk::WalkState;
+
 // ── Types ─────────────────────────────────────────────────────────────────────
 
-type FastHashMap<K, V> = HashMap<K, V, Xxh3Builder>;
-
-// ── Node ──────────────────────────────────────────────────────────────────────
-//
-//  bit layout (LSB first):
-//    bit  0   : can_extend_right — exactly one right canonical neighbour exists
-//    bit  1   : can_extend_left  — exactly one left  canonical neighbour exists
-//    bit  2   : visited
-//    bits 3–4 : right_nuc — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 0 = 1
-//    bits 5–6 : left_nuc  — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 1 = 1
-//    bit  7   : marked as start node (1)
-//
-//  "can_extend" = false covers both 0 neighbours and ≥2 neighbours; the only
-//  information needed for traversal is "exactly one".
-
-#[repr(transparent)]
-#[derive(Debug, Clone, Copy, Default)]
-pub struct Node(u8);
-
-const CAN_EXTEND_RIGHT_MASK: u8 = 0b0000_0001; // bit 0: can_extend_right — exactly one right canonical neighbour exists
-const CAN_EXTEND_LEFT_MASK: u8 = 0b0000_0010; // bit 1: can_extend_left  — exactly one left  canonical neighbour exists
-const IS_VISITED_MASK: u8 = 0b0000_0100; // bit 2: visited
-const RIGHT_NUC_MASK: u8 = 0b0001_1000; // bits 3–4: right_nuc — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 0 = 1
-const LEFT_NUC_MASK: u8 = 0b0110_0000; // bits 5–6: left_nuc  — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 1 = 1
-const IS_START_MASK: u8 = 0b1000_0000; // bit 7: marked as start node
-
-impl Node {
-    /// Returns `true` if the node can be extended to the right.
-    ///
-    /// A single right neighbour exists.
-    #[inline]
-    pub fn can_extend_right(self) -> bool {
-        self.0 & CAN_EXTEND_RIGHT_MASK != 0
-    }
-
-    /// Returns `true` if the node can be extended to the left.
-    ///
-    /// A single left neighbour exists.
-    #[inline]
-    pub fn can_extend_left(self) -> bool {
-        self.0 & CAN_EXTEND_LEFT_MASK != 0
-    }
-
-    /// Returns `true` if the node has been visited.
-    #[inline]
-    pub fn is_visited(self) -> bool {
-        self.0 & IS_VISITED_MASK != 0
-    }
-
-    /// Returns `true` if the node is a start node.
-    #[inline]
-    pub fn is_start(self) -> bool {
-        self.0 & IS_START_MASK != 0
-    }
-
-    #[inline]
-    pub fn set_start(&mut self) {
-        self.0 |= IS_START_MASK;
-    }
-
-    pub fn unset_start(&mut self) {
-        self.0 &= !IS_START_MASK;
-    }
-
-    /// Index of the unique right neighbour (0=A, 1=C, 2=G, 3=T).
-    /// Only meaningful when `can_extend_right()` is true.
-    #[inline]
-    pub fn right_nuc(self) -> u8 {
-        debug_assert!(
-            self.can_extend_right(),
-            "from: right_nuc -> The node cannot be extended to the right"
-        );
-        (self.0 >> 3) & 0b11
-    }
-
-    /// Index of the unique left neighbour (0=A, 1=C, 2=G, 3=T).
-    /// Only meaningful when `can_extend_left()` is true.
-    #[inline]
-    pub fn left_nuc(self) -> u8 {
-        debug_assert!(
-            self.can_extend_left(),
-            "from: left_nuc -> The node cannot be extended to the left"
-        );
-        (self.0 >> 5) & 0b11
-    }
-
-    /// Marks the node as visited.
-    #[inline]
-    pub fn set_visited(&mut self) {
-        debug_assert!(
-            !self.is_visited(),
-            "from: is_visited -> The node has already been visited"
-        );
-        self.0 |= IS_VISITED_MASK;
-    }
-
-    /// `nuc` = Some(i) → exactly one neighbour (bit 0 set, bits 3–4 = nucleotide index).
-    /// `nuc` = None    → 0 or ≥2 neighbours; `count` encoded in bits 3–4 as count.sat_sub(1).
-    pub fn set_right(&mut self, count: u8, nuc: Option<u8>) {
-        self.0 &= !(CAN_EXTEND_RIGHT_MASK | RIGHT_NUC_MASK);
-        if count == 1 {
-            self.0 |= CAN_EXTEND_RIGHT_MASK;
-            if let Some(n) = nuc {
-                self.0 |= (n & 0b11) << 3;
-                return;
-            }
-            unreachable!("nuc must be Some when count is 1");
-        }
-        self.0 |= (count.saturating_sub(1).min(3)) << 3;
-    }
-
-    /// `nuc` = Some(i) → exactly one neighbour (bit 0 set, bits 3–4 = nucleotide index).
-    /// `nuc` = None    → 0 or ≥2 neighbours; `count` encoded in bits 3–4 as count.sat_sub(1).
-    pub fn set_left(&mut self, count: u8, nuc: Option<u8>) {
-        self.0 &= !(CAN_EXTEND_LEFT_MASK | LEFT_NUC_MASK);
-        if count == 1 {
-            self.0 |= CAN_EXTEND_LEFT_MASK;
-            if let Some(n) = nuc {
-                self.0 |= (n & 0b11) << 5;
-                return;
-            }
-            unreachable!("nuc must be Some when count is 1");
-        }
-        self.0 |= (count.saturating_sub(1).min(3)) << 5;
-    }
-}
-
-impl fmt::Display for Node {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        const NUC: [char; 4] = ['A', 'C', 'G', 'T'];
-        let r = if self.can_extend_right() {
-            format!("→{}", NUC[self.right_nuc() as usize])
-        } else if (self.0 >> 3) & 0b11 == 0 {
-            "→0".to_string()
-        } else {
-            "→≥2".to_string()
-        };
-        let l = if self.can_extend_left() {
-            format!("←{}", NUC[self.left_nuc() as usize])
-        } else if (self.0 >> 5) & 0b11 == 0 {
-            "←0".to_string()
-        } else {
-            "←≥2".to_string()
-        };
-        let v = if self.is_visited() { "V" } else { "." };
-        write!(f, "Node({r} {l} {v})")
-    }
-}
-
-pub struct WalkState {
-    kmer: CanonicalKmer,
-    node: Node,
-    direct: bool,
-}
-
-impl WalkState {
-    pub fn new(kmer: CanonicalKmer, node: Node, direct: bool) -> Self {
-        debug_assert!(!node.is_visited(), "Cannot walk over a visited node");
-        Self { kmer, node, direct }
-    }
-
-    pub fn leavable(&self, graph: &GraphDeBruijn) -> bool {
-        self.walk(graph).is_some()
-    }
-
-    pub fn reachable(&self, graph: &GraphDeBruijn) -> bool {
-        WalkState {
-            kmer: self.kmer,
-            node: self.node,
-            direct: !self.direct,
-        }
-        .leavable(graph)
-    }
-
-    pub fn walk(&self, graph: &GraphDeBruijn) -> Option<(WalkState, u8)> {
-        if self.direct {
-            if !self.node.can_extend_right() {
-                return None;
-            }
-            let nuc = self.node.right_nuc();
-            let next = self.kmer.into_kmer().push_right(nuc);
-            let cnext = next.canonical();
-            let dnext = next.raw() == cnext.raw();
-            let next_node = Node(graph.nodes.get(&cnext).unwrap().load(Ordering::Relaxed));
-            if next_node.is_visited() {
-                return None;
-            }
-            let reachable = if dnext {
-                next_node.can_extend_left()
-            } else {
-                next_node.can_extend_right()
-            };
-            reachable.then_some((
-                WalkState {
-                    kmer: cnext,
-                    node: next_node,
-                    direct: dnext,
-                },
-                nuc,
-            ))
-        } else {
-            if !self.node.can_extend_left() {
-                return None;
-            }
-            let nuc = self.node.left_nuc();
-            let next = self.kmer.into_kmer().push_left(nuc);
-            let cnext = next.canonical();
-            let dnext = next.raw() != cnext.raw();
-            let next_node = Node(graph.nodes.get(&cnext).unwrap().load(Ordering::Relaxed));
-            if next_node.is_visited() {
-                return None;
-            }
-            let reachable = if dnext {
-                next_node.can_extend_right()
-            } else {
-                next_node.can_extend_left()
-            };
-            reachable.then_some((
-                WalkState {
-                    kmer: cnext,
-                    node: next_node,
-                    direct: dnext,
-                },
-                3 - nuc,
-            ))
-        }
-    }
-}
+pub(super) type FastHashMap<K, V> = HashMap<K, V, Xxh3Builder>;
 
 // ── GraphDeBruijn ─────────────────────────────────────────────────────────────
 
 pub struct GraphDeBruijn {
-    nodes: FastHashMap<CanonicalKmer, AtomicU8>,
+    pub(super) nodes: FastHashMap<CanonicalKmer, AtomicU8>,
 }
 
 impl GraphDeBruijn {
@@ -346,7 +120,7 @@ impl GraphDeBruijn {
         Some(WalkState::new(kmer, node, true))
     }
 
-    fn unitig_nucleotides(&self, kmer: CanonicalKmer, k: usize) -> Option<UnitigNucIter<'_>> {
+    pub(super) fn unitig_nucleotides(&self, kmer: CanonicalKmer, k: usize) -> Option<UnitigNucIter<'_>> {
         let old = self
             .nodes
             .get(&kmer)?
@@ -362,13 +136,7 @@ impl GraphDeBruijn {
                 .fetch_or(IS_VISITED_MASK, Ordering::AcqRel);
             (ext_old & IS_VISITED_MASK == 0).then_some((next_state, nuc))
         });
-        Some(UnitigNucIter {
-            graph: self,
-            start: kmer,
-            pos: 0,
-            k,
-            next_step,
-        })
+        Some(UnitigNucIter::new(self, kmer, k, next_step))
     }
 
     pub fn for_each_unitig(&self, f: impl Fn(UnitigNucIter<'_>) + Sync) {
@@ -467,12 +235,7 @@ impl GraphDeBruijn {
     }
 
     fn is_start(&self, query: CanonicalKmer, node: Node) -> bool {
-        !WalkState {
-            kmer: query,
-            node,
-            direct: true,
-        }
-        .reachable(self)
+        !WalkState::new(query, node, true).reachable(self)
     }
 
     pub fn try_for_each_unitig<E, F>(&self, f: F) -> Result<(), E>
@@ -514,44 +277,6 @@ impl GraphDeBruijn {
     }
 }
 
-// ── UnitigNucIter ─────────────────────────────────────────────────────────────
-
-pub struct UnitigNucIter<'a> {
-    graph: &'a GraphDeBruijn,
-    start: CanonicalKmer,
-    pos: usize,
-    k: usize,
-    next_step: Option<(WalkState, u8)>,
-}
-
-impl Iterator for UnitigNucIter<'_> {
-    type Item = u8;
-
-    fn next(&mut self) -> Option<u8> {
-        if self.pos < self.k {
-            let nuc = self.start.nucleotide(self.pos);
-            self.pos += 1;
-            Some(nuc)
-        } else if let Some((state, nuc)) = self.next_step.take() {
-            self.next_step = state.walk(self.graph).and_then(|(next_state, next_nuc)| {
-                let old = self
-                    .graph
-                    .nodes
-                    .get(&next_state.kmer)?
-                    .fetch_or(IS_VISITED_MASK, Ordering::AcqRel);
-                (old & IS_VISITED_MASK == 0).then_some((next_state, next_nuc))
-            });
-            Some(nuc)
-        } else {
-            None
-        }
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (self.k - self.pos.min(self.k), None)
-    }
-}
-
 /// Returns the count of neighbors and the index of the first
 /// neighbor if exactly one of the four canonical neighbours exists in
 /// the graph, where `i` is its index (0=A, 1=C, 2=G, 3=T).
@@ -580,8 +305,3 @@ fn count_neighbors(
         (0, None)
     }
 }
-
-// ── tests ─────────────────────────────────────────────────────────────────────
-#[cfg(test)]
-#[path = "tests/debruijn.rs"]
-mod tests;

src/obidebruinj/src/debruijn/mod.rs

@@ -0,0 +1,20 @@
+//! De Bruijn graph over canonical k-mers, built for unitig extraction.
+//!
+//! Submodules: [`node`] (packed per-kmer neighbour/visited/start flags),
+//! [`walk`] (single-step traversal), [`graph`] ([`GraphDeBruijn`] itself),
+//! [`unitig_iter`] (nucleotide-by-nucleotide unitig walk iterator).
+
+mod graph;
+mod node;
+mod unitig_iter;
+mod walk;
+
+pub use graph::GraphDeBruijn;
+
+// Only used by `tests/debruijn.rs` (`use super::*`) below.
+#[cfg(test)]
+use obikseq::{CanonicalKmer, Sequence};
+
+#[cfg(test)]
+#[path = "../tests/debruijn.rs"]
+mod tests;

src/obidebruinj/src/debruijn/node.rs

@@ -0,0 +1,148 @@
+use std::fmt;
+
+// ── Node ──────────────────────────────────────────────────────────────────────
+//
+//  bit layout (LSB first):
+//    bit  0   : can_extend_right — exactly one right canonical neighbour exists
+//    bit  1   : can_extend_left  — exactly one left  canonical neighbour exists
+//    bit  2   : visited
+//    bits 3–4 : right_nuc — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 0 = 1
+//    bits 5–6 : left_nuc  — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 1 = 1
+//    bit  7   : marked as start node (1)
+//
+//  "can_extend" = false covers both 0 neighbours and ≥2 neighbours; the only
+//  information needed for traversal is "exactly one".
+
+#[repr(transparent)]
+#[derive(Debug, Clone, Copy, Default)]
+pub struct Node(pub(super) u8);
+
+const CAN_EXTEND_RIGHT_MASK: u8 = 0b0000_0001; // bit 0: can_extend_right — exactly one right canonical neighbour exists
+const CAN_EXTEND_LEFT_MASK: u8 = 0b0000_0010; // bit 1: can_extend_left  — exactly one left  canonical neighbour exists
+pub(super) const IS_VISITED_MASK: u8 = 0b0000_0100; // bit 2: visited
+const RIGHT_NUC_MASK: u8 = 0b0001_1000; // bits 3–4: right_nuc — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 0 = 1
+const LEFT_NUC_MASK: u8 = 0b0110_0000; // bits 5–6: left_nuc  — index 0–3 (A/C/G/T) of that neighbour; valid iff bit 1 = 1
+const IS_START_MASK: u8 = 0b1000_0000; // bit 7: marked as start node
+
+impl Node {
+    /// Returns `true` if the node can be extended to the right.
+    ///
+    /// A single right neighbour exists.
+    #[inline]
+    pub fn can_extend_right(self) -> bool {
+        self.0 & CAN_EXTEND_RIGHT_MASK != 0
+    }
+
+    /// Returns `true` if the node can be extended to the left.
+    ///
+    /// A single left neighbour exists.
+    #[inline]
+    pub fn can_extend_left(self) -> bool {
+        self.0 & CAN_EXTEND_LEFT_MASK != 0
+    }
+
+    /// Returns `true` if the node has been visited.
+    #[inline]
+    pub fn is_visited(self) -> bool {
+        self.0 & IS_VISITED_MASK != 0
+    }
+
+    /// Returns `true` if the node is a start node.
+    #[inline]
+    pub fn is_start(self) -> bool {
+        self.0 & IS_START_MASK != 0
+    }
+
+    #[inline]
+    pub fn set_start(&mut self) {
+        self.0 |= IS_START_MASK;
+    }
+
+    pub fn unset_start(&mut self) {
+        self.0 &= !IS_START_MASK;
+    }
+
+    /// Index of the unique right neighbour (0=A, 1=C, 2=G, 3=T).
+    /// Only meaningful when `can_extend_right()` is true.
+    #[inline]
+    pub fn right_nuc(self) -> u8 {
+        debug_assert!(
+            self.can_extend_right(),
+            "from: right_nuc -> The node cannot be extended to the right"
+        );
+        (self.0 >> 3) & 0b11
+    }
+
+    /// Index of the unique left neighbour (0=A, 1=C, 2=G, 3=T).
+    /// Only meaningful when `can_extend_left()` is true.
+    #[inline]
+    pub fn left_nuc(self) -> u8 {
+        debug_assert!(
+            self.can_extend_left(),
+            "from: left_nuc -> The node cannot be extended to the left"
+        );
+        (self.0 >> 5) & 0b11
+    }
+
+    /// Marks the node as visited.
+    #[inline]
+    pub fn set_visited(&mut self) {
+        debug_assert!(
+            !self.is_visited(),
+            "from: is_visited -> The node has already been visited"
+        );
+        self.0 |= IS_VISITED_MASK;
+    }
+
+    /// `nuc` = Some(i) → exactly one neighbour (bit 0 set, bits 3–4 = nucleotide index).
+    /// `nuc` = None    → 0 or ≥2 neighbours; `count` encoded in bits 3–4 as count.sat_sub(1).
+    pub fn set_right(&mut self, count: u8, nuc: Option<u8>) {
+        self.0 &= !(CAN_EXTEND_RIGHT_MASK | RIGHT_NUC_MASK);
+        if count == 1 {
+            self.0 |= CAN_EXTEND_RIGHT_MASK;
+            if let Some(n) = nuc {
+                self.0 |= (n & 0b11) << 3;
+                return;
+            }
+            unreachable!("nuc must be Some when count is 1");
+        }
+        self.0 |= (count.saturating_sub(1).min(3)) << 3;
+    }
+
+    /// `nuc` = Some(i) → exactly one neighbour (bit 0 set, bits 3–4 = nucleotide index).
+    /// `nuc` = None    → 0 or ≥2 neighbours; `count` encoded in bits 3–4 as count.sat_sub(1).
+    pub fn set_left(&mut self, count: u8, nuc: Option<u8>) {
+        self.0 &= !(CAN_EXTEND_LEFT_MASK | LEFT_NUC_MASK);
+        if count == 1 {
+            self.0 |= CAN_EXTEND_LEFT_MASK;
+            if let Some(n) = nuc {
+                self.0 |= (n & 0b11) << 5;
+                return;
+            }
+            unreachable!("nuc must be Some when count is 1");
+        }
+        self.0 |= (count.saturating_sub(1).min(3)) << 5;
+    }
+}
+
+impl fmt::Display for Node {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        const NUC: [char; 4] = ['A', 'C', 'G', 'T'];
+        let r = if self.can_extend_right() {
+            format!("→{}", NUC[self.right_nuc() as usize])
+        } else if (self.0 >> 3) & 0b11 == 0 {
+            "→0".to_string()
+        } else {
+            "→≥2".to_string()
+        };
+        let l = if self.can_extend_left() {
+            format!("←{}", NUC[self.left_nuc() as usize])
+        } else if (self.0 >> 5) & 0b11 == 0 {
+            "←0".to_string()
+        } else {
+            "←≥2".to_string()
+        };
+        let v = if self.is_visited() { "V" } else { "." };
+        write!(f, "Node({r} {l} {v})")
+    }
+}

src/obidebruinj/src/debruijn/unitig_iter.rs

@@ -0,0 +1,61 @@
+use obikseq::CanonicalKmer;
+use std::sync::atomic::Ordering;
+
+use super::graph::GraphDeBruijn;
+use super::node::IS_VISITED_MASK;
+use super::walk::WalkState;
+
+// ── UnitigNucIter ─────────────────────────────────────────────────────────────
+
+pub struct UnitigNucIter<'a> {
+    graph: &'a GraphDeBruijn,
+    start: CanonicalKmer,
+    pos: usize,
+    k: usize,
+    next_step: Option<(WalkState, u8)>,
+}
+
+impl<'a> UnitigNucIter<'a> {
+    pub(super) fn new(
+        graph: &'a GraphDeBruijn,
+        start: CanonicalKmer,
+        k: usize,
+        next_step: Option<(WalkState, u8)>,
+    ) -> Self {
+        Self {
+            graph,
+            start,
+            pos: 0,
+            k,
+            next_step,
+        }
+    }
+}
+
+impl Iterator for UnitigNucIter<'_> {
+    type Item = u8;
+
+    fn next(&mut self) -> Option<u8> {
+        if self.pos < self.k {
+            let nuc = self.start.nucleotide(self.pos);
+            self.pos += 1;
+            Some(nuc)
+        } else if let Some((state, nuc)) = self.next_step.take() {
+            self.next_step = state.walk(self.graph).and_then(|(next_state, next_nuc)| {
+                let old = self
+                    .graph
+                    .nodes
+                    .get(&next_state.kmer)?
+                    .fetch_or(IS_VISITED_MASK, Ordering::AcqRel);
+                (old & IS_VISITED_MASK == 0).then_some((next_state, next_nuc))
+            });
+            Some(nuc)
+        } else {
+            None
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.k - self.pos.min(self.k), None)
+    }
+}

src/obidebruinj/src/debruijn/walk.rs

@@ -0,0 +1,85 @@
+use obikseq::{CanonicalKmer, Sequence};
+use std::sync::atomic::Ordering;
+
+use super::graph::GraphDeBruijn;
+use super::node::Node;
+
+pub struct WalkState {
+    pub(super) kmer: CanonicalKmer,
+    pub(super) node: Node,
+    pub(super) direct: bool,
+}
+
+impl WalkState {
+    pub fn new(kmer: CanonicalKmer, node: Node, direct: bool) -> Self {
+        debug_assert!(!node.is_visited(), "Cannot walk over a visited node");
+        Self { kmer, node, direct }
+    }
+
+    pub fn leavable(&self, graph: &GraphDeBruijn) -> bool {
+        self.walk(graph).is_some()
+    }
+
+    pub fn reachable(&self, graph: &GraphDeBruijn) -> bool {
+        WalkState {
+            kmer: self.kmer,
+            node: self.node,
+            direct: !self.direct,
+        }
+        .leavable(graph)
+    }
+
+    pub fn walk(&self, graph: &GraphDeBruijn) -> Option<(WalkState, u8)> {
+        if self.direct {
+            if !self.node.can_extend_right() {
+                return None;
+            }
+            let nuc = self.node.right_nuc();
+            let next = self.kmer.into_kmer().push_right(nuc);
+            let cnext = next.canonical();
+            let dnext = next.raw() == cnext.raw();
+            let next_node = Node(graph.nodes.get(&cnext).unwrap().load(Ordering::Relaxed));
+            if next_node.is_visited() {
+                return None;
+            }
+            let reachable = if dnext {
+                next_node.can_extend_left()
+            } else {
+                next_node.can_extend_right()
+            };
+            reachable.then_some((
+                WalkState {
+                    kmer: cnext,
+                    node: next_node,
+                    direct: dnext,
+                },
+                nuc,
+            ))
+        } else {
+            if !self.node.can_extend_left() {
+                return None;
+            }
+            let nuc = self.node.left_nuc();
+            let next = self.kmer.into_kmer().push_left(nuc);
+            let cnext = next.canonical();
+            let dnext = next.raw() != cnext.raw();
+            let next_node = Node(graph.nodes.get(&cnext).unwrap().load(Ordering::Relaxed));
+            if next_node.is_visited() {
+                return None;
+            }
+            let reachable = if dnext {
+                next_node.can_extend_right()
+            } else {
+                next_node.can_extend_left()
+            };
+            reachable.then_some((
+                WalkState {
+                    kmer: cnext,
+                    node: next_node,
+                    direct: dnext,
+                },
+                3 - nuc,
+            ))
+        }
+    }
+}

src/obidebruinj/src/tests/debruijn.rs

@@ -1,6 +1,17 @@
 use super::*;
 use obikseq::{k, set_k, unitig::Unitig, Kmer};
 
+// `obikseq::params` is process-wide (see obikseq/src/params.rs): tests in this
+// file don't all use the same `k` (`push_palindrome_single_node` needs an
+// even k=4 — no odd-length self-revcomp palindrome exists — while the rest
+// use k=5), and they run concurrently by default. Serialize the
+// set_k-through-use critical section across this file's tests so one test's
+// `k` can never be overwritten mid-flight by another.
+static K_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+fn lock_k() -> std::sync::MutexGuard<'static, ()> {
+    K_LOCK.lock().unwrap_or_else(|e| e.into_inner())
+}
+
 // Build a graph from an ASCII sequence, inserting all canonical k-mers.
 fn graph_from_ascii(seq: &[u8]) -> GraphDeBruijn {
     let mut g = GraphDeBruijn::new();
@@ -37,6 +48,7 @@ fn collect_unitigs(g: &GraphDeBruijn) -> Vec<Unitig> {
 #[test]
 fn push_deduplicates_revcomp() {
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let kmer = Kmer::from_ascii(b"ACGTA").unwrap();
     let mut g = GraphDeBruijn::new();
@@ -49,6 +61,7 @@ fn push_deduplicates_revcomp() {
 fn push_palindrome_single_node() {
     // ACGT is its own revcomp
     let k = 4;
+    let _guard = lock_k();
     set_k(k);
     let kmer = Kmer::from_ascii(b"ACGT").unwrap();
     assert_eq!(kmer, kmer.revcomp(), "test requires a palindrome");
@@ -71,6 +84,7 @@ fn linear_chain_graph() -> (GraphDeBruijn, Vec<CanonicalKmer>) {
 #[test]
 fn degrees_linear_chain_node_count() {
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let (g, kmers) = linear_chain_graph();
     assert_eq!(g.len(), kmers.len());
@@ -82,6 +96,7 @@ fn degrees_linear_chain_extensions() {
     // Note: start_iter must not be consumed standalone — its second pass only
     // finds true cycle nodes when interleaved with chain traversal (iter_unitig).
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let seq = b"AAAAGGGG";
     let g = graph_from_ascii(seq);
@@ -118,6 +133,7 @@ fn kmers_from_unitigs(unitigs: &[Unitig]) -> Vec<CanonicalKmer> {
 fn unitig_roundtrip_linear() {
     // Non-repetitive sequence: all k-mers must be recovered across unitigs.
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let seq = b"ACCTGGCTA";
     let g = graph_from_ascii(seq);
@@ -136,6 +152,7 @@ fn unitig_roundtrip_longer_sequence() {
     // Longer non-repetitive sequence with no repeated k-mer of length k.
     // ACGTGGCTATCGAC with k=5 → 10 distinct k-mers, one linear chain.
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let seq = b"ACGTGGCTATCGAC";
     let g = graph_from_ascii(seq);
@@ -152,6 +169,7 @@ fn unitig_roundtrip_longer_sequence() {
 fn unitig_isolated_node() {
     // Single k-mer with no neighbours
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let kmer = Kmer::from_ascii(b"ACGTA").unwrap();
     let mut g = GraphDeBruijn::new();
@@ -165,6 +183,7 @@ fn unitig_isolated_node() {
 #[test]
 fn unitig_two_isolated_nodes() {
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let mut g = GraphDeBruijn::new();
     // Two k-mers that share no (k-1)-overlap
@@ -177,6 +196,7 @@ fn unitig_two_isolated_nodes() {
 #[test]
 fn unitig_two_truly_distinct_isolated_nodes() {
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let mut g = GraphDeBruijn::new();
     g.push(Kmer::from_ascii(b"AAAAC").unwrap().canonical());
@@ -192,7 +212,8 @@ fn unitig_two_truly_distinct_isolated_nodes() {
 
 #[test]
 fn no_kmer_lost_or_duplicated() {
-    let k = 7;
+    let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let seq = b"ACGTACGTACGTTTTTACGTACGT";
     let g = graph_from_ascii(seq);
@@ -218,6 +239,7 @@ fn cycle_kmers_not_lost() {
     // start_iter first pass yields nothing (all nodes internal); second pass
     // picks up cycle entries. All 4 k-mers must appear in the unitigs.
     let k = 5;
+    let _guard = lock_k();
     set_k(k);
     let seq = b"ACGTACGT";
     let g = graph_from_ascii(seq);
@@ -240,6 +262,7 @@ fn branching_graph_no_kmer_lost_or_duplicated() {
     // Each "node" is a distinct 5-mer; edges share a 4-mer suffix/prefix.
     // We use long non-repetitive sequences and extract only the required kmers.
     let k: usize = 5;
+    let _guard = lock_k();
     set_k(k);
     let mut g = GraphDeBruijn::new();
 

src/obikindex/Cargo.toml

@@ -6,12 +6,11 @@ edition = "2024"
 [dependencies]
 obikseq          = { path = "../obikseq" }
 obikpartitionner = { path = "../obikpartitionner" }
+obitaxonomy      = { path = "../obitaxonomy" }
 obiskio          = { path = "../obiskio" }
 obisys           = { path = "../obisys" }
 obicompactvec    = { path = "../obicompactvec" }
 obilayeredmap    = { path = "../obilayeredmap" }
-obiskbuilder     = { path = "../obiskbuilder" }
-obipipeline      = { path = "../obipipeline" }
 ndarray          = "0.16"
 rayon             = "1"
 crossbeam-channel = "0.5"
@@ -24,6 +23,7 @@ hwlocality       = { version = "1.0.0-alpha.11", features = ["vendored"], option
 [dev-dependencies]
 obiread  = { path = "../obiread" }
 tempfile = "3"
+tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
 
 [features]
 default = ["numa"]

src/obikindex/src/index.rs

@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 
-use obikpartitionner::{KmerPartition, KmerSpectrum};
+use obikpartitionner::{KmerPartition, KmerSpectrum, PARTITIONS_SUBDIR};
 use obilayeredmap;
 use obisys::{Reporter, Stage, progress_bar};
 use rayon::prelude::*;
@@ -68,17 +68,93 @@ impl KmerIndex {
         IndexMeta::exists(path.as_ref())
     }
 
+    /// Make `output` ready to receive a newly created index.
+    ///
+    /// If an index already exists there, remove it when `force` is set,
+    /// otherwise fail. A bare directory with no `index.meta` (e.g. one left
+    /// behind by a `DirLock`) is not considered a pre-existing index.
+    pub fn clear_output_for_create<P: AsRef<Path>>(output: P, force: bool) -> OKIResult<()> {
+        let output = output.as_ref();
+        if Self::exists(output) {
+            if force {
+                fs::remove_dir_all(output).map_err(OKIError::Io)?;
+            } else {
+                return Err(OKIError::Io(std::io::Error::new(
+                    std::io::ErrorKind::AlreadyExists,
+                    format!("{}: output directory already exists", output.display()),
+                )));
+            }
+        }
+        Ok(())
+    }
+
+    /// Lay out a fresh index skeleton at `output`: the root directory,
+    /// `index.meta` (from `meta`), and an opened, empty partition set.
+    ///
+    /// For construction paths that build partitions from scratch (`select`,
+    /// `rebuild`). `merge` bootstraps by copying a source index instead, so
+    /// it does not use this.
+    pub(crate) fn create_skeleton<P: AsRef<Path>>(
+        output: P,
+        meta: &IndexMeta,
+    ) -> OKIResult<KmerPartition> {
+        let output = output.as_ref();
+        fs::create_dir_all(output).map_err(OKIError::Io)?;
+        meta.write(output).map_err(OKIError::Io)?;
+        fs::create_dir_all(output.join(PARTITIONS_SUBDIR)).map_err(OKIError::Io)?;
+        Ok(KmerPartition::open_with_config(
+            output,
+            meta.config.kmer_size,
+            meta.config.minimizer_size,
+            meta.config.n_bits,
+        )?)
+    }
+
+    /// Mark `output` as fully indexed, pack its column matrices, and reopen it.
+    ///
+    /// Shared tail of the `select`/`rebuild` construction paths, once their
+    /// partitions have been written.
+    pub(crate) fn finalize_indexed<P: AsRef<Path>>(
+        output: P,
+        rep: &mut Reporter,
+    ) -> OKIResult<Self> {
+        let output = output.as_ref();
+        fs::File::create(output.join(SENTINEL_INDEXED)).map_err(OKIError::Io)?;
+        let idx = KmerIndex::open(output)?;
+        let t_pack = Stage::start("pack");
+        idx.pack_matrices(false)?;
+        rep.push(t_pack.stop());
+        Ok(idx)
+    }
+
     /// Current construction state, as reported by sentinel files on disk.
     pub fn state(&self) -> IndexState {
         IndexState::detect(&self.root_path).unwrap_or(IndexState::Empty)
     }
 
+    /// The index's root directory — needed by out-of-crate extension code
+    /// (e.g. `obikphylo`) that opens its own `KmerPartition` handle onto
+    /// the same on-disk index.
+    pub fn root_path(&self) -> &Path { &self.root_path }
     pub fn meta(&self) -> &IndexMeta { &self.meta }
     pub fn meta_mut(&mut self) -> &mut IndexMeta { &mut self.meta }
     pub fn kmer_size(&self) -> usize { self.meta.config.kmer_size }
     pub fn minimizer_size(&self) -> usize { self.meta.config.minimizer_size }
     pub fn n_partitions(&self) -> usize { self.partition.n_partitions() }
 
+    /// Number of layers per partition.
+    ///
+    /// Structural property of the index, fixed at build time and
+    /// homogeneous across all partitions — reading it off partition 0
+    /// is enough, no need to scan every partition.
+    pub fn n_layers_per_partition(&self) -> OKIResult<usize> {
+        use obilayeredmap::meta::PartitionMeta;
+        let index_dir = self.partition.part_dir(0).join("index");
+        let meta = PartitionMeta::load(&index_dir)
+            .map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
+        Ok(meta.n_layers)
+    }
+
     /// Expose the inner partition so the caller can run scatter into it.
     /// Call `mark_scattered` once scatter is complete.
     pub fn partition_mut(&mut self) -> &mut KmerPartition {
@@ -198,8 +274,14 @@ impl KmerIndex {
     ///
     /// Reduces per-query file-open overhead from O(n_genomes) to O(1) per partition.
     /// Column files are kept in place; packed files take priority when opening.
-    pub fn pack_matrices(&self) -> OKIResult<()> {
-        use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix};
+    ///
+    /// If `sparse` is set, presence matrices go one step further, from the
+    /// dense `.pbmx` form into `obicompactvec::PersistentSparseBitMatrix`'s
+    /// on-disk format (see `docmd/architecture/siblings.md`) — count
+    /// matrices are unaffected, sparse count matrices aren't implemented
+    /// (see the sparse-matrix design plan's "Explicitly deferred").
+    pub fn pack_matrices(&self, sparse: bool) -> OKIResult<()> {
+        use obicompactvec::{pack_bit_matrix, pack_compact_int_matrix, pack_sparse_bit_matrix};
         use obilayeredmap::meta::PartitionMeta;
 
         let n = self.n_partitions();
@@ -216,7 +298,13 @@ impl KmerIndex {
                     let layer_dir = index_dir.join(format!("layer_{l}"));
                     let presence_dir = layer_dir.join("presence");
                     let counts_dir   = layer_dir.join("counts");
-                    if presence_dir.exists() { pack_bit_matrix(&presence_dir).map_err(OKIError::Io)?; }
+                    if presence_dir.exists() {
+                        if sparse {
+                            pack_sparse_bit_matrix(&presence_dir).map_err(OKIError::Io)?;
+                        } else {
+                            pack_bit_matrix(&presence_dir).map_err(OKIError::Io)?;
+                        }
+                    }
                     if counts_dir.exists()   { pack_compact_int_matrix(&counts_dir).map_err(OKIError::Io)?; }
                 }
                 Ok(())

src/obikindex/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod error;
 pub mod meta;
+pub mod predicate;
 pub mod state;
 mod distance;
 mod dump;
@@ -9,7 +10,6 @@ mod numa;
 mod rebuild;
 mod reindex;
 mod select;
-mod siblings;
 mod stats;
 
 pub use error::{OKIError, OKIResult};
@@ -17,6 +17,7 @@ pub use distance::{DistanceMetric, DistanceOutput};
 pub use index::KmerIndex;
 pub use merge::MergeMode;
 pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME};
+pub use predicate::{GroupFilterParams, MetaPred};
 pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED};
 pub use stats::IndexBitsPerKmer;
-pub use siblings::{RawSnpDistanceOutput, SiblingAnnexStats, SnpAlignment};
+pub use numa::PartitionRunner;

src/obikindex/src/merge.rs

@@ -130,16 +130,7 @@ impl KmerIndex {
         let (source_labels, all_genomes) = compute_labels(sources, rename_duplicates)?;
 
         // ── Prepare output directory ──────────────────────────────────────────
-        if output.exists() {
-            if force {
-                fs::remove_dir_all(output)?;
-            } else {
-                return Err(OKIError::Io(io::Error::new(
-                    io::ErrorKind::AlreadyExists,
-                    format!("{}: output directory already exists", output.display()),
-                )));
-            }
-        }
+        KmerIndex::clear_output_for_create(output, force)?;
 
         // ── Bootstrap: copy first source to output ────────────────────────────
         info!(
@@ -258,7 +249,7 @@ impl KmerIndex {
             let pb = spinner("pack");
             pb.set_message("consolidating column files …");
             let dst2 = KmerIndex::open(output)?;
-            dst2.pack_matrices()?;
+            dst2.pack_matrices(false)?;
             pb.finish_and_clear();
             rep.push(t.stop());
         }

src/obikindex/src/numa/mod.rs

@@ -0,0 +1,17 @@
+//! NUMA-aware partition runner via hwlocality.
+//!
+//! Detects NUMA topology using hwloc (cross-platform: Linux, macOS, etc.) and
+//! builds one Rayon ThreadPool per NUMA node with threads pinned to that node's
+//! CPUs. Linux first-touch policy then places graph allocations in local DRAM
+//! automatically — no explicit memory binding needed.
+//!
+//! UMA systems (single socket, Apple Silicon, etc.) are the degenerate case:
+//! one synthetic node containing all cores, no pool, no pinning.
+//!
+//! Submodules: [`topology`] (NUMA detection, per-node pools, thread pinning),
+//! [`runner`] ([`PartitionRunner`], the adaptive worker-activation scheduler).
+
+mod runner;
+mod topology;
+
+pub use runner::PartitionRunner;

src/obikindex/src/numa/runner.rs

@@ -1,141 +1,11 @@
-// NUMA-aware partition runner via hwlocality.
-//
-// Detects NUMA topology using hwloc (cross-platform: Linux, macOS, etc.) and
-// builds one Rayon ThreadPool per NUMA node with threads pinned to that node's
-// CPUs. Linux first-touch policy then places graph allocations in local DRAM
-// automatically — no explicit memory binding needed.
-//
-// UMA systems (single socket, Apple Silicon, etc.) are the degenerate case:
-// one synthetic node containing all cores, no pool, no pinning.
-
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use crossbeam_channel::unbounded;
-#[cfg(feature = "numa")]
-use hwlocality::Topology;
-#[cfg(feature = "numa")]
-use hwlocality::cpu::binding::CpuBindingFlags;
-#[cfg(feature = "numa")]
-use hwlocality::cpu::cpuset::CpuSet;
-#[cfg(feature = "numa")]
-use hwlocality::object::types::ObjectType;
 use obisys::{CpuSample, IoSample};
 use tracing::debug;
 
-// ── Public interface ──────────────────────────────────────────────────────────
-
-pub struct NumaSetup {
-    /// One entry per NUMA node. `None` on UMA systems (no pool, no pinning).
-    pub pools: Vec<Option<Arc<rayon::ThreadPool>>>,
-    /// CPU indices for each NUMA node, in node order.
-    pub cpus_per_node: Vec<Vec<usize>>,
-}
-
-impl NumaSetup {
-    /// Maximum worker slots per node (one per physical core in the node).
-    pub fn workers_per_node(&self) -> usize {
-        self.cpus_per_node
-            .first()
-            .map(|c| c.len().max(1))
-            .unwrap_or(1)
-    }
-}
-
-/// Detect NUMA topology and build per-node Rayon pools.
-/// Always succeeds: falls back to a single synthetic UMA node on failure.
-#[cfg(feature = "numa")]
-pub fn build() -> NumaSetup {
-    if let Ok(topology) = Topology::new() {
-        let nodes: Vec<Vec<usize>> = topology
-            .objects_with_type(ObjectType::NUMANode)
-            .filter_map(|obj| obj.cpuset())
-            .map(|cpuset| {
-                cpuset
-                    .iter_set()
-                    .map(|idx| usize::from(idx))
-                    .collect::<Vec<_>>()
-            })
-            .filter(|v| !v.is_empty())
-            .collect();
-
-        if nodes.len() > 1 {
-            if let Some(pools) = nodes
-                .iter()
-                .map(|cpus| build_pool(cpus).map(|p| Some(Arc::new(p))))
-                .collect::<Option<Vec<_>>>()
-            {
-                debug!(
-                    "NUMA topology: {} node(s), {} core(s)/node",
-                    nodes.len(),
-                    nodes.first().map_or(0, |v| v.len()),
-                );
-                return NumaSetup {
-                    pools,
-                    cpus_per_node: nodes,
-                };
-            }
-        }
-    }
-
-    // UMA fallback: single synthetic node, all cores, no pool, no pinning.
-    let n_cores = std::thread::available_parallelism()
-        .map(|n| n.get())
-        .unwrap_or(1);
-    debug!("UMA: single synthetic node, {} core(s)", n_cores);
-    NumaSetup {
-        pools: vec![None],
-        cpus_per_node: vec![(0..n_cores).collect()],
-    }
-}
-
-#[cfg(not(feature = "numa"))]
-pub fn build() -> NumaSetup {
-    let n_cores = std::thread::available_parallelism()
-        .map(|n| n.get())
-        .unwrap_or(1);
-    debug!("UMA: single synthetic node, {} core(s)", n_cores);
-    NumaSetup {
-        pools: vec![None],
-        cpus_per_node: vec![(0..n_cores).collect()],
-    }
-}
-
-/// Bind the calling thread to `cpu_indices` using hwloc.
-/// Silently returns on any error so the thread still runs, just unbound.
-#[cfg(feature = "numa")]
-pub fn pin_current_thread(cpu_indices: &[usize]) {
-    let Ok(topology) = Topology::new() else {
-        return;
-    };
-    let mut cpuset = CpuSet::new();
-    for &idx in cpu_indices {
-        cpuset.set(idx);
-    }
-    let _ = topology.bind_cpu(&cpuset, CpuBindingFlags::THREAD);
-}
-
-#[cfg(not(feature = "numa"))]
-pub fn pin_current_thread(_cpu_indices: &[usize]) {}
-
-// ── Internal helpers ──────────────────────────────────────────────────────────
-
-#[cfg(feature = "numa")]
-fn build_pool(cpus: &[usize]) -> Option<rayon::ThreadPool> {
-    let cpus = cpus.to_vec();
-    rayon::ThreadPoolBuilder::new()
-        .num_threads(cpus.len())
-        .spawn_handler(move |thread| {
-            let cpus = cpus.clone();
-            std::thread::Builder::new().spawn(move || {
-                pin_current_thread(&cpus);
-                thread.run();
-            })?;
-            Ok(())
-        })
-        .build()
-        .ok()
-}
+use super::topology::{build, pin_current_thread};
 
 // ── PartitionRunner ─────────────────────────────────────────────────────────
 
@@ -212,6 +82,32 @@ impl PartitionRunner {
         Self { nodes }
     }
 
+    /// Like [`new`](Self::new), but caps total worker slots (summed across
+    /// nodes) at `max_total_workers` — split evenly across nodes, each
+    /// further capped by that node's actual core count. For callers whose
+    /// own per-worker closure does further internal parallel work (so the
+    /// natural per-node core count would oversubscribe if used as the
+    /// *outer* degree of parallelism too).
+    pub fn new_capped(max_total_workers: usize) -> Self {
+        let ns = build();
+        let n_nodes = ns.pools.len().max(1);
+        let per_node_cap = (max_total_workers / n_nodes).max(1);
+        debug!(
+            "PartitionRunner (capped): {} node(s) × up to {} worker(s)/node ({} total requested)",
+            n_nodes, per_node_cap, max_total_workers,
+        );
+        let nodes = ns
+            .pools
+            .into_iter()
+            .zip(ns.cpus_per_node)
+            .map(|(pool, cpu_ids)| {
+                let node_cores = cpu_ids.len().max(1);
+                NodeConfig { pool, cpu_ids, max_workers: per_node_cap.min(node_cores) }
+            })
+            .collect();
+        Self { nodes }
+    }
+
     /// Run `f(i)` for every index in `order`.
     ///
     /// Workers are pre-spawned dormant and activated adaptively, per node:
@@ -299,20 +195,27 @@ impl PartitionRunner {
                     let pool = node.pool.clone();
 
                     s.spawn(move || {
+                        let tid = std::thread::current().id();
+                        debug!(?tid, "PartitionRunner worker: waiting on activation");
                         if arx.recv().is_err() {
+                            debug!(?tid, "PartitionRunner worker: activation channel closed, exiting");
                             return;
                         }
+                        debug!(?tid, "PartitionRunner worker: activated");
                         if !cpu_ids.is_empty() {
                             pin_current_thread(cpu_ids);
                         }
                         for i in &prx {
+                            debug!(?tid, partition = i, "PartitionRunner worker: picked partition");
                             let t = Instant::now();
                             let r = match &pool {
                                 Some(p) => p.install(|| f(i)),
                                 None => f(i),
                             };
+                            debug!(?tid, partition = i, "PartitionRunner worker: partition done");
                             etx.send(WorkerEvent::Completed(i, r, t.elapsed())).ok();
                         }
+                        debug!(?tid, "PartitionRunner worker: no more partitions, exiting");
                     });
                 }
             }
@@ -323,13 +226,18 @@ impl PartitionRunner {
             // ── Controller ────────────────────────────────────────────────────
             let mut activation = NodeActivation::new(&activate_txs, &node_caps, max_workers);
             activation.activate_initial(INITIAL_DIVISOR, n_total);
+            debug!(n_total, activated = activation.total(), "PartitionRunner controller: initial activation");
 
             let mut cpu_sample = CpuSample::now();
             let mut io_sample = IoSample::now();
             let mut completed = 0usize;
 
             while completed < n_total {
-                let Ok(event) = event_rx.recv() else { break };
+                debug!(completed, n_total, "PartitionRunner controller: waiting for an event");
+                let Ok(event) = event_rx.recv() else {
+                    debug!("PartitionRunner controller: event channel closed, stopping");
+                    break;
+                };
                 match event {
                     WorkerEvent::Completed(i, r, dur) => {
                         match r {

src/obikindex/src/numa/topology.rs

@@ -0,0 +1,121 @@
+use std::sync::Arc;
+
+#[cfg(feature = "numa")]
+use hwlocality::Topology;
+#[cfg(feature = "numa")]
+use hwlocality::cpu::binding::CpuBindingFlags;
+#[cfg(feature = "numa")]
+use hwlocality::cpu::cpuset::CpuSet;
+#[cfg(feature = "numa")]
+use hwlocality::object::types::ObjectType;
+use tracing::debug;
+
+// ── Public interface ──────────────────────────────────────────────────────────
+
+pub struct NumaSetup {
+    /// One entry per NUMA node. `None` on UMA systems (no pool, no pinning).
+    pub pools: Vec<Option<Arc<rayon::ThreadPool>>>,
+    /// CPU indices for each NUMA node, in node order.
+    pub cpus_per_node: Vec<Vec<usize>>,
+}
+
+impl NumaSetup {
+    /// Maximum worker slots per node (one per physical core in the node).
+    pub fn workers_per_node(&self) -> usize {
+        self.cpus_per_node
+            .first()
+            .map(|c| c.len().max(1))
+            .unwrap_or(1)
+    }
+}
+
+/// Detect NUMA topology and build per-node Rayon pools.
+/// Always succeeds: falls back to a single synthetic UMA node on failure.
+#[cfg(feature = "numa")]
+pub fn build() -> NumaSetup {
+    if let Ok(topology) = Topology::new() {
+        let nodes: Vec<Vec<usize>> = topology
+            .objects_with_type(ObjectType::NUMANode)
+            .filter_map(|obj| obj.cpuset())
+            .map(|cpuset| {
+                cpuset
+                    .iter_set()
+                    .map(|idx| usize::from(idx))
+                    .collect::<Vec<_>>()
+            })
+            .filter(|v| !v.is_empty())
+            .collect();
+
+        if nodes.len() > 1 {
+            if let Some(pools) = nodes
+                .iter()
+                .map(|cpus| build_pool(cpus).map(|p| Some(Arc::new(p))))
+                .collect::<Option<Vec<_>>>()
+            {
+                debug!(
+                    "NUMA topology: {} node(s), {} core(s)/node",
+                    nodes.len(),
+                    nodes.first().map_or(0, |v| v.len()),
+                );
+                return NumaSetup {
+                    pools,
+                    cpus_per_node: nodes,
+                };
+            }
+        }
+    }
+
+    // UMA fallback: single synthetic node, all cores, no pool, no pinning.
+    let n_cores = obisys::effective_parallelism();
+    debug!("UMA: single synthetic node, {} core(s)", n_cores);
+    NumaSetup {
+        pools: vec![None],
+        cpus_per_node: vec![(0..n_cores).collect()],
+    }
+}
+
+#[cfg(not(feature = "numa"))]
+pub fn build() -> NumaSetup {
+    let n_cores = obisys::effective_parallelism();
+    debug!("UMA: single synthetic node, {} core(s)", n_cores);
+    NumaSetup {
+        pools: vec![None],
+        cpus_per_node: vec![(0..n_cores).collect()],
+    }
+}
+
+/// Bind the calling thread to `cpu_indices` using hwloc.
+/// Silently returns on any error so the thread still runs, just unbound.
+#[cfg(feature = "numa")]
+pub fn pin_current_thread(cpu_indices: &[usize]) {
+    let Ok(topology) = Topology::new() else {
+        return;
+    };
+    let mut cpuset = CpuSet::new();
+    for &idx in cpu_indices {
+        cpuset.set(idx);
+    }
+    let _ = topology.bind_cpu(&cpuset, CpuBindingFlags::THREAD);
+}
+
+#[cfg(not(feature = "numa"))]
+pub fn pin_current_thread(_cpu_indices: &[usize]) {}
+
+// ── Internal helpers ──────────────────────────────────────────────────────────
+
+#[cfg(feature = "numa")]
+fn build_pool(cpus: &[usize]) -> Option<rayon::ThreadPool> {
+    let cpus = cpus.to_vec();
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(cpus.len())
+        .spawn_handler(move |thread| {
+            let cpus = cpus.clone();
+            std::thread::Builder::new().spawn(move || {
+                pin_current_thread(&cpus);
+                thread.run();
+            })?;
+            Ok(())
+        })
+        .build()
+        .ok()
+}

src/obikindex/src/predicate.rs

@@ -0,0 +1,252 @@
+use std::collections::HashMap;
+
+use obikpartitionner::GroupQuorumFilter;
+use obitaxonomy::{TaxPath, TaxPattern};
+
+use crate::meta::{GenomeInfo, IndexMeta};
+
+// ── Operator ──────────────────────────────────────────────────────────────────
+
+enum PredOp { Wildcard, Eq, Ne, Matches, NotMatches }
+
+// ── MetaPred ──────────────────────────────────────────────────────────────────
+
+/// A single predicate on genome metadata: `key OP val1|val2|…`
+///
+/// Operators: `=` (exact), `!=` (not equal), `~` (path ancestor), `!~` (not ancestor).
+/// Multiple values separated by `|` are OR'd.
+pub struct MetaPred {
+    key:    String,
+    op:     PredOp,
+    values: Vec<String>,
+}
+
+impl MetaPred {
+    /// Parse a predicate string of the form `key=v1|v2`, `key!=v`, `key~path`, `key!~path`.
+    /// The special values `*` and `all` (case-insensitive) match every genome.
+    pub fn parse(s: &str) -> Result<Self, String> {
+        let t = s.trim();
+        if t == "*" || t.eq_ignore_ascii_case("all") {
+            return Ok(Self { key: String::new(), op: PredOp::Wildcard, values: vec![] });
+        }
+
+        let (op, key, rhs) =
+            if let Some(pos) = s.find("!=") {
+                (PredOp::Ne,         &s[..pos], &s[pos+2..])
+            } else if let Some(pos) = s.find("!~") {
+                (PredOp::NotMatches, &s[..pos], &s[pos+2..])
+            } else if let Some(pos) = s.find('=') {
+                (PredOp::Eq,         &s[..pos], &s[pos+1..])
+            } else if let Some(pos) = s.find('~') {
+                (PredOp::Matches,    &s[..pos], &s[pos+1..])
+            } else {
+                return Err(format!("no operator found in predicate: {s}"));
+            };
+
+        let key = key.trim().to_string();
+        if key.is_empty() { return Err(format!("empty key in predicate: {s}")); }
+
+        let values: Vec<String> = rhs.split('|').map(|v| v.trim().to_string()).collect();
+        if values.iter().any(|v| v.is_empty()) {
+            return Err(format!("empty value in predicate: {s}"));
+        }
+        Ok(Self { key, op, values })
+    }
+
+    /// Evaluate against one genome's metadata.
+    /// Returns `None` when the key is absent (NA propagation).
+    pub(crate) fn eval(&self, meta: &HashMap<String, String>) -> Option<bool> {
+        if matches!(self.op, PredOp::Wildcard) { return Some(true); }
+        let value = meta.get(&self.key)?;
+        Some(match self.op {
+            PredOp::Wildcard   => unreachable!(),
+            PredOp::Eq         => self.values.iter().any(|v| v == value),
+            PredOp::Ne         => self.values.iter().all(|v| v != value),
+            PredOp::Matches    => self.values.iter().any(|v| path_matches(value, v)),
+            PredOp::NotMatches => self.values.iter().all(|v| !path_matches(value, v)),
+        })
+    }
+}
+
+impl GenomeInfo {
+    /// Evaluate a single metadata predicate against this genome.
+    /// Returns `None` when the predicate's key is absent (NA propagation).
+    pub fn matches(&self, pred: &MetaPred) -> Option<bool> {
+        pred.eval(&self.meta)
+    }
+}
+
+// ── Path matching ─────────────────────────────────────────────────────────────
+
+/// True if the stored taxonomy `value` matches `pattern`.
+///
+/// `value` must be a valid `TaxPath` (starts with `taxonomy:/`).
+/// `pattern` is a `TaxPattern` query (see `obitaxonomy::TaxPattern` for syntax).
+/// Returns `false` if either fails to parse.
+fn path_matches(value: &str, pattern: &str) -> bool {
+    let Ok(path) = TaxPath::parse(value)    else { return false };
+    let Ok(pat)  = TaxPattern::parse(pattern) else { return false };
+    pat.matches(&path)
+}
+
+// ── Three-value group evaluation ──────────────────────────────────────────────
+
+/// AND of all predicates (ingroup semantics).
+/// Short-circuits on `Some(false)`; propagates `None` if no predicate returns `false`.
+fn eval_and(preds: &[MetaPred], meta: &HashMap<String, String>) -> Option<bool> {
+    let mut has_na = false;
+    for pred in preds {
+        match pred.eval(meta) {
+            Some(false) => return Some(false),
+            Some(true)  => {}
+            None        => has_na = true,
+        }
+    }
+    if has_na { None } else { Some(true) }
+}
+
+/// OR of all predicates (outgroup semantics).
+/// Short-circuits on `Some(true)`; propagates `None` if no predicate returns `true`.
+fn eval_or(preds: &[MetaPred], meta: &HashMap<String, String>) -> Option<bool> {
+    let mut has_na = false;
+    for pred in preds {
+        match pred.eval(meta) {
+            Some(true)  => return Some(true),
+            Some(false) => {}
+            None        => has_na = true,
+        }
+    }
+    if has_na { None } else { Some(false) }
+}
+
+// ── Genome classification ─────────────────────────────────────────────────────
+
+enum Membership { Ingroup, Outgroup, Uncategorized }
+
+fn classify(
+    genomes: &[GenomeInfo],
+    ingroup:  &[MetaPred],
+    outgroup: &[MetaPred],
+) -> Vec<Membership> {
+    genomes.iter().map(|g| {
+        let in_r  = if ingroup.is_empty()  { None } else { eval_and(ingroup,  &g.meta) };
+        let out_r = if outgroup.is_empty() { None } else { eval_or(outgroup,  &g.meta) };
+
+        // Ingroup wins over outgroup.
+        if in_r == Some(true)  { return Membership::Ingroup;  }
+        if out_r == Some(true) { return Membership::Outgroup; }
+        Membership::Uncategorized
+    }).collect()
+}
+
+// ── Group quorum filter construction ──────────────────────────────────────────
+
+pub struct GroupFilterParams {
+    pub threshold:          u32,
+    pub min_count:          Option<isize>,
+    pub max_count:          Option<isize>,
+    pub min_frac:           Option<f64>,
+    pub max_frac:           Option<f64>,
+    pub min_outgroup_count: Option<isize>,
+    pub max_outgroup_count: Option<isize>,
+    pub min_outgroup_frac:  Option<f64>,
+    pub max_outgroup_frac:  Option<f64>,
+}
+
+impl IndexMeta {
+    /// Returns indices of genomes matching `pred_str` (single predicate).
+    pub fn matching_genome_indices(&self, pred_str: &str) -> Result<Vec<usize>, String> {
+        let pred = MetaPred::parse(pred_str)?;
+        Ok(self.genomes.iter().enumerate()
+            .filter_map(|(i, g)| {
+                if g.matches(&pred) == Some(true) { Some(i) } else { std::option::Option::None }
+            })
+            .collect())
+    }
+
+    /// Build a `GroupQuorumFilter` from parsed predicates, evaluated against `self.genomes`.
+    ///
+    /// - No groups defined: `ingroup_idx` = all genomes (implicit ingroup).
+    /// - `ingroup` predicates only: outgroup indices are empty.
+    /// - `outgroup` predicates only: ingroup indices are empty.
+    /// - Both defined: ingroup wins on overlap; uncategorized genomes are ignored.
+    pub fn build_group_filter(
+        &self,
+        ingroup_preds:  &[MetaPred],
+        outgroup_preds: &[MetaPred],
+        p:              GroupFilterParams,
+    ) -> Result<GroupQuorumFilter, String> {
+        let (ingroup_idx, outgroup_idx) = if ingroup_preds.is_empty() && outgroup_preds.is_empty() {
+            ((0..self.genomes.len()).collect(), vec![])
+        } else {
+            let members = classify(&self.genomes, ingroup_preds, outgroup_preds);
+            let in_idx: Vec<usize>  = members.iter().enumerate()
+                .filter(|(_, m)| matches!(m, Membership::Ingroup))
+                .map(|(i, _)| i).collect();
+            let out_idx: Vec<usize> = members.iter().enumerate()
+                .filter(|(_, m)| matches!(m, Membership::Outgroup))
+                .map(|(i, _)| i).collect();
+            (in_idx, out_idx)
+        };
+
+        let in_size  = ingroup_idx.len();
+        let out_size = outgroup_idx.len();
+
+        let ingroup_quorum_explicit  = p.min_count.is_some() || p.max_count.is_some()
+            || p.min_frac.is_some() || p.max_frac.is_some();
+        let outgroup_quorum_explicit = p.min_outgroup_count.is_some() || p.max_outgroup_count.is_some()
+            || p.min_outgroup_frac.is_some() || p.max_outgroup_frac.is_some();
+
+        let default_min_frac           = if !ingroup_preds.is_empty()  && !ingroup_quorum_explicit  { 1.0      } else { 0.0      };
+        let default_max_outgroup_count = if !outgroup_preds.is_empty() && !outgroup_quorum_explicit { 0        } else { out_size };
+
+        // Resolve a signed count: negative means an offset from the group size
+        // (e.g. -1 = all but one), floored at 1 so the negative form always keeps
+        // constraining the group — even a singleton group, where n-1 would be 0
+        // and would otherwise drop the constraint entirely.
+        let resolve = |v: isize, size: usize| -> usize {
+            if v < 0 { (size as isize + v).max(1) as usize } else { v as usize }
+        };
+
+        let min_count          = p.min_count.map(|v| resolve(v, in_size)).unwrap_or(0);
+        let max_count          = p.max_count.map(|v| resolve(v, in_size)).unwrap_or(in_size);
+        let min_frac           = p.min_frac.unwrap_or(default_min_frac);
+        let max_frac           = p.max_frac.unwrap_or(1.0);
+        let min_outgroup_count = p.min_outgroup_count.map(|v| resolve(v, out_size)).unwrap_or(0);
+        let max_outgroup_count = p.max_outgroup_count.map(|v| resolve(v, out_size)).unwrap_or(default_max_outgroup_count);
+        let min_outgroup_frac  = p.min_outgroup_frac.unwrap_or(0.0);
+        let max_outgroup_frac  = p.max_outgroup_frac.unwrap_or(1.0);
+
+        for (v, lo, hi) in [
+            ("--min-frac/--max-frac",                        min_frac,           max_frac),
+            ("--min-outgroup-frac/--max-outgroup-frac",       min_outgroup_frac,  max_outgroup_frac),
+        ] {
+            if !(0.0..=1.0).contains(&lo) || !(0.0..=1.0).contains(&hi) {
+                return Err(format!("{v}: fraction values must be in [0.0, 1.0]"));
+            }
+            if lo > hi {
+                return Err(format!("{v}: min ({lo}) is greater than max ({hi})"));
+            }
+        }
+        if min_count > max_count {
+            return Err(format!("--min-count/--max-count: min ({min_count}) is greater than max ({max_count})"));
+        }
+        if min_outgroup_count > max_outgroup_count {
+            return Err(format!("--min-outgroup-count/--max-outgroup-count: min ({min_outgroup_count}) is greater than max ({max_outgroup_count})"));
+        }
+
+        Ok(GroupQuorumFilter {
+            ingroup_idx,
+            outgroup_idx,
+            threshold: p.threshold,
+            min_count,
+            max_count,
+            min_frac,
+            max_frac,
+            min_outgroup_count,
+            max_outgroup_count,
+            min_outgroup_frac,
+            max_outgroup_frac,
+        })
+    }
+}

src/obikindex/src/rebuild.rs

@@ -1,15 +1,13 @@
-use std::fs;
-use std::io;
 use std::path::Path;
 
-use obikpartitionner::{KmerFilter, KmerPartition, MergeMode};
+use obikpartitionner::{KmerFilter, MergeMode};
 use obisys::{Reporter, Stage, progress_bar};
 use tracing::info;
 
 use crate::error::{OKIError, OKIResult};
 use crate::index::KmerIndex;
 use crate::meta::IndexMeta;
-use crate::state::{IndexState, SENTINEL_INDEXED};
+use crate::state::IndexState;
 
 impl KmerIndex {
     /// Rebuild `src` into a new compact single-layer index at `output`.
@@ -40,36 +38,18 @@ impl KmerIndex {
             ));
         }
 
-        if output.exists() {
-            if force {
-                fs::remove_dir_all(output)?;
-            } else {
-                return Err(OKIError::Io(io::Error::new(
-                    io::ErrorKind::AlreadyExists,
-                    format!("{}: output directory already exists", output.display()),
-                )));
-            }
-        }
+        KmerIndex::clear_output_for_create(output, force)?;
 
         // ── Create output directory + metadata ────────────────────────────────
-        fs::create_dir_all(output)?;
         let mut meta = IndexMeta::new(src.meta.config.clone());
         meta.config.with_counts = mode == MergeMode::Count;
         meta.genomes = src.meta.genomes.clone();
-        meta.write(output)?;
 
         let n_genomes = src.meta.genomes.len();
         let n_partitions = src.partition.n_partitions();
 
         // ── Create an empty destination KmerPartition ─────────────────────────
-        // Create the partitions/ subdirectory so KmerPartition::open_with_config works.
-        fs::create_dir_all(output.join(obikpartitionner::PARTITIONS_SUBDIR))?;
-        let dst_partition = KmerPartition::open_with_config(
-            output,
-            meta.config.kmer_size,
-            meta.config.minimizer_size,
-            meta.config.n_bits,
-        )?;
+        let dst_partition = KmerIndex::create_skeleton(output, &meta)?;
 
         info!(
             "rebuild: {} partition(s), {} genome(s), mode={:?}",
@@ -94,13 +74,6 @@ impl KmerIndex {
 
         rep.push(t.stop());
 
-        // Write SENTINEL_INDEXED — output is ready to use.
-        fs::File::create(output.join(SENTINEL_INDEXED))?;
-
-        let idx = KmerIndex::open(output)?;
-        let t_pack = Stage::start("pack");
-        idx.pack_matrices()?;
-        rep.push(t_pack.stop());
-        Ok(idx)
+        KmerIndex::finalize_indexed(output, rep)
     }
 }

src/obikindex/src/select.rs

@@ -1,15 +1,13 @@
-use std::fs;
-use std::io;
 use std::path::Path;
 
-use obikpartitionner::{KmerPartition, OutputCol, PARTITIONS_SUBDIR};
+use obikpartitionner::{KmerPartition, OutputCol};
 use obisys::{Reporter, Stage, progress_bar};
 use tracing::info;
 
 use crate::error::{OKIError, OKIResult};
 use crate::index::KmerIndex;
 use crate::meta::{GenomeInfo, IndexMeta};
-use crate::state::{IndexState, SENTINEL_INDEXED};
+use crate::state::IndexState;
 
 impl KmerIndex {
     /// Create a new index at `output` by projecting/aggregating the genome columns
@@ -33,35 +31,18 @@ impl KmerIndex {
             return Err(OKIError::NotIndexed(src.root_path.clone()));
         }
 
-        if output.exists() {
-            if force {
-                fs::remove_dir_all(output)?;
-            } else {
-                return Err(OKIError::Io(io::Error::new(
-                    io::ErrorKind::AlreadyExists,
-                    format!("{}: output directory already exists", output.display()),
-                )));
-            }
-        }
+        KmerIndex::clear_output_for_create(output, force)?;
 
-        fs::create_dir_all(output)?;
         let mut meta = IndexMeta::new(src.meta.config.clone());
         meta.config.with_counts = !output_presence;
         meta.genomes = specs.iter()
             .map(|s| GenomeInfo::new(s.label.clone()))
             .collect();
-        meta.write(output)?;
 
         let n_src_genomes  = src.meta.genomes.len();
         let n_partitions   = src.partition.n_partitions();
 
-        fs::create_dir_all(output.join(PARTITIONS_SUBDIR))?;
-        let dst_partition = KmerPartition::open_with_config(
-            output,
-            meta.config.kmer_size,
-            meta.config.minimizer_size,
-            meta.config.n_bits,
-        )?;
+        let dst_partition = KmerIndex::create_skeleton(output, &meta)?;
 
         info!(
             "select: {} partition(s), {} source genome(s) → {} output column(s)",
@@ -83,13 +64,7 @@ impl KmerIndex {
         pb.finish_and_clear();
         rep.push(t.stop());
 
-        fs::File::create(output.join(SENTINEL_INDEXED))?;
-
-        let idx = KmerIndex::open(output)?;
-        let t_pack = Stage::start("pack");
-        idx.pack_matrices()?;
-        rep.push(t_pack.stop());
-        Ok(idx)
+        KmerIndex::finalize_indexed(output, rep)
     }
 
     /// Rewrite the genome columns of this index in-place according to `specs`.
@@ -143,7 +118,7 @@ impl KmerIndex {
         self.meta.write(&self.root_path)?;
 
         let t_pack = Stage::start("pack");
-        self.pack_matrices()?;
+        self.pack_matrices(false)?;
         rep.push(t_pack.stop());
         Ok(())
     }

src/obikmer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "obikmer"
-version = "1.1.40"
+version = "1.1.44"
 edition = "2024"
 
 [[bin]]
@@ -19,18 +19,21 @@ obikpartitionner = { path = "../obikpartitionner" }
 obisys        = { path = "../obisys" }
 obiskio       = { path = "../obiskio" }
 obikindex     = { path = "../obikindex", default-features = false }
+obikphylo     = { path = "../obikphylo" }
 obitaxonomy   = { path = "../obitaxonomy" }
 obilayeredmap = { path = "../obilayeredmap" }
 clap          = { version = "4", features = ["derive"] }
+serde         = { version = "1", features = ["derive"] }
 serde_json    = "1"
+serde_yaml    = "0.9.33"
 csv           = "1"
-kodama        = "0.2"
+kodama        = "0.3.0"
 speedytree    = "0.1"
 rayon         = "1"
-indicatif     = "0.17"
+indicatif     = "0.18"
 tracing       = "0.1.44"
 tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
-pprof = { version = "0.13", features = ["prost-codec"], optional = true }
+pprof = { version = "0.15", features = ["prost-codec"], optional = true }
 
 [features]
 default   = ["numa"]

src/obikmer/src/cli.rs

@@ -38,9 +38,7 @@ pub struct CommonArgs {
     #[arg(
         short = 'T',
         long,
-        default_value_t = std::thread::available_parallelism()
-            .map(|n| n.get())
-            .unwrap_or(1)
+        default_value_t = obisys::effective_parallelism()
     )]
     pub threads: usize,
 

src/obikmer/src/cmd/distance.rs

@@ -1,392 +0,0 @@
-use std::io::{self, BufWriter, Write};
-use std::path::PathBuf;
-
-use clap::Args;
-use kodama::{Method, linkage};
-use obifastwrite::{JsonVal, write_record};
-use obikindex::{DistanceMetric, KmerIndex, RawSnpDistanceOutput, SiblingAnnexStats, SnpAlignment};
-use speedytree::{DistanceMatrix, Hybrid, NeighborJoiningSolver, to_newick};
-use tracing::info;
-
-#[derive(clap::ValueEnum, Clone, Copy, Debug)]
-pub enum MetricArg {
-    Jaccard,
-    Mash,
-    Hamming,
-    BrayCurtis,
-    #[value(name = "relfreq-bray-curtis")]
-    RelfreqBrayCurtis,
-    Euclidean,
-    #[value(name = "relfreq-euclidean")]
-    RelfreqEuclidean,
-    Hellinger,
-    #[value(name = "hellinger-euclidean")]
-    HellingerEuclidean,
-}
-
-impl From<MetricArg> for DistanceMetric {
-    fn from(m: MetricArg) -> Self {
-        match m {
-            MetricArg::Jaccard            => DistanceMetric::Jaccard,
-            MetricArg::Mash               => DistanceMetric::Mash,
-            MetricArg::Hamming            => DistanceMetric::Hamming,
-            MetricArg::BrayCurtis         => DistanceMetric::BrayCurtis,
-            MetricArg::RelfreqBrayCurtis  => DistanceMetric::RelfreqBrayCurtis,
-            MetricArg::Euclidean          => DistanceMetric::Euclidean,
-            MetricArg::RelfreqEuclidean   => DistanceMetric::RelfreqEuclidean,
-            MetricArg::Hellinger          => DistanceMetric::Hellinger,
-            MetricArg::HellingerEuclidean => DistanceMetric::HellingerEuclidean,
-        }
-    }
-}
-
-#[derive(Args)]
-pub struct DistanceArgs {
-    /// Index directory
-    pub index: PathBuf,
-
-    /// Distance metric to compute
-    #[arg(long, value_enum, default_value = "jaccard")]
-    pub metric: MetricArg,
-
-    /// Minimum count to consider a kmer present when computing Jaccard on count indexes
-    #[arg(long, default_value = "1")]
-    pub presence_threshold: u32,
-
-    /// Also output the shared-kmer count matrix (CSV)
-    #[arg(long)]
-    pub shared_kmers: bool,
-
-    /// Compute and write a Neighbor-Joining tree (Newick)
-    #[arg(long)]
-    pub nj: bool,
-
-    /// Compute and write a UPGMA tree (Newick)
-    #[arg(long)]
-    pub upgma: bool,
-
-    /// Build the sibling-count/minorant annex on this (multi-genome) index
-    /// — see `docmd/theory/evolutionary_distances.md`, Step 2b. Construction
-    /// only; does not by itself compute or write any statistics.
-    #[arg(long)]
-    pub sibling_annex: bool,
-
-    /// Tally the sibling-count distribution (CSV) of an already-built annex
-    /// (run with `--sibling-annex` first, in this invocation or an earlier
-    /// one). A separate, occasional diagnostic pass — not run every time the
-    /// annex itself is (re)built.
-    #[arg(long)]
-    pub sibling_stats: bool,
-
-    /// Compute the raw p-distance restricted to loci that are single-copy
-    /// in both genomes of each pair (an already-built sibling annex is
-    /// required — run with `--sibling-annex` first, in this invocation or
-    /// an earlier one). A quick way to test the central-position SNP
-    /// estimator against a real index; not the full `SnpTally` design.
-    #[arg(long)]
-    pub raw_snp_distance: bool,
-
-    /// Write a SNP-only pseudo-alignment (FASTA, IUPAC-coded) from an
-    /// already-built sibling annex — one row per genome, one column per
-    /// variable family (monomorphic families skipped), no flanking
-    /// sequence. See `docmd/theory/evolutionary_distances.md`,
-    /// "Multi-genome framing: family as pseudo-alignment column".
-    #[arg(long)]
-    pub snp: bool,
-
-    /// Output prefix: <prefix>_dist.csv, <prefix>_shared.csv,
-    /// <prefix>_siblings.csv, <prefix>_rawsnp.csv, <prefix>_snp.fasta,
-    /// <prefix>_nj.nwk, <prefix>_upgma.nwk.
-    /// If omitted, the distance matrix is written to stdout.
-    #[arg(short, long)]
-    pub output: Option<PathBuf>,
-}
-
-pub fn run(args: DistanceArgs) {
-    let idx = KmerIndex::open(&args.index).unwrap_or_else(|e| {
-        eprintln!("error opening index: {e}");
-        std::process::exit(1);
-    });
-
-
-    let labels: Vec<String> = idx.meta().genomes.iter().map(|g| g.label.clone()).collect();
-    let n = labels.len();
-
-    // ── Sibling-count/minorant annex (independent of the distance metric) ──
-    // Construction (`--sibling-annex`) and stats (`--sibling-stats`) are
-    // deliberately decoupled: the annex is meant to be (re)built routinely,
-    // the distribution only occasionally, on demand.
-    if args.sibling_annex {
-        info!("building sibling-count/minorant annex");
-        idx.build_sibling_annex().unwrap_or_else(|e| {
-            eprintln!("error building sibling annex: {e}");
-            std::process::exit(1);
-        });
-    }
-    if args.sibling_stats {
-        let stats = idx.sibling_annex_stats().unwrap_or_else(|e| {
-            eprintln!("error computing sibling-annex stats: {e}");
-            std::process::exit(1);
-        });
-        write_sibling_stats_csv(&stats, &labels, &args.output);
-    }
-    if args.raw_snp_distance {
-        let result = idx.raw_snp_distance().unwrap_or_else(|e| {
-            eprintln!("error computing raw SNP distance: {e}");
-            std::process::exit(1);
-        });
-        write_raw_snp_distance_csv(&result, &labels, &args.output);
-    }
-    if args.snp {
-        let alignment = idx.snp_pseudo_alignment().unwrap_or_else(|e| {
-            eprintln!("error computing SNP pseudo-alignment: {e}");
-            std::process::exit(1);
-        });
-        write_snp_fasta(&alignment, &labels, &args.output);
-    }
-
-    // `--sibling-annex`/`--sibling-stats`/`--raw-snp-distance`/`--snp` are
-    // their own operation, not a modifier on top of a distance-metric
-    // computation — a metric was never requested by asking for any of them,
-    // so there is nothing for the rest of this function to compute. Not a
-    // historical accident to keep: stop here rather than always also
-    // running a Jaccard (or whichever `--metric` defaults to) pass and
-    // printing an unrequested matrix.
-    if args.sibling_annex || args.sibling_stats || args.raw_snp_distance || args.snp {
-        return;
-    }
-
-    info!(
-        "computing {:?} distances for {} genome(s)",
-        args.metric, n
-    );
-
-    let need_shared = args.shared_kmers || args.nj || args.upgma;
-    let result = idx
-        .distance(args.metric.into(), need_shared, args.presence_threshold)
-        .unwrap_or_else(|e| {
-            eprintln!("error computing distances: {e}");
-            std::process::exit(1);
-        });
-
-    // ── Distance matrix → CSV ─────────────────────────────────────────────────
-    let write_dist_csv = |w: &mut dyn Write| {
-        write!(w, "genome").unwrap();
-        for g in &labels { write!(w, ",{g}").unwrap(); }
-        writeln!(w).unwrap();
-        for (i, g) in labels.iter().enumerate() {
-            write!(w, "{g}").unwrap();
-            for j in 0..n {
-                write!(w, ",{:.6}", result.matrix[[i, j]]).unwrap();
-            }
-            writeln!(w).unwrap();
-        }
-    };
-
-    match &args.output {
-        Some(prefix) => {
-            let path = format!("{}_dist.csv", prefix.display());
-            let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-                eprintln!("error creating {path}: {e}");
-                std::process::exit(1);
-            }));
-            write_dist_csv(&mut f);
-            info!("distance matrix → {path}");
-        }
-        None => {
-            let stdout = io::stdout();
-            let mut out = BufWriter::new(stdout.lock());
-            write_dist_csv(&mut out);
-        }
-    }
-
-    // ── Shared-kmer matrix → CSV ──────────────────────────────────────────────
-    if args.shared_kmers {
-        if let Some(shared) = &result.shared_kmers {
-            let path = args.output.as_ref()
-                .map(|p| format!("{}_shared.csv", p.display()))
-                .unwrap_or_else(|| "shared.csv".into());
-            let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-                eprintln!("error creating {path}: {e}");
-                std::process::exit(1);
-            }));
-            write!(f, "genome").unwrap();
-            for g in &labels { write!(f, ",{g}").unwrap(); }
-            writeln!(f).unwrap();
-            for (i, g) in labels.iter().enumerate() {
-                write!(f, "{g}").unwrap();
-                for j in 0..n { write!(f, ",{}", shared[[i, j]]).unwrap(); }
-                writeln!(f).unwrap();
-            }
-            info!("shared-kmer matrix → {path}");
-        }
-    }
-
-    // ── NJ tree via speedytree ────────────────────────────────────────────────
-    if args.nj {
-        let rows: Vec<Vec<f64>> = (0..n)
-            .map(|i| (0..n).map(|j| result.matrix[[i, j]]).collect())
-            .collect();
-        let dm = DistanceMatrix::build(rows, labels.clone()).unwrap_or_else(|e| {
-            eprintln!("error building distance matrix for NJ: {e}");
-            std::process::exit(1);
-        });
-        let tree = NeighborJoiningSolver::<Hybrid>::default(dm).solve().unwrap_or_else(|e| {
-            eprintln!("error computing NJ tree: {e}");
-            std::process::exit(1);
-        });
-        let newick = to_newick(&tree);
-        let path = args.output.as_ref()
-            .map(|p| format!("{}_nj.nwk", p.display()))
-            .unwrap_or_else(|| "nj.nwk".into());
-        std::fs::write(&path, &newick).unwrap_or_else(|e| {
-            eprintln!("error writing {path}: {e}");
-            std::process::exit(1);
-        });
-        info!("NJ tree → {path}");
-    }
-
-    // ── UPGMA tree via kodama ─────────────────────────────────────────────────
-    if args.upgma {
-        let mut condensed: Vec<f64> = Vec::with_capacity(n * (n - 1) / 2);
-        for i in 0..n {
-            for j in (i + 1)..n {
-                condensed.push(result.matrix[[i, j]]);
-            }
-        }
-        let dendro = linkage(&mut condensed, n, Method::Average);
-        let newick = upgma_to_newick(&dendro, &labels);
-        let path = args.output.as_ref()
-            .map(|p| format!("{}_upgma.nwk", p.display()))
-            .unwrap_or_else(|| "upgma.nwk".into());
-        std::fs::write(&path, &newick).unwrap_or_else(|e| {
-            eprintln!("error writing {path}: {e}");
-            std::process::exit(1);
-        });
-        info!("UPGMA tree → {path}");
-    }
-}
-
-// ── Family-size distribution → CSV ──────────────────────────────────────────
-//
-// Each row is a family (the up-to-4 k-mers sharing flanks, differing only at
-// the centre), counted once — at its minorant — regardless of how many of
-// its members are observed. Family size 1..4 (not "sibling count" 0..3):
-// see `docmd/theory/evolutionary_distances.md`, "Definitions".
-
-fn write_sibling_stats_csv(stats: &SiblingAnnexStats, labels: &[String], output: &Option<PathBuf>) {
-    // One row per genome (4 columns, family size 1-4: number of families of
-    // that size for which the genome carries at least one member), plus a
-    // `global` row — the actual deduplicated family-size histogram
-    // (`stats.counts`), NOT a sum of the per-genome columns (a family shared
-    // by several genomes would otherwise be counted once per genome it
-    // appears in, inflating the total beyond the real family count).
-    let path = output.as_ref()
-        .map(|p| format!("{}_siblings.csv", p.display()))
-        .unwrap_or_else(|| "siblings.csv".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    writeln!(f, "genome,1,2,3,4").unwrap();
-    for (label, counts) in labels.iter().zip(stats.per_genome.iter()) {
-        writeln!(f, "{label},{},{},{},{}", counts[0], counts[1], counts[2], counts[3]).unwrap();
-    }
-    writeln!(
-        f, "global,{},{},{},{}",
-        stats.counts[0], stats.counts[1], stats.counts[2], stats.counts[3],
-    ).unwrap();
-    let total: u64 = stats.counts.iter().sum();
-    info!("family-size distribution → {path} (total {total} famil{})",
-        if total == 1 { "y" } else { "ies" });
-}
-
-// ── Raw single-copy SNP distance → CSV ──────────────────────────────────────
-//
-// p_hat[i,j] = snp[i,j] / (snp[i,j] + shared[i,j]) over loci single-copy in
-// both i and j — see `RawSnpDistanceOutput` / `KmerIndex::raw_snp_distance`.
-// A single file: the distance matrix, with an eligible-loci count alongside
-// each value so a 0/0 pair (no eligible locus at all) is distinguishable
-// from a genuinely identical pair.
-
-fn write_raw_snp_distance_csv(result: &RawSnpDistanceOutput, labels: &[String], output: &Option<PathBuf>) {
-    let path = output.as_ref()
-        .map(|p| format!("{}_rawsnp.csv", p.display()))
-        .unwrap_or_else(|| "rawsnp.csv".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    let n = labels.len();
-    write!(f, "genome").unwrap();
-    for g in labels { write!(f, ",{g}").unwrap(); }
-    writeln!(f).unwrap();
-    for (i, g) in labels.iter().enumerate() {
-        write!(f, "{g}").unwrap();
-        for j in 0..n {
-            let snp = result.snp[[i, j]];
-            let shared = result.shared[[i, j]];
-            let eligible = snp + shared;
-            if eligible == 0 {
-                write!(f, ",NA").unwrap();
-            } else {
-                write!(f, ",{:.6}", snp as f64 / eligible as f64).unwrap();
-            }
-        }
-        writeln!(f).unwrap();
-    }
-    info!("raw single-copy SNP distance matrix → {path}");
-}
-
-// ── SNP-only pseudo-alignment → FASTA ───────────────────────────────────────
-//
-// One record per genome, IUPAC-coded, no flanking sequence — see
-// `SnpAlignment` / `KmerIndex::snp_pseudo_alignment`. Uses the project's
-// existing FASTA writer (`obifastwrite::write_record`) rather than
-// hand-rolling one.
-
-fn write_snp_fasta(alignment: &SnpAlignment, labels: &[String], output: &Option<PathBuf>) {
-    let path = output.as_ref()
-        .map(|p| format!("{}_snp.fasta", p.display()))
-        .unwrap_or_else(|| "snp.fasta".into());
-    let mut f = BufWriter::new(std::fs::File::create(&path).unwrap_or_else(|e| {
-        eprintln!("error creating {path}: {e}");
-        std::process::exit(1);
-    }));
-    let n_sites = alignment.sequences.first().map(|s| s.len()).unwrap_or(0);
-    for (label, seq) in labels.iter().zip(alignment.sequences.iter()) {
-        write_record(seq, label, &[("n_sites", JsonVal::Num(n_sites as u64))], &mut f).unwrap_or_else(|e| {
-            eprintln!("error writing {path}: {e}");
-            std::process::exit(1);
-        });
-    }
-    info!("SNP pseudo-alignment → {path} ({n_sites} site{})",
-        if n_sites == 1 { "" } else { "s" });
-}
-
-// ── UPGMA Newick from kodama dendrogram ───────────────────────────────────────
-
-fn upgma_to_newick(dendro: &kodama::Dendrogram<f64>, names: &[String]) -> String {
-    let n = names.len();
-    // node_labels[i]: Newick subtree string for node i (leaves 0..n, internals n..)
-    let mut labels: Vec<String> = names.to_vec();
-    // height of each node: leaves = 0, internal = dissimilarity/2
-    let mut heights: Vec<f64> = vec![0.0; 2 * n - 1];
-
-    for (k, step) in dendro.steps().iter().enumerate() {
-        let new_node = n + k;
-        let h = step.dissimilarity / 2.0;
-        heights[new_node] = h;
-        let c1 = step.cluster1;
-        let c2 = step.cluster2;
-        let bl1 = (h - heights[c1]).max(0.0);
-        let bl2 = (h - heights[c2]).max(0.0);
-        labels.push(format!(
-            "({label1}:{bl1:.6},{label2}:{bl2:.6})",
-            label1 = labels[c1],
-            label2 = labels[c2],
-        ));
-    }
-
-    format!("{};", labels.last().unwrap())
-}