diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index eeb93388..00000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index b19afb41..00fa7f0e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,36 @@ .venv/ +.DS_Store +.claude/ +.kilo/ +.serena/ +.zed/ +.ast-cache/ +CLAUDE.md +memory/ +sandbox/ src/target data-stress *.fasta *.fasta.gz +*.fastq +*.fastq.gz +*.vcf *.zst *.zst.meta *.pb -./**/*.json +**/*.json *.bin *.log *.csv +*.meta +*.pfiv +*.done +*.efh +*.efl +*.bin.idx +*.prsb Betula_exilis--IGA-24-33 -benchmark/genomes -benchmark/genomes_orig -benchmark/simulated_data -benchmark/specimen_index_presence -benchmark/specimen_index_count -benchmark/global_index_presence -benchmark/global_index_presence_orig -benchmark/global_index_presence_sav -benchmark/all_specific -benchmark/global_index_count -benchmark/stats -benchmark/reference_index -benchmark/reference_dist -benchmark/obikmer_dist -benchmark/specific_index_count -benchmark/specific_index_presence +benchmark/run/ TNT phyg biblio diff --git a/.serena/project.yml b/.serena/project.yml index 1a35e2f9..f40f4750 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,39 +1,6 @@ -# the name by which the project can be referenced within Serena +# the name by which the project can be referenced within Serena/when chatting with the LLM. project_name: "obikmer" - -# list of languages for which language servers are started; choose from: -# al angular ansible bash clojure -# cpp cpp_ccls crystal csharp csharp_omnisharp -# dart elixir elm erlang fortran -# fsharp go groovy haskell haxe -# hlsl html java json julia -# kotlin lean4 lua luau markdown -# matlab msl nix ocaml pascal -# perl php php_phpactor powershell python -# python_jedi python_ty r rego ruby -# ruby_solargraph rust scala scss solidity -# svelte swift systemverilog terraform toml -# typescript typescript_vts vue yaml zig -# (This list may be outdated. For the current list, see values of Language enum here: -# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py -# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) -# Note: -# - For C, use cpp -# - For JavaScript, use typescript -# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) -# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) -# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) -# - For Free Pascal/Lazarus, use pascal -# Special requirements: -# Some languages require additional setup/installations. -# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers -# When using multiple languages, the first language server that supports a given file will be used for that file. -# The first language is the default language and the respective language server will be used as a fallback. -# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. -languages: -- rust - # the encoding used by text files in the project # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings encoding: "utf-8" @@ -55,23 +22,19 @@ ignore_all_files_in_gitignore: true # advanced configuration option allowing to configure language server-specific options. # Maps the language key to the options. -# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. -# No documentation on options means no options are available. +# The settings are considered only if the project is trusted (see global configuration to define trusted projects). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings ls_specific_settings: {} -# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). -# Paths can be absolute or relative to the project root. -# Each folder is registered as an LSP workspace folder, enabling language servers to discover -# symbols and references across package boundaries. -# Currently supported for: TypeScript. -# Example: -# additional_workspace_folders: -# - ../sibling-package -# - ../shared-lib -additional_workspace_folders: [] - # list of additional paths to ignore in this project. # Same syntax as gitignore, so you can use * and **. +# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases. +# Example: +# ignored_paths: +# - "examples/**" +# - ".worktrees/**" +# - "**/bin/**" +# - "**/obj/**" # Note: global ignored_paths from serena_config.yml are also applied additively. ignored_paths: [] @@ -131,3 +94,76 @@ read_only_memory_patterns: [] # Extends the list from the global configuration, merging the two lists. # Example: ["_archive/.*", "_episodes/.*"] ignored_memory_patterns: [] + +# list of additional workspace folder paths for cross-package reference support. +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries, but these folders are not indexed by Serena, +# i.e. the respective symbols will not be found using Serena's symbol search tools. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +ls_additional_workspace_folders: [] + +# list of language servers to start when using the LSP backend; choose from: +# ada al angular ansible bash +# bsl clojure cpp cpp_ccls crystal +# csharp csharp_omnisharp cue dart deno +# elixir elm erlang fortran fsharp +# gdscript gleam go groovy haskell +# haxe hlsl html java json +# julia kotlin latex lean4 lua +# luau markdown matlab msl nextflow +# nix ocaml pascal perl php +# php_phpactor php_phpantom powershell python python_basedpyright +# python_jedi python_pyrefly python_ty qml r +# rego ruby ruby_solargraph rust scala +# scss solidity svelte swift systemverilog +# terraform toml typescript typescript_vts vue +# wolfram yaml zig +# (This list may be outdated; generated with scripts/print_language_list.py; +# For the current list, see values of the LanguageServerId enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py) +# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some language servers require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple language servers, the first language server that supports a given file will be used for that file. +# The first language server is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +language_servers: +- rust + +# list of workspace folder paths (LSP backend only). +# These folders will be used to build up Serena's symbol index. +# Paths must be within the project root and should thus be relative to the project root. +# Furthermore, the paths should not be filtered by ignore settings. +# Default setting: The entire project root folder (".") is considered. +# In (large) monorepos, this can be used to index only subfolders of the project root, e.g. +# ls_workspace_folders: +# - "./subproject1" +# - "./subproject2" +ls_workspace_folders: +- . + +# optional shell command to run before the language backend (LSP or JetBrains) is initialised. +# the command runs in the project root directory and is only executed if the project is trusted +# (see trusted_project_path_patterns in the global configuration). +# serena waits for the command to exit: a non-zero exit code is logged as an error but does not +# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety +# backstop for non-terminating commands; on expiry the process is killed and activation continues. +# example: activation_command: "npx nx run-many -t build" +activation_command: + +# maximum time in seconds to wait for activation_command to complete before killing it (default 180s). +# must be a positive number. +activation_command_timeout: 180.0 diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index aa836553..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,103 +0,0 @@ -**PROMPT** - -Tu es ma base de connaissance et mon bloc-notes intelligent sur le projet **obikmer**. Tu ne proposes pas, tu ne codes pas spontanément — tu réponds à mes questions et tu structures mes idées au fur et à mesure que je les exprime. - -**Règle absolue : une question appelle une réponse, pas une action.** -Ne modifier aucun fichier à moins d'une demande explicite de modification. En particulier : observer un bug ou une incohérence dans le code montré ne constitue pas un mandat pour le corriger. Le code montré peut refléter une intention en cours — modifier sans mandat risque d'introduire un vrai bug là où tu croyais corriger. - -**Règle absolue : ne jamais substituer une dépendance ou une bibliothèque sans validation explicite.** -Si une dépendance demandée pose problème (erreur de compilation, bug, API manquante), exposer le problème et proposer des alternatives — ne jamais switcher silencieusement vers une autre bibliothèque. Le choix des dépendances est une décision d'architecture qui appartient au développeur. - -**Règle absolue : le code existant est une hypothèse, pas une vérité.** -Quand une nouvelle construction (type, itérateur, abstraction) rend du code historique injustifié, le signaler immédiatement et proposer de le supprimer — ne pas conserver les deux en parallèle par inertie. Le développeur demande explicitement de remettre en cause le code base : ne pas attendre qu'il insiste. - -Tu maintiens en **anglais**, dense et sans remplissage, les documents suivants : -- `DevDocMD/index.md` — document de discussion de base, enrichi progressivement au fil de nos échanges ; il reflète l'état courant de la réflexion sur le projet -- les autres fichiers Markdown dans `DevDocMD/` selon leur thème respectif - -Les snippets de code y sont courts et illustrent uniquement des principes architecturaux. Nos échanges se font en **français**. - ---- - -## Contexte du projet - -`obikmer` est un outil Rust de manipulation, comptage, indexation et opérations ensemblistes sur des séquences ADN représentées comme des ensembles de kmers. - -**Contraintes fondamentales** -- Efficacité maximale en calcul, mémoire et disque -- Données métagénomiques : plusieurs dizaines de Gbases, milliards de kmers -- k impair, k ∈ [11, 31], fixé à l'exécution -- Formats d'entrée : FASTA, FASTQ, gzip, streaming stdin - -**Opérations prioritaires** -- Comptage de kmers (fréquences) -- Recherche / requête rapide -- Opérations ensemblistes (union, intersection, diff) - -**Ce qui a déjà été discuté** -- Encodage 2 bits/base → kmer tient dans un `u64` -- Forme canonique : `min(kmer, revcomp)` pour réduire l'espace de moitié - ---- - -## Infrastructure de documentation - -La documentation est gérée via **MkDocs + thème Material**, avec publication sur **GitHub Pages**. Deux arbres de documentation indépendants, deux configurations à la racine du dépôt (voir aussi `UserDocMD/` ci-dessous pour la doc utilisateur) : - -**Structure des répertoires** - -``` -DevDocMD/ ← sources Markdown, doc développeur (discussion, historique, rationale) -UserDocMD/ ← sources Markdown, doc utilisateur (état factuel courant, sans code Rust) -mkdocs.yml ← config doc développeur : docs_dir DevDocMD, site_dir DevDoc -mkdocs-user.yml ← config doc utilisateur : docs_dir UserDocMD, site_dir doc -DevDoc/ ← site HTML généré (doc développeur) -doc/ ← site HTML généré (doc utilisateur — publié par GitHub Pages, répertoire par défaut) -.venv/ ← environnement Python (ignoré par git) -``` - -**Commandes Makefile** - -| Commande | Effet | -|---|---| -| `make doc` | Construit la doc développeur dans `DevDoc/` | -| `make doc-serve` | Serveur local (doc développeur), rechargement automatique | -| `make clean-doc` | Supprime `DevDoc/` | -| `make doc-user` | Construit la doc utilisateur dans `doc/` | -| `make doc-user-serve` | Serveur local (doc utilisateur), rechargement automatique | -| `make clean-doc-user` | Supprime `doc/` | -| `make clean` | Supprime `DevDoc/`, `doc/` et `.venv/` | - -Le `.venv/` est dans `.gitignore`. `DevDoc/` et `doc/` (sorties HTML) sont versionnés — `doc/` spécifiquement parce que c'est le répertoire par défaut que GitHub Pages sert. - -Lors de l'ajout de nouveaux fichiers Markdown dans `DevDocMD/`, mettre à jour la section `nav:` de `mkdocs.yml` ; dans `UserDocMD/`, mettre à jour `mkdocs-user.yml`. - ---- - -Je continue à poser mes questions et à guider la discussion. - ---- - -## MCP Tools - -**Règle absolue : avant tout travail de code, appeler `mcp__serena__initial_instructions` pour charger les instructions Serena.** - -### Hiérarchie des outils pour ce projet Rust - -**Navigation et édition de code → serena en priorité** -- Trouver un symbole, une déclaration, les implémentations d'un trait : `mcp__serena__find_symbol`, `mcp__serena__find_declaration`, `mcp__serena__find_implementations` -- Trouver les usages d'un symbole : `mcp__serena__find_referencing_symbols` -- Diagnostics LSP (erreurs de compilation) : `mcp__serena__get_diagnostics_for_file` -- Vue d'ensemble d'un fichier : `mcp__serena__get_symbols_overview` -- Modifier le corps d'une fonction/impl : `mcp__serena__replace_symbol_body` -- Ne pas utiliser `cclsp` quand serena couvre le besoin - -**Analyse architecturale → jcodemunch** -- Hotspots, couplage, dead code, dépendances entre modules -- Utiliser avant de refactorer une zone critique - -**Raisonnement complexe → sequential-thinking** -- Décisions d'architecture, choix d'algorithme, trade-offs non triviaux - -**Documentation de crates → context7** -- Toujours consulter avant d'utiliser une API de bibliothèque externe diff --git a/DevDoc/404.html b/DevDoc/404.html index 54445983..b04c63bb 100644 --- a/DevDoc/404.html +++ b/DevDoc/404.html @@ -722,14 +722,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -939,6 +939,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + diff --git a/DevDoc/architecture/index_architecture.refs/index.html b/DevDoc/architecture/index_architecture.refs/index.html index 06376b2b..78ae7534 100644 --- a/DevDoc/architecture/index_architecture.refs/index.html +++ b/DevDoc/architecture/index_architecture.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + @@ -1214,10 +1270,10 @@

    Coverage: architecture/index_architecture.md

    Code couvert

    diff --git a/DevDoc/architecture/numa_partition_runner/index.html b/DevDoc/architecture/numa_partition_runner/index.html index b6fa936f..c79dca08 100644 --- a/DevDoc/architecture/numa_partition_runner/index.html +++ b/DevDoc/architecture/numa_partition_runner/index.html @@ -729,14 +729,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -946,6 +946,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + @@ -1311,6 +1367,17 @@ + + +
  • + + + + Panic propagation (fixed, 2026-08-28) + + + +
  • @@ -1497,6 +1564,17 @@ +
    • + +
  • + + + + Panic propagation (fixed, 2026-08-28) + + + +
  • @@ -1780,11 +1858,27 @@ dormant workers have been woken and grows every node by the same amount per step, capped by that node's remaining dormant workers and by the run's total budget (n_total) — balance across nodes is now guaranteed by construction, not incidental to channel implementation details.

    +

    Panic propagation (fixed, 2026-08-28)

    +

    A worker whose f(i) panicked used to never send its WorkerEvent::Completed +— the controller's while completed < n_total loop then waited forever for +an event that partition could no longer produce, since other live +workers/timer threads kept event_rx open. Discovered via obikselect +panicking on an unimplemented Sparse matrix case (see implementation/select.md): +the process hung indefinitely instead of erroring out.

    +

    Fixed: each worker wraps f(i) in std::panic::catch_unwind and sends a new +WorkerEvent::Panicked(i, payload) instead of silently dropping the +partition. The controller counts it toward completed (unblocking the loop) +and keeps the first payload seen; once run returns, panic::resume_unwind +re-raises it on the caller's thread — the original message/backtrace still +surfaces, from the right place, instead of a silent deadlock. Takes priority +over a plain Err (a panic means a bug, not a normal typed failure).

    Open questions

    @@ -1504,7 +1560,7 @@ Replace available_parallelism() with per-NUMA core count for spawn criterion -obikpartitionner/src/merge_layer.rs +obikindex/src/partition/merge_layer.rs No change — merge_partition already works inside any Rayon context @@ -1512,7 +1568,7 @@ No change — par_iter and current_num_threads are pool-context-aware -obikpartitionner/src/partition.rs +obikindex/src/partition/partition.rs No change — same reason diff --git a/DevDoc/architecture/query.refs/index.html b/DevDoc/architecture/query.refs/index.html index 7c458324..ef859997 100644 --- a/DevDoc/architecture/query.refs/index.html +++ b/DevDoc/architecture/query.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + @@ -1215,7 +1271,7 @@

    Code couvert

    Notes

    diff --git a/DevDoc/architecture/query/index.html b/DevDoc/architecture/query/index.html index 00ca5e89..2ed91fb1 100644 --- a/DevDoc/architecture/query/index.html +++ b/DevDoc/architecture/query/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + @@ -1499,7 +1555,7 @@

    Algorithm

    -

    The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside process_chunk (query.rs); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (obikpartitionner::query_partition_with), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.

    +

    The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside process_chunk (query.rs); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (obikindex::partition::query_partition_with), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.

    for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
     build QueryBatch (QueryBatch::from_records):
         decompose all sequences into superkmers (SuperKmerIter) — construction only,
@@ -1574,9 +1630,9 @@
 
    The -z CLI option overrides the index metadata value. A higher z increases stringency (lower FP, some true positives may be discarded at sequence ends); a lower z increases sensitivity.
    
 
    
 
    Layer lookup: MphfLayer::find
    
-
    MphfLayer::open(dir, mode: &IndexMode) receives the mode from PartitionMeta — no per-layer file is read. The caller (QueryLayer) never chooses the dispatch path: it is fixed at open time by LayerEvidence. See obilayeredmap for the full find / find_strict API.
    
+
    MphfLayer::open(dir, mode: &IndexMode) receives the mode from PartitionMeta — no per-layer file is read. The caller (QueryLayer) never chooses the dispatch path: it is fixed at open time by LayerEvidence. See obikindex::layer for the full find / find_strict API.
    
 
    QueryLayer variant selection
    
-
    QueryLayer::open (obikpartitionner/src/query_layer.rs:28-45) only ever returns two variants — Presence or Count, checked in this order:
    
+
    QueryLayer::open (obikindex/src/partition/query_layer.rs:28-45) only ever returns two variants — Presence or Count, checked in this order:
    
 
@@ -1746,7 +1802,7 @@ Fix direction: restructure query's pipe with an initial Flat<
 
    3. Chunk-size memory formula ignores n_genomes.
 chunk_bytes = available_memory_bytes() / (n_workers * 16) (query.rs:407-414) assumes a fixed ~8–16× overhead per raw input byte. But KmerResults::new (query.rs:165-179) allocates data: Vec<u32> sized total_kmers_in_chunk × n_genomes — dense, for every k-mer position in the chunk, hit or not — plus win_min and (with --detail) cov, same scaling. Real per-chunk memory is O(n_genomes), not constant; the formula doesn't know n_genomes at all. This is the direct cause of the OOM kill on indexes with many reference genomes.
    
 
    4. MPHF lookup and matrix-row fetch are fused, not staged.
-QueryLayer::find_into (obikpartitionner/src/query_layer.rs:48-67) does the MPHF find and the fill_row matrix read in one call per k-mer, inside a single-threaded loop (query_partition_with). There is no separation between "is this k-mer indexed" (cheap, O(1), independent of n_genomes) and "what are its per-genome values" (the expensive, n_genomes-scaling part).
    
+QueryLayer::find_into (obikindex/src/partition/query_layer.rs:48-67) does the MPHF find and the fill_row matrix read in one call per k-mer, inside a single-threaded loop (query_partition_with). There is no separation between "is this k-mer indexed" (cheap, O(1), independent of n_genomes) and "what are its per-genome values" (the expensive, n_genomes-scaling part).
    
 
    5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.
 QueryBatch::from_records currently dereplicates at the superkmer level (HashMap<RoutableSuperKmer, Vec<SKDesc>>, query.rs:112). This misses redundancy between k-mers shared by different superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer construction (SuperKmerIter) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on CanonicalKmer, in the same pass: HashMap<CanonicalKmer, Vec<(seq_idx, pos)>>. This also means the MPHF find itself runs once per distinct k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.
    
 
    6. Stage 1 output: bucket confirmed hits by layer, keyed by MPHF slot.
@@ -1778,12 +1834,12 @@ Stage 2's output should be HashMap<genome_idx, Vec<(seq_idx, positio
 
    None of this is implemented yet — parked here as a coherent roadmap while the design is discussed further. Suggested dependency order: (1) I/O parallelism → (3) genome-aware chunk sizing → (4)–(9) staged/k-mer-deduped/NUMA-aware-partition-and-column-major/sparse query engine (larger refactor, biggest structural payoff — reuses PartitionRunner rather than inventing a new parallelism mechanism) → (2) parallel gzip (separate, orthogonal, tracked in chunkreader.md) → secondary diagnostics patterns.
    
 
    
 
    Implementation plan
    
-
    Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current query output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning obikmer, obikpartitionner, and obicompactvec — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense KmerResults) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if rapidgzip-rs is validated (see chunkreader.md).
    
+
    Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current query output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning obikmer, obikindex::partition, and obicompactvec — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense KmerResults) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if rapidgzip-rs is validated (see chunkreader.md).
    
 
    Instrumentation is deliberately sequenced before the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.
    
 
    Performance measurement on the reference 192-core/8-NUMA machine is done by the project owner, not from this development environment (macOS, 16 cores — PartitionRunner's NUMA pinning is Linux-only, so even phase 4's mechanism can't be functionally exercised for its actual purpose here). Each phase below is therefore written to be self-measuring: the debug-level logging it adds must be enough, on its own, to judge whether that phase's algorithmic choice paid off from a cluster run's logs, without needing to attach a profiler.
    
 
    Conventions applied to every phase below
    
 
    Debug logging. Every phase that changes an algorithmic choice (not phase 0, which is the logging) adds tracing::debug!/trace! at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF find calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured debug! per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.
    
-
    Unit tests. This project's convention (obiread, obikseq, obidebruinj, obicompactvec, obilayeredmap, obiskio, obifastwrite) is #[cfg(test)] #[path = "tests/<name>.rs"] mod tests; at the bottom of the source file, with the actual test code in a sibling src/tests/<name>.rs. Neither obikmer nor obikpartitionner (the two crates phases 3 and 5 touch most) currently have a src/tests/ directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
    
+
    Unit tests. This project's convention (obiread, obikseq, obidebruinj, obicompactvec, obikindex::layer, obiskio, obifastwrite) is #[cfg(test)] #[path = "tests/<name>.rs"] mod tests; at the bottom of the source file, with the actual test code in a sibling src/tests/<name>.rs. Neither obikmer nor obikindex::partition (the two crates phases 3 and 5 touch most) currently have a src/tests/ directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
    
 
    Workflow (jj). Work happens in a fresh jj commit, easy to abandon. jj new between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then query_layer.rs split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.
    
 
    Phase 0 — Instrumentation (prerequisite for measuring every later phase)
    
 
    Goal: make core utilization, throughput, and per-stage timing visible on a real run, so phases 1–5 can be justified with numbers instead of assumption.
    
@@ -1826,26 +1882,26 @@ Stage 2's output should be HashMap<genome_idx, Vec<(seq_idx, positio
 
  • obikmer/src/cmd/query.rs:
    • 
 
  • Replace QueryBatch::from_records's dedup map (HashMap<RoutableSuperKmer, Vec<SKDesc>>, current query.rs:112) with a per-partition HashMap<CanonicalKmer, Vec<(seq_idx: u32, pos: u32)>>, built in the same SuperKmerIter pass: superkmer construction and partition routing (part_idx from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each CanonicalKmer within a superkmer is inserted individually instead of the whole superkmer being the dedup key.
    • 
 
  • Verified: CanonicalKmer (obikseq/src/kmer.rs:390, pub type CanonicalKmer = CanonicalKmerOf<KLen>) — the underlying CanonicalKmerOf<L> derives Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash (kmer.rs:269). Usable as a HashMap/HashSet key as-is, no change needed.
    • 
-
  • obikpartitionner/src/query_layer.rs:
    • 
+
  • obikindex/src/partition/query_layer.rs:
    • 
 
  • Split QueryLayer::find_into (query_layer.rs:48-67) into two methods: find_slot(&self, kmer: CanonicalKmer) -> Option<usize> (MPHF only, no matrix touch) and keep fill_row as-is for phase 4 to call later.
    • 
 
  • Replace query_partition_with's inner loop (query_layer.rs:103-113) with a version that, for each unique CanonicalKmer, calls find_slot across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records (layer_idx, slot).
    • 
 
  • New return shape for the partition-level query, replacing today's on_hit(sk_idx, kmer_idx, row) callback: HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>> (roadmap point 6) — built directly from the k-mer dedup map's Vec<(seq_idx,pos)> values, keyed by the resolved slot instead of the k-mer.
    • 
 
  • This phase alone has no throughput benefit yet (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.
    • 
 
  • Log, per chunk: total k-mer occurrences vs. unique CanonicalKmer count (the dedup ratio — the entire justification for this phase) and the resulting MPHF find call count. If the dedup ratio is close to 1.0 on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.
    • 
-
  • Unit tests: create obikmer/src/cmd/tests/query.rs (new src/tests/ dir for this crate, following the project's #[cfg(test)] #[path = "tests/query.rs"] mod tests; convention) and obikpartitionner/src/tests/query_layer.rs (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the find_slot/bucket-by-layer-and-slot construction against a small hand-built QueryLayer fixture, asserting the (layer_idx, slot, seq_idx, pos) tuples match what the old per-occurrence loop would have produced.
    • 
+
  • Unit tests: create obikmer/src/cmd/tests/query.rs (new src/tests/ dir for this crate, following the project's #[cfg(test)] #[path = "tests/query.rs"] mod tests; convention) and obikindex/src/partition/tests/query_layer.rs (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the find_slot/bucket-by-layer-and-slot construction against a small hand-built QueryLayer fixture, asserting the (layer_idx, slot, seq_idx, pos) tuples match what the old per-occurrence loop would have produced.
    • 
 
 
    Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred
    
 
    Goal (revised during implementation): replace fill_row-per-hit (row-major, worst-case mmap locality) with a column-major scan. PartitionRunner turned out to be the wrong mechanism for this at this call granularity — see below; the column-major fetch itself is implemented and validated, without it.
    
 
    What shipped:
 - obicompactvec: the per-column accessors this phase needed already existed — PersistentCompactIntMatrix::col_view(c) and PersistentBitMatrix::col_view(c) are public, and IntSliceView::get(slot)/BitSliceView::get(slot) are public — the original plan underestimated how much of this plumbing the pairwise-distance code (dump/select/stats) had already required. The one real gap: PersistentBitMatrix::col_view() panics on the Implicit variant (the documented mono-genome fast path, bitmatrix.rs). Added PersistentBitMatrix::get(c, slot) -> u32 (bitmatrix.rs), a non-panicking column-major point lookup that returns 1 for Implicit regardless of c — the smallest surface needed, not a new col_get API from scratch.
-- obikpartitionner/src/query_layer.rs: query_partition_with is now two explicit stages, matching roadmap points 6–8: stage 1 (MPHF-only, per unique k-mer, bucket hits by (layer_idx, slot), emits QueryHit::Found) then stage 2 (per layer with ≥1 hit, column-major: for each genome column g in 0..layer.n_cols().min(n_genomes), scan that layer's bucketed slots and call col_value(g, slot), emitting QueryHit::Value(descs, g, value) on nonzero). QueryHit is a single enum delivered through one FnMut(QueryHit) callback — an earlier two-closure design (on_found + on_value) didn't borrow-check, since the caller's single mutable accumulator (KmerResults) can't be captured by two separate FnMut closures passed to the same call.
+- obikindex/src/partition/query_layer.rs: query_partition_with is now two explicit stages, matching roadmap points 6–8: stage 1 (MPHF-only, per unique k-mer, bucket hits by (layer_idx, slot), emits QueryHit::Found) then stage 2 (per layer with ≥1 hit, column-major: for each genome column g in 0..layer.n_cols().min(n_genomes), scan that layer's bucketed slots and call col_value(g, slot), emitting QueryHit::Value(descs, g, value) on nonzero). QueryHit is a single enum delivered through one FnMut(QueryHit) callback — an earlier two-closure design (on_found + on_value) didn't borrow-check, since the caller's single mutable accumulator (KmerResults) can't be captured by two separate FnMut closures passed to the same call.
 - obikmer/src/cmd/query.rs: KmerResults::set (row-major, whole-row-at-once) replaced by mark_found (stage 1: flag a position as indexed, independent of any genome's value) and set_one (stage 2: write one genome's value at one position). QueryStats extended with n_columns_scanned/n_col_get_calls, logged per chunk.
 - Total get()-equivalent calls are unchanged from the row-major version (n_hits × n_cols in the worst case, confirmed by n_col_get_calls in the debug log) — the win is locality (sequential access within one layer's column at a time, across mmap'd regions, instead of jumping across all columns per hit), exactly as predicted.
    
 
    What did not ship, and why — PartitionRunner is architecturally the wrong tool here:
 Reading obikindex/src/numa.rs's actual run() body (not just its doc comments) shows every call spawns a timer thread plus one OS thread per worker slot on every NUMA node (std::thread::scope + one s.spawn() per node per max_workers) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned per call. This is fine for its actual, established usage in this codebase (merge.rs, index.rs's build_layers): one PartitionRunner::new() + one run() call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for query's call pattern: query_partition_with runs once per (chunk, partition), potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact.
 The column-major loop in stage 2 is therefore a plain sequential loop for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) rayon's already-warm global pool (into_par_iter()) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern numa_worker_pools.md warns about for a different workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one mmap'd region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so PartitionRunner is invoked once per whole query run (or per large batch of chunks) rather than per (chunk, partition), amortising its spawn cost the way merge/build_layers do — a bigger structural change than this phase's scope.
 - Log (implemented): QueryStats::n_columns_scanned/n_col_get_calls, folded into the existing per-chunk debug!("k-mer dedup + column-major fetch", ...) line (query.rs) alongside phase 3's dedup counters.
-- Unit tests: extended obikpartitionner/src/tests/query_layer.rs (phase 3's file) — query_partition_with's empty/missing-index paths updated for the new QueryStats fields and single-callback signature.
+- Unit tests: extended obikindex/src/partition/tests/query_layer.rs (phase 3's file) — query_partition_with's empty/missing-index paths updated for the new QueryStats fields and single-callback signature.
 - Validation performed: full workspace build + cargo test --workspace, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same kmer_count/kmer_strict_matches on every record); (2) the existing 20-genome benchmark/global_index_presence index — runs correctly, n_hits=0 for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the Implicit/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) the critical correctness case: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from genomeA matched only genomeA (kmer_count identical to the pre-dedup occurrence count, zero leakage into genomeB's column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in n_cols, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly.
 - Not yet done: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-n_genomes layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.
    
 
    Phase 5 — Sparse Findere rework (roadmap point 9)
    
diff --git a/DevDoc/architecture/rebuild_filter/index.html b/DevDoc/architecture/rebuild_filter/index.html
index 9e8cbce5..a21d25de 100644
--- a/DevDoc/architecture/rebuild_filter/index.html
+++ b/DevDoc/architecture/rebuild_filter/index.html
@@ -727,14 +727,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1389,9 +1445,9 @@ IO pattern: sequential read of each source column → random write into new colu
 
    
 
 
    
 
    Files to modify
    
 
      
-
    • src/obikpartitionner/src/rebuild_layer.rs — rebuild_partition and iter_src_layers
      • 
+
    • src/obikindex/src/partition/rebuild_layer.rs — rebuild_partition and iter_src_layers
      • 
 
    • Possibly src/obicompactvec/ — add column iterator API if not already present
      • 
-
    • src/obilayeredmap/ — check if per-column sequential access is exposed on SrcLayerData
      • 
+
    • src/obikindex/src/layer/ — check if per-column sequential access is exposed on SrcLayerData
      • 
 
    
 
    Open questions
    
 
         
       
diff --git a/DevDoc/architecture/sequences/invariant/index.html b/DevDoc/architecture/sequences/invariant/index.html
index 72751107..38f3fb60 100644
--- a/DevDoc/architecture/sequences/invariant/index.html
+++ b/DevDoc/architecture/sequences/invariant/index.html
@@ -9,7 +9,7 @@
       
       
       
-        
+        
       
       
         
@@ -731,14 +731,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -948,6 +948,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/architecture/siblings/index.html b/DevDoc/architecture/siblings/index.html
index 8062296e..8be09b91 100644
--- a/DevDoc/architecture/siblings/index.html
+++ b/DevDoc/architecture/siblings/index.html
@@ -731,14 +731,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -948,6 +948,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1289,6 +1345,39 @@
     
   
   
+
+      
+        
  • 
+  
+    
+      
+        query never benefits from sparse row-major access (found 2026-08-19, not implemented)
+      
+    
+  
+  
+
    • 
+      
+        
  • 
+  
+    
+      
+        Implemented (2026-08-20)
+      
+    
+  
+  
+
    • 
+      
+        
  • 
+  
+    
+      
+        PersistentCompactIntMatrix::Sparse — implemented (2026-08-26)
+      
+    
+  
+  
 
    • 
       
     
@@ -1552,6 +1641,39 @@
     
   
   
+
+      
+        
  • 
+  
+    
+      
+        query never benefits from sparse row-major access (found 2026-08-19, not implemented)
+      
+    
+  
+  
+
    • 
+      
+        
  • 
+  
+    
+      
+        Implemented (2026-08-20)
+      
+    
+  
+  
+
    • 
+      
+        
  • 
+  
+    
+      
+        PersistentCompactIntMatrix::Sparse — implemented (2026-08-26)
+      
+    
+  
+  
 
    • 
       
     
@@ -1587,7 +1709,7 @@ one-way function, not an invertible bijection with a stored inverse. Any
 method that reconstructs a kmer from a bare slot number is wrong by
 construction, regardless of the mechanism used (MPHF re-hash, or evidence
 decode + direct unitig read). See MphfLayer::kmer_at
-(obilayeredmap/src/mphf_layer.rs) — flagged for removal, currently called
+(obikindex/src/layer/mphf_layer.rs) — flagged for removal, currently called
 from obikphylo/siblings/build.rs and family_scan.rs (since removed — see
 "Pending work" status below).
    
 
    Two pipelines, never mixed
    
@@ -1650,7 +1772,7 @@ partition is unknown) and must keep going through
 index().
    
 
    Pending work — done
    
 
    The plan above shipped: obikphylo (a new crate — phylo-domain extension
-traits over obikindex::KmerIndex/obilayeredmap::Layer<D>, replacing the
+traits over obikindex::KmerIndex/obikindex::layer::Layer<D>, replacing the
 old obikindex::siblings module) builds and reads the annex purely in
 iteration order (SiblingLayerExt::iter_siblings/iter_minorants, both with
 batch variants, mirroring Layer<D>'s own KmerIter/KmerBatchIter
@@ -2218,13 +2340,13 @@ BRWT-style column-correlation exploitation.
    
 
    PersistentSparseBitMatrix went from a validated but unused type to a
 real, selectable on-disk format:
    
 
      
-
    • Generic Layer<D>: obilayeredmap::Layer<D>'s presence-only methods
+
    • Generic Layer<D>: obikindex::layer::Layer<D>'s presence-only methods
   (n_cols, sub_matrix, fill_sub_matrix) are generic over any
   D: LayerData<Item = Box<[bool]>> + BinaryMatrix, not hardcoded to
   PersistentBitMatrix — PersistentSparseBitMatrix implements
   LayerData (open/read) the same way. find_slot/index_batch were
   already generic over any D: LayerData, so they needed no change.
-  Verified by obilayeredmap's
+  Verified by obikindex::layer's
   presence_layer_generic_over_sparse_matches_dense test: build a dense
   presence layer, convert it to sparse via build_from_dense, open both
   as Layer<PersistentBitMatrix>/Layer<PersistentSparseBitMatrix> on
@@ -2234,7 +2356,7 @@ real, selectable on-disk format:
      
   AtomicUsizes in test builds, not thread-local, so a test using a
   different k races every other test in the same crate binary; a k=11
   version of this test passed alone but failed under the full
-  obilayeredmap suite for exactly that reason before being fixed.)
      • 
+  obikindex::layer suite for exactly that reason before being fixed.)
 
    • obikphylo::siblings::cache::Mat gained a third variant,
   SparsePresence(Layer<PersistentSparseBitMatrix>), alongside Count
   and Presence — every method (find_slot, index_batch,
@@ -2264,7 +2386,7 @@ real, selectable on-disk format:
      
   dense-path test (sibling_annex_one_sibling_each) exactly — proves the
   sparse format round-trips through the real build pipeline
   (PartitionCache sparse-detection included), not just the
-  obicompactvec/obilayeredmap unit layers below it.
      • 
+  obicompactvec/obikindex::layer unit layers below it.
 
    
 
    Full workspace cargo test (all crates, unit + doc tests) green after
 this change.
    
@@ -2310,7 +2432,7 @@ this change.
    
   (activation: either given; defaults 1.0/0.5 for whichever is unset).
 
 
    Full workspace cargo test green after this change (167 unit tests in
-obicompactvec+obilayeredmap+obikphylo alone, plus every other
+obicompactvec+obikindex::layer+obikphylo alone, plus every other
 crate's suite, no regressions).
    
 
    Still open, not part of this change (per "Correction to the 'single
 pass' framing" above): --raw-snp-distance/--raw-snp-counts (the
@@ -2319,6 +2441,261 @@ scan the full unsampled index — never threaded --subsample/
 out of scope here since the reported problem was specifically about the
 --sankoff/--tnt pipeline's redundant/inconsistent scans, not these
 two standalone flags.
    
+
    query never benefits from sparse row-major access (found 2026-08-19, not implemented)
    
+
    Benchmarked obikmer query against global_index_presence (dense-packed)
+vs. global_index_presence_sparse (pack --sparse), 100k simulated reads
+× 2 specimens (benchmark/, see
+benchmark_query_testing.md).
+Correctness: 0 mismatches — sparse and dense return bit-identical query
+results. Performance: sparse consistently slower than dense (~30-50%,
+reproducible across two runs with warm disk cache), the opposite of
+pack --sparse's stated intent ("faster for single-row access... like
+query").
    
+
    Root cause, read from source, not measured in isolation:
+KmerPartition::query_partition_with (obikindex/src/partition/query_layer.rs:155-220)
+is architecturally column-major: stage 2 walks for g in 0..n_cols { for
+slot in hit_slots { layer.col_value(g, slot) } }, documented (correctly)
+as the right locality strategy for the packed/columnar formats, where
+col_value → PersistentBitMatrix::get is a genuine O(1) mmap'd column
+read (persistent.rs:110-113).
    
+
    For Self::Sparse, that same get(c, slot) (persistent.rs:114-118)
+allocates a full n_cols-wide buffer and calls fill_row — materializing
+the entire row — just to return one cell. Called from inside the
+column-major double loop, this reconstructs the same row once per genome
+column touched: O(hits × n_cols) full-row rebuilds instead of O(hits).
+PersistentSparseBitMatrix's own native row-major decode
+(for_each_genome_in_row, sparse.rs:164-177, used correctly by its own
+row/fill_row/fill_sub_matrix) is never reached from the query path
+at all.
    
+
    fill_sub_matrix (the existing BinaryMatrix trait primitive,
+traits.rs:13-37) is not the right replacement for query either, even
+once its own dispatch bug is fixed (see next section) — its output shape
+is inherently column-dense: out[col] gets an entry for every column,
+including columns with zero hits among the requested slots. On real
+sparse data (a hit typically touching a handful of genomes out of dozens)
+that's still O(n_cols) output regardless of true sparsity. What query
+actually wants is the sparse triple stream (slot, col, value) it already
+consumes as QueryHit::Value — not a materialized sub-matrix.
    
+
    Proposed primitive (design only, not implemented — explicit ask: keep
+count matrices not excluded, even though effort right now is
+presence/absence only):
    
+
    Not a closure-driven for_each — a real Iterator, one concrete struct
+per matrix format, so the traversal state (current position in the sorted
+slot list, current column, permutation, sparse-row decode cursor…) lives
+in named struct fields instead of being threaded implicitly through
+recursion or a captured closure. RPITIT (stable since 1.75, and this
+workspace is edition 2024) means the trait method can return it without
+naming or boxing the concrete type:
    
+
    /// Yields every nonzero cell among `slots`, in implementation-defined order.
+fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a;
+// item: (idx into `slots`, col, value)
+
    
+
    This is the one foundational primitive per format — both fill_sub_matrix
+and a for_each-style callback become trivial, free consumers of it
+(.for_each(f) is already Iterator::for_each from std; fill_sub_matrix
+becomes "drain the iterator, scatter into out[][]"), instead of two
+independently-maintained traversals that can silently diverge (see the bug
+below — this is exactly how it happened).
    
+
      
+
    • On PersistentSparseBitMatrix: the struct is nearly free to write —
+  it wraps the existing (currently private) for_each_genome_in_row
+  per-row decode, advancing to the next slots entry on exhaustion. O(Σ
+  row nnz), zero n_cols-wide allocation.
      • 
+
    • On PersistentBitMatrix::{Packed,Columnar}: revised — cheaper than
+  first thought, by reusing the same split already used for
+  fill_matrix's own implementation instead of hand-writing a resumable
+  state machine at the matrix level. The base-vector layer
+  (BitSliceView, views.rs) already separates the two concerns:
+  fill_batch_sorted (views.rs:55-60, sorted-slot batch lookup) and a
+  genuine per-bit Iterator (BitSliceIter, views.rs:94+) sit side by
+  side there, one level below the matrix. Adding a
+  "positions among sorted_slots where the bit is set" iterator at that
+  same vector level is a filter over the existing get() — no new
+  state machine, since std::iter::Filter already is one, generated by
+  the compiler:
+  
      fn nonzero_among_sorted<'s>(&'s self, sorted_slots: &'s [usize]) -> impl Iterator<Item = usize> + 's {
+    sorted_slots.iter().copied().filter(move |&slot| self.get(slot))
+}
+
      
+  The matrix-level nonzero_iter then composes these per column with
+  flat_map over 0..n_cols (each column's hits, tagged with c,
+  slot mapped back through the sort permutation fill_batch/
+  fill_batch_sorted already carry) — again a combinator chain, not a
+  hand-rolled struct. Same algorithm, same mmap/sort locality as today's
+  fill_sub_matrix; just assembled from std iterator adaptors instead
+  of a loop body writing into a buffer, mirroring the vector/matrix split
+  the codebase already uses for fill_batch_sorted rather than
+  introducing a new shape.
      • 
+
    • Implicit: trivial (slots.iter().map(|&i| (i, 0, 1)), one column,
+  always present).
      • 
+
    • On PersistentCompactIntMatrix (counts): same treatment as
+  Packed/Columnar — no sparse count format exists yet ("Explicitly
+  deferred" per traits.rs:9-12), so no native low-effort case the way
+  Sparse has one, but not excluded either: the iterator's Item is
+  already (usize, usize, u32), not bool, specifically so presence
+  (0/1) and counts (arbitrary u32) share one primitive instead of a
+  bool/u32 split forcing counts out of the design. Ready for a native
+  sparse-count struct later without a signature change.
      • 
+
    
+
    Would let query_partition_with's stage 2 collapse to one
+for (i, g, v) in layer.matrix().nonzero_iter(&hit_slots) { on_event(...) }
+per layer, format-agnostic, each backend's struct deciding the actual
+traversal.
    
+
    This also closes the existing dispatch bug for free, by construction:
+PersistentBitMatrix::fill_sub_matrix (persistent.rs:190-215, the enum
+wrapper backing BinaryMatrix's trait impl) today does not delegate to
+PersistentSparseBitMatrix::fill_sub_matrix for Self::Sparse — it
+reimplements the same naive per-(column, slot) fill_row_bool loop
+instead, bypassing the efficient native method one file over
+(sparse.rs:249-258). obikphylo::siblings::cache::Mat
+(cache.rs:138-145) independently built its own parallel enum wrapper
+that dispatches correctly — evidence this was worked around rather than
+fixed at the source: two hand-written traversals for the same format,
+free to drift apart, and they did. If fill_sub_matrix itself is
+rewritten as "drain nonzero_iter, scatter into out[][]", there is only
+one traversal per format left to get right — the bug class doesn't just
+get fixed once, it stops being possible to reintroduce.
    
+
    Implemented (2026-08-20)
    
+
    Built as designed above, with one deviation from the original sketch:
+nonzero_iter ended up Box<dyn Iterator<...>>, not a bare impl
+Iterator, because Columnar/Packed/Sparse/Implicit are genuinely
+different concrete types and this method isn't on a trait (kept off
+BinaryMatrix deliberately — that trait is used as dyn BinaryMatrix in
+tests/sparse.rs, and RPITIT methods aren't dyn-compatible). One Box
+per nonzero_iter call, not per cell — negligible next to what it
+replaces.
    
+
      
+
    • BitSliceView::nonzero_among_sorted / IntSliceView::nonzero_among_sorted
+  (obicompactvec/src/views.rs): the vector-level filter/filter_map
+  primitive, exactly as sketched — no new state machine, std's own.
      • 
+
    • PersistentSparseBitMatrix::nonzero_iter (bitmatrix/sparse.rs): native,
+  std::iter::from_fn over one buffered row at a time via the existing
+  for_each_genome_in_row — no n_cols-wide allocation, ever.
      • 
+
    • PersistentBitMatrix::nonzero_iter (bitmatrix/persistent.rs): dispatches
+  to the above for Sparse; for Columnar/Packed, loops columns,
+  collects each column's nonzero_among_sorted hits via .extend() (not
+  flat_map — a flat_map closure can't lazily return something
+  borrowing its own captured sort permutation across separate calls
+  without either boxing per-column or fighting the borrow checker; eager
+  collection into one Vec sidesteps it, at zero cost since
+  fill_sub_matrix already fully materialized anyway). Implicit trivial.
      • 
+
    • PersistentBitMatrix::fill_sub_matrix and sub_matrix rewritten to
+  drain nonzero_iter — the dispatch bug is gone because there is now
+  only one traversal per format, not because the old one was patched.
+  PersistentCompactIntMatrix::nonzero_iter added the same way (counts
+  not excluded, per the earlier ask) — no native low-effort case, since no
+  sparse count format exists, but on the same primitive, ready for one.
      • 
+
    • KmerPartition::query_partition_with (obikindex/src/partition/query_layer.rs):
+  stage 2's column-major for g { for slot { col_value } } replaced by one
+  layer.nonzero_iter(&slot_list) call per layer, format-agnostic.
      • 
+
    • Tests: nonzero_iter_matches_dense, nonzero_iter_matches_row, and —
+  the one that actually targets the dispatch bug rather than each type's
+  own correctness — enum_wrapper_dispatches_to_native_sparse (builds
+  PersistentBitMatrix::Sparse(...) directly, not through open, since
+  open only auto-detects Sparse from a presence/ dir layout).
+  cargo test --workspace: green, no regressions.
      • 
+
    
+
    Measured: re-ran the benchmark/ query branch (100k reads × 2
+specimens, same setup as the original finding). Correctness still 0
+mismatches. The dense/sparse performance gap is gone — previously sparse
+~30-50% slower than dense, reproducibly; now within ~1-3% either way
+(7.42s dense vs 7.60s sparse for Escherichia_coli--K-12_MG1655; 5.25s vs
+5.30s for Saccharolobus_islandicus--M.16.4) — noise-level, not a
+systematic gap. pack --sparse's claimed query win isn't confirmed
+outright by this (sparse should arguably now beat dense on truly sparse
+real data, not just tie), but the pathological regression is fixed.
    
+
    PersistentCompactIntMatrix::Sparse — implemented (2026-08-26)
    
+
    Closes the gap flagged throughout this document ("no sparse count format
+exists yet", traits.rs:9-12's "Explicitly deferred"): obicompactvec
+already had PersistentSparseCompactIntMatrix (row-major, built on top of
+PersistentSparseBitMatrix as its "which columns are non-zero" support,
+values not deduplicated — see that struct's own doc comment), but it was
+never wired into PersistentCompactIntMatrix, the dense-dispatching enum
+every real consumer (TypedLayer<PersistentCompactIntMatrix>,
+KmerLayer::Count) actually holds. Concretely: kmer_index.rs::
+pack_matrices(sparse=true) already called pack_sparse_compact_int_matrix
+on every layer's counts/ — but PersistentCompactIntMatrix::open had no
+code path back to what that just wrote, so a Count layer became
+unreadable ("no count matrix found ... run 'obikmer upgrade'") the moment
+anyone ran pack --sparse on an index with count layers. Root cause, not a
+workaround: add the missing Sparse variant.
    
+
      
+
    • 
+
      Enum + dispatch (intmatrix.rs): PersistentCompactIntMatrix::Sparse
+  (PersistentSparseCompactIntMatrix), detected in open/detect_storage
+  via a singleton_values.pciv marker (mirrors PersistentBitMatrix's own
+  sparse_meta.json check), reported via storage_kind(). col/
+  col_view/col_persist panic/Unsupported on Sparse, same convention
+  as the bit side. sub_matrix/fill_sub_matrix and nonzero_iter
+  unified the same way PersistentBitMatrix's already are (drain
+  nonzero_iter, one traversal per format — see "Implemented
+  (2026-08-20)" above); nonzero_iter had to become Box<dyn Iterator<...>>
+  for the same reason (Columnar/Packed/Sparse are different concrete
+  types). No change needed in obikindex at all — KmerLayer::Count
+  already only ever holds TypedLayer<PersistentCompactIntMatrix>, so the
+  enum absorbing Sparse fixes the unreadable-layer bug for free, same as
+  PersistentBitMatrix::Sparse already did on the presence side.
      
+
      • 
+
    • 
+
      CountPartials, non-naive (sparse_intmatrix.rs): unlike
+  PersistentSparseBitMatrix's dict-driven col_weights_and_pair_counts,
+  values here aren't deduplicated (two rows can share the same non-zero
+  column set via the same dict_id while carrying different counts), so
+  the "weight by how many rows share a dict entry" shortcut doesn't carry
+  over. What does: a single row-major pass (row_major_pairwise, decodes
+  each row once via for_each_cell_in_row, nests over that row's own
+  co-present columns) — O(Σ k̄²) over populated rows instead of the naive
+  O(n_cols² × n) column-pair rescan, same complexity class as the bit
+  side minus the dict multiplicity discount. Kernels used: min(a,b)
+  (bray, relfreq-bray — both vanish when either side is absent, so no
+  correction needed), a·b and √(a·b) (euclidean/relfreq-euclidean and
+  hellinger — these do need a correction, reconstructed from per-column
+  marginals via Σ(a-b)² = Σa²+Σb²-2Σab, since (a-0)² = a² ≠ 0 unlike
+  the min-based formulas). threshold_jaccard(1) shortcuts straight to
+  support's own BitPartials::partial_jaccard (threshold 1 is exactly
+  presence); threshold_jaccard(0) is closed-form (every u32 is ≥ 0).
      
+
      • 
+
    • 
+
      Two pre-existing bugs found and fixed while wiring the threshold==1
+  shortcut (bitmatrix/sparse.rs, BitPartials for
+  PersistentSparseBitMatrix, present since the 2026-08-15 implementation
+  above, never caught because no test compared Sparse's raw partial_*
+  output against dense on real data — only the diagonal-blind
+  jaccard_dist_matrix/hamming_dist_matrix finalisations were tested):
      
+
      • 
+
    • partial_jaccard's diagonal was (0, 2×col_weights[i]) instead of a
+     genuine self-comparison (col_weights[i], col_weights[i]) —
+     col_weights_and_pair_counts's inter never pairs a column with
+     itself by construction.
      • 
+
    • partial_hamming's off-diagonal formula itself was wrong: total -
+     union (count of rows where neither column is present) instead of
+     the actual Hamming distance col_weights[i] + col_weights[j] -
+     2×inter[i,j] (symmetric-difference size). Only coincides with the
+     correct value when col_weights[i] + col_weights[j] == total, so
+     small/synthetic test data could easily have hidden it.
      • 
+
    
+
    Neither surfaced through jaccard_dist_matrix/hamming_dist_matrix
+  (both explicitly zero their own diagonal at finalisation, and the
+  off-diagonal partial_hamming bug had gone untested against dense
+  entirely) — only visible to a caller of the raw partial_* methods
+  directly, which is exactly what partial_threshold_jaccard(1)'s new
+  shortcut became. Fixed at the source, not patched around at the call
+  site; regression test added:
+  tests::sparse::partial_jaccard_and_hamming_match_dense_including_diagonal.
    
+
      
+
    • Tests: tests::intmatrix::sparse_roundtrip_matches_columnar/
+  sparse_roundtrip_from_packed (the open-dispatch fix, both build
+  paths); tests::intmatrix::sparse_count_partials_match_dense (all six
+  CountPartials formulas, thresholds 0/1/2/3, against Columnar on
+  asymmetric-presence data — this is what caught the diagonal gap in the
+  int side's own new code before it shipped, the same way it exposed the
+  two pre-existing bit-side bugs above); obikindex's
+  count_layer_transparently_reads_sparse_after_pack — the actual
+  end-to-end regression test for the original "layer unreadable after
+  pack --sparse" bug, built → packed sparse → reopened, compared against
+  the pre-pack dense read. cargo test -p obicompactvec -p obikindex:
+  green, no regressions (180 + 12 tests).
      • 
+
    
 
 
 
diff --git a/doc/usage/name-tree/index.html b/DevDoc/implementation/benchmark_query_testing/index.html
similarity index 56%
rename from doc/usage/name-tree/index.html
rename to DevDoc/implementation/benchmark_query_testing/index.html
index 5757fbc2..c8e7c979 100644
--- a/doc/usage/name-tree/index.html
+++ b/DevDoc/implementation/benchmark_query_testing/index.html
@@ -9,10 +9,10 @@
       
       
       
-        
+        
       
       
-        
+        
       
       
         
@@ -23,7 +23,7 @@
     
     
       
-        name-tree - obikmer — User Guide
+        Benchmark: query-path testing - obikmer
       
     
     
@@ -64,7 +64,7 @@
     
    
       
         
-        
+        
           Skip to content
         
       
@@ -80,7 +80,7 @@
 
 
    
   
       
diff --git a/DevDoc/implementation/chunkreader/index.html b/DevDoc/implementation/chunkreader/index.html
index 69852bca..b1ed11a9 100644
--- a/DevDoc/implementation/chunkreader/index.html
+++ b/DevDoc/implementation/chunkreader/index.html
@@ -405,11 +405,11 @@
 
 
 
  • 
-
+
 
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -497,6 +497,30 @@
   
 
     
+  
+
+
    • 
+
  • 
+
+
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+
+
    • 
+
  • 
+
+
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
   
 
 
    • 
diff --git a/DevDoc/implementation/evidence_elimination.refs/index.html b/DevDoc/implementation/evidence_elimination.refs/index.html
index 8215d476..1fa176b3 100644
--- a/DevDoc/implementation/evidence_elimination.refs/index.html
+++ b/DevDoc/implementation/evidence_elimination.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1214,9 +1270,9 @@
 
    Coverage: implementation/evidence_elimination.md
    
 
    Code couvert
    
 
      
-
    • obilayeredmap/src/fingerprint.rs — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
      • 
-
    • obilayeredmap/src/mphf_layer.rs — build_approx_evidence(dir, b, z), find_approx()
      • 
-
    • obilayeredmap/src/meta.rs — EvidenceKind::Approx { b, z }, LayerMeta
      • 
+
    • obikindex/src/layer/fingerprint.rs — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
      • 
+
    • obikindex/src/layer/mphf_layer.rs — build_approx_evidence(dir, b, z), find_approx()
      • 
+
    • obikindex/src/layer/meta.rs — EvidenceKind::Approx { b, z }, LayerMeta
      • 
 
    • obikindex/src/reindex.rs — KmerIndex::reindex(), conversion exact↔approx en place
      • 
 
    • obikmer/src/cmd/reindex.rs — CLI reindex, options --approx, -z, --evidence-bits, --fp, --block-size
      • 
 
    • obikmer/src/cmd/index.rs — resolve_approx_params(), options --approx, -z, --evidence-bits, --fp
      • 
diff --git a/DevDoc/implementation/evidence_elimination/index.html b/DevDoc/implementation/evidence_elimination/index.html
index 17a240d5..70621d90 100644
--- a/DevDoc/implementation/evidence_elimination/index.html
+++ b/DevDoc/implementation/evidence_elimination/index.html
@@ -12,7 +12,7 @@
         
       
       
-        
+        
       
       
         
@@ -868,14 +868,14 @@
   
   
     
    • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1085,6 +1085,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      • 
+  
+
+              
+            
           
    
         
       
diff --git a/DevDoc/implementation/filtering/index.html b/DevDoc/implementation/filtering/index.html
index 283f5b8d..6fdab235 100644
--- a/DevDoc/implementation/filtering/index.html
+++ b/DevDoc/implementation/filtering/index.html
@@ -733,14 +733,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1208,6 +1208,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -2130,7 +2186,7 @@ obikmer phylo myindexImplementation
 
         
       
diff --git a/DevDoc/implementation/kmer/index.html b/DevDoc/implementation/kmer/index.html
index 060b0c84..f367ae72 100644
--- a/DevDoc/implementation/kmer/index.html
+++ b/DevDoc/implementation/kmer/index.html
@@ -868,14 +868,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1085,6 +1085,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/implementation/obilayeredmap.refs/index.html b/DevDoc/implementation/layer_tier.refs/index.html
similarity index 92%
rename from DevDoc/implementation/obilayeredmap.refs/index.html
rename to DevDoc/implementation/layer_tier.refs/index.html
index 0b716f14..92d01fcf 100644
--- a/DevDoc/implementation/obilayeredmap.refs/index.html
+++ b/DevDoc/implementation/layer_tier.refs/index.html
@@ -19,7 +19,7 @@
     
     
       
-        Obilayeredmap.refs - obikmer
+        Layer tier.refs - obikmer
       
     
     
@@ -60,7 +60,7 @@
     
    
       
         
-        
+        
           Skip to content
         
       
@@ -96,7 +96,7 @@
         
    
           
             
-              Obilayeredmap.refs
+              Layer tier.refs
             
           
         
    
@@ -727,14 +727,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1211,15 +1267,15 @@
 
 
 
-
    Coverage: implementation/obilayeredmap.md
    
+
    Coverage: implementation/layer_tier.md
    
 
    Code couvert
    
 
      
-
    • obilayeredmap/src/mphf_layer.rs — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
      • 
-
    • obilayeredmap/src/layer.rs — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
      • 
-
    • obilayeredmap/src/map.rs — LayeredMap, push_layer(), query()
      • 
-
    • obilayeredmap/src/evidence.rs — Evidence, EvidenceWriter, encodage chunk_id:rank
      • 
-
    • obilayeredmap/src/fingerprint.rs — FingerprintVec, FingerprintVecWriter, matches()
      • 
-
    • obilayeredmap/src/meta.rs — LayerMeta, EvidenceKind (Exact / Approx { b, z })
      • 
+
    • obikindex/src/layer/mphf_layer.rs — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
      • 
+
    • obikindex/src/layer/typed_layer.rs — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
      • 
+
    • obikindex/src/layer/map.rs — LayeredMap, push_layer(), query()
      • 
+
    • obikindex/src/layer/evidence.rs — Evidence, EvidenceWriter, encodage chunk_id:rank
      • 
+
    • obikindex/src/layer/fingerprint.rs — FingerprintVec, FingerprintVecWriter, matches()
      • 
+
    • obikindex/src/layer/meta.rs — LayerMeta, EvidenceKind (Exact / Approx { b, z })
      • 
 
    
 
    Notes
    
 
    FORT RISQUE DE DÉRIVE. C'est le fichier le plus affecté par les changements récents :
diff --git a/DevDoc/implementation/obilayeredmap/index.html b/DevDoc/implementation/layer_tier/index.html
similarity index 84%
rename from DevDoc/implementation/obilayeredmap/index.html
rename to DevDoc/implementation/layer_tier/index.html
index 8ff0ac9a..2dc88b7a 100644
--- a/DevDoc/implementation/obilayeredmap/index.html
+++ b/DevDoc/implementation/layer_tier/index.html
@@ -23,7 +23,7 @@
     
     
       
-        obilayeredmap crate - obikmer
+        obikindex layer tier - obikmer
       
     
     
@@ -64,7 +64,7 @@
     
    
       
         
-        
+        
           Skip to content
         
       
@@ -100,7 +100,7 @@
         
    
           
             
-              obilayeredmap crate
+              obikindex layer tier
             
           
         
    
@@ -749,7 +749,7 @@
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -767,7 +767,7 @@
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -940,6 +940,67 @@
       
     
   
+
+      
+        
  • 
+  
+    
+      
+        Layer\<D> — raw mapping, iteration, and batch access
+      
+    
+  
+  
+    
+  
 
    • 
       
         
  • 
@@ -1263,6 +1324,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1638,6 +1755,67 @@
       
     
   
+
+      
+        
  • 
+  
+    
+      
+        Layer\<D> — raw mapping, iteration, and batch access
+      
+    
+  
+  
+    
+  
 
    • 
       
         
  • 
@@ -1773,9 +1951,9 @@
 
 
 
-
    obilayeredmap — layered kmer index crate
    
+
    obikindex::layer — the Layer tier
    
 
    Purpose
    
-
    obilayeredmap implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a ptr_hash MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.
    
+
    obikindex::layer (the layer/ submodule of the obikindex crate — a standalone obilayeredmap crate until 2026-08-21, folded back in alongside obikpartition as part of a broader Index { Partition { Layer } } submodule regrouping) implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a ptr_hash MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.
    
 
    
 
    Three usage modes
    
 
    The MPHF + evidence infrastructure is the same for all modes. The payload varies.
    
@@ -2000,6 +2178,38 @@

    • Mode 3 (PersistentBitMatrix) has no push_layer on LayeredMap; callers build directly via Layer<PersistentBitMatrix>::build_presence.

    +

    Layer\<D> — raw mapping, iteration, and batch access

    +

    Beyond query/find (membership-checked), Layer<D> exposes lower-level access used by consumers that already know a kmer is in the layer (e.g. cross-partition sibling resolution) or that need to sweep every kmer/slot without paying for a membership check each time.

    +

    Raw kmer → slot mapping

    +
    pub fn index(&self, kmer: CanonicalKmer) -> usize
+pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec<usize>
+
    +

    Pure MPHF mapping, no evidence/fingerprint check — equivalent to MphfOnly::index. Only meaningful when the caller already knows kmer belongs to the layer; on an absent kmer the MPHF still returns some slot (undefined, not None).

    +

    Kmer iteration

    +

    Four iterators, all built from unitigs.bin (physical layout order, not correlated with MPHF slot numbers):

    +
    pub fn iter_kmers(&self) -> KmerIter<'_>
+pub fn enumerate_kmers(&self) -> Enumerate<KmerIter<'_>>              // (order_index, kmer)
+pub fn iter_kmers_batch(&self, n: usize) -> KmerBatchIter<'_>          // Vec<CanonicalKmer> of size ≤ n
+pub fn enumerate_kmers_batch(&self, n: usize) -> impl Iterator<Item = (usize, Vec<CanonicalKmer>)> + Send + 'static
+
    +

    KmerIter/KmerBatchIter own a clone of the underlying Arc<UnitigFileReader> rather than borrowing self — Send + 'static, streamed from disk one kmer at a time, never materialised as a whole. Multiple instances can coexist concurrently, each with its own cursor. enumerate_kmers_batch's index is the batch's starting offset in iteration order (a multiple of n except for the final, possibly shorter, batch).

    +

    Batch lookup on payload vectors/views

    +

    PersistentCompactIntVec, PersistentBitVec, IntSliceView, BitSliceView all expose:

    +
    fn get_batch(&self, slots: &[usize]) -> Vec<T>
+fn fill_batch(&self, slots: &[usize], out: &mut [T])
+
    +

    Both sort slots internally for sequential mmap access, then reorder results back to the caller's original order. fill_batch fills a caller-provided buffer, avoiding the Vec allocation.

    +

    sub_matrix / fill_sub_matrix

    +
    // Layer<PersistentCompactIntMatrix>
+pub fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<u32>>            // column-first
+pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<u32>])
+
+// Layer<PersistentBitMatrix> (and any D: BinaryMatrix, e.g. PersistentSparseBitMatrix)
+pub fn sub_matrix(&self, slots: &[usize]) -> Vec<Vec<bool>>
+pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec<bool>])
+
    +

    Column-first to match the on-disk column-major layout. fill_sub_matrix sorts slots once, then calls each column's fill_batch in turn — no redundant per-column sort. On PersistentSparseBitMatrix (k-mer-major, no column method) this degrades to a row-by-row decode; see siblings.md.

    +

    LayeredStore\<S> and aggregation traits

    LayeredStore<S> is a generic aggregation wrapper over Vec<S>. It propagates three traits from obicompactvec::traits up the hierarchy via blanket impls:

    pub struct LayeredStore<S>(pub Vec<S>);
diff --git a/DevDoc/implementation/merge.refs/index.html b/DevDoc/implementation/merge.refs/index.html
index 81792f6b..8f4e288c 100644
--- a/DevDoc/implementation/merge.refs/index.html
+++ b/DevDoc/implementation/merge.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1215,9 +1271,9 @@
 
    Code couvert
    
 
      
 
    • obikindex/src/merge.rs — KmerIndex::merge(), validation de compatibilité d'évidence, validate_evidence_compat()
      • 
-
    • obikpartitionner/src/merge_layer.rs — merge_partition(), construction de la nouvelle layer, paramètre block_bits
      • 
-
    • obikpartitionner/src/rebuild_layer.rs — rebuild_partition(), paramètre block_bits
      • 
-
    • obilayeredmap/src/layer.rs — Layer::append_genome_column() (PersistentCompactIntMatrix et PersistentBitMatrix)
      • 
+
    • obikindex/src/partition/merge_layer.rs — merge_partition(), construction de la nouvelle layer, paramètre block_bits
      • 
+
    • obikindex/src/partition/rebuild_layer.rs — rebuild_partition(), paramètre block_bits
      • 
+
    • obikindex/src/layer/typed_layer.rs — Layer::append_genome_column() (PersistentCompactIntMatrix et PersistentBitMatrix)
      • 
 
    • obicompactvec/src/intmatrix.rs — append_column pour PersistentCompactIntMatrix
      • 
 
    • obicompactvec/src/bitmatrix.rs — append_column pour PersistentBitMatrix
      • 
 
    
diff --git a/DevDoc/implementation/merge/index.html b/DevDoc/implementation/merge/index.html
index d8edd17a..9f91a463 100644
--- a/DevDoc/implementation/merge/index.html
+++ b/DevDoc/implementation/merge/index.html
@@ -733,14 +733,14 @@
   
   
     
  • 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1034,6 +1034,17 @@
     
   
   
+
    • 
+      
+        
  • 
+  
+    
+      
+        Known issue (not yet fixed, 2026-08-28)
+      
+    
+  
+  
 
    • 
       
     
@@ -1157,6 +1168,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
  • 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
    • 
+  
+
+              
+            
           
         
       
@@ -1542,6 +1609,17 @@
     
   
   
+
+      
+        
  • 
+  
+    
+      
+        Known issue (not yet fixed, 2026-08-28)
+      
+    
+  
+  
 
    • 
       
     
@@ -1739,6 +1817,14 @@ spectrums/
   <label>.json               ← one file per genome, rebuilt from all sources
 index.meta                   ← complete genome list + evidence kind written at bootstrap
    +

    mphf.bin/unitigs.bin/evidence.bin/unitigs.bin.idx/fingerprint.bin/layer_meta.json marked "unchanged" above are hard-linked from the base source's own files during the bootstrap copy (2026-08-28), not copied — merge_partition never rewrites them for pre-existing layers, only the presence/counts subdirectory gets widened in place, so only that subdirectory is a real, independent copy. Falls back to a real copy per file if linking itself fails (different filesystems). Verified: source files are byte-identical (checksummed) before/after a normal merge.

    +

    Known issue (not yet fixed, 2026-08-28)

    +

    Merging an index against itself — literally the same directory passed twice as separate source arguments (e.g. obikmer merge -o out IDX IDX --rename-duplicates) — panics deep in the MPHF's rank-select structure (common_traits::select_in_word, assertion failed: rank < self.count_ones()), inside MphfLayer::find called from merge_partition's "is this source kmer already in dst" check against the bootstrap-copied dst_layers. Root cause not identified; ruled out so far:

    + +

    Only reproduces when sources[0] and sources[1] are the exact same on-disk path opened as two independent KmerIndex handles — an artificial scenario (nobody merges an index with itself intentionally), deprioritized rather than investigated further for now. Does terminate cleanly rather than hang, since PartitionRunner's panic propagation fix (see architecture/numa_partition_runner.md) now surfaces it as a normal process panic/exit 101 instead of a silent deadlock.

    diff --git a/DevDoc/implementation/merge_parallelism/index.html b/DevDoc/implementation/merge_parallelism/index.html index 2a11b80e..b7395e98 100644 --- a/DevDoc/implementation/merge_parallelism/index.html +++ b/DevDoc/implementation/merge_parallelism/index.html @@ -733,14 +733,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1152,6 +1152,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + diff --git a/DevDoc/implementation/mphf.refs/index.html b/DevDoc/implementation/mphf.refs/index.html index 522d221a..331707e7 100644 --- a/DevDoc/implementation/mphf.refs/index.html +++ b/DevDoc/implementation/mphf.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + @@ -1214,8 +1270,8 @@

    Coverage: implementation/mphf.md

    Code couvert

    Notes

    FORT RISQUE DE DÉRIVE. Changements récents : diff --git a/DevDoc/implementation/mphf/index.html b/DevDoc/implementation/mphf/index.html index a6d6ce9a..010a5215 100644 --- a/DevDoc/implementation/mphf/index.html +++ b/DevDoc/implementation/mphf/index.html @@ -946,14 +946,14 @@

  • - + - obilayeredmap crate + obikindex layer tier @@ -1163,6 +1163,62 @@ + + + + + + +
  • + + + + + + + + Benchmark: query-path testing + + + + + + + + +
    • + + + + + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + + @@ -1577,7 +1633,7 @@

    Why two phases are needed

    Kmer indexing per partition proceeds in two phases. The separation is necessary because the exact number of surviving unique kmers is not known until after counting and filtering low-abundance kmers.

    Phase 1 — provisional MPHF + kmer spectrum

    -

    Implemented in obikpartitionner::KmerPartition::count_kmer() → count_partition().

    +

    Implemented in obikindex::partition::KmerPartition::count_kmer() → count_partition().

    1. External sort: read the dereplicated superkmer file; extract the raw u64 canonical kmer value for every kmer of every superkmer. Sort in RAM-bounded chunks (adaptive budget: 40% of available RAM ÷ n_threads, minimum 1 M kmers per chunk), then k-way merge with inline dedup. Result: sorted_unique.bin — a flat array of f0 distinct sorted u64 values. Exact kmer count f0 is known at this point.
    2. Build provisional MPHF (ptr_hash, same configuration as phase 2) over sorted_unique.bin using new_from_par_iter. Delete sorted_unique.bin immediately after. Persist to mphf1.bin.
      3. @@ -1667,7 +1723,7 @@

      Multilayer index architecture

      Layer structure

      -

      Each layer is a self-contained unit. See obilayeredmap for the full on-disk layout. The MPHF-relevant files are:

      +

      Each layer is a self-contained unit. See obikindex::layer for the full on-disk layout. The MPHF-relevant files are:

      layer_i/
   unitigs.bin      — packed 2-bit nucleotide sequences (kmer evidence source)
   unitigs.bin.idx  — random-access block index (block_bits controls granularity)
@@ -1702,7 +1758,7 @@ MphfLayer::build_approx_evidence(dir, b, z)
     Uses open_sequential(); never writes .idx

      There is no build_evidence dispatch wrapper. Callers choose the appropriate post-hoc build directly.

      -

      In obikpartitionner, build_index_layer receives block_bits: u8 from IndexConfig::block_bits and forwards it directly to Layer::build and Layer::build_approx_evidence.

      +

      In obikindex::partition, build_index_layer receives block_bits: u8 from IndexConfig::block_bits and forwards it directly to Layer::build and Layer::build_approx_evidence.

      Membership verification

      ptr_hash maps any input to a valid slot — it does not natively detect absent keys. Membership is verified using the evidence entry:

       @@ -1219,7 +1275,7 @@

      Notes

      Document stable (librairie générique, peu de risque de dérive). -Vérifier si obipipeline est toujours utilisé dans la phase scatter de obikpartitionner +Vérifier si obipipeline est toujours utilisé dans la phase scatter de obikindex::partition ou s'il a été remplacé par Rayon dans certains chemins.

      diff --git a/DevDoc/implementation/obipipeline/index.html b/DevDoc/implementation/obipipeline/index.html index 9c6babe6..edebe6b6 100644 --- a/DevDoc/implementation/obipipeline/index.html +++ b/DevDoc/implementation/obipipeline/index.html @@ -896,14 +896,14 @@
    3. - + - obilayeredmap crate + obikindex layer tier @@ -1113,6 +1113,62 @@ + + + + + + +
    4. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      5. + + + + + + + + + + +
    5. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      6. + + + + diff --git a/DevDoc/implementation/obitaxonomy/index.html b/DevDoc/implementation/obitaxonomy/index.html index d477e6ab..b1886a24 100644 --- a/DevDoc/implementation/obitaxonomy/index.html +++ b/DevDoc/implementation/obitaxonomy/index.html @@ -12,7 +12,7 @@ - + @@ -733,14 +733,14 @@
    6. - + - obilayeredmap crate + obikindex layer tier @@ -1125,6 +1125,62 @@ + + + + + + +
    7. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      8. + + + + + + + + + + +
    8. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      9. + + + + diff --git a/DevDoc/implementation/partition_layer_cache/index.html b/DevDoc/implementation/partition_layer_cache/index.html new file mode 100644 index 00000000..1cce3095 --- /dev/null +++ b/DevDoc/implementation/partition_layer_cache/index.html @@ -0,0 +1,3723 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Partition and layer caching (discussion) - obikmer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + + + + Skip to content + + +
      +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      + +
      + + + + + +

      Partition and layer caching (discussion)

      +

      Superseded (2026-08-21): obikpartition and obilayeredmap are no +longer separate workspace crates — both were folded back into obikindex +as submodules (obikindex::partition, obikindex::layer), alongside the +crate's original content as obikindex::index, purely to reduce the +crate count (no behavior change). Every mention of obikpartition/ +obilayeredmap as a crate below, and every dependency-direction +argument phrased in terms of "which crate depends on which" (e.g. "this +crate depends only on obilayeredmap and below, never on obikindex"), +describes that now-superseded split-crate architecture and is kept as-is +for historical context — read obikpartition::X as obikindex:: +partition::X and obilayeredmap::X as obikindex::layer::X throughout. +The underlying module boundary and its rationale (Layer tier / Partition +tier / Index tier, each depending only downward) are unchanged; only the +crate-vs-module packaging changed. See obikindex::layer +for the current module doc.

      +

      Superseded, second event, same day (2026-08-21): obikpartitionner +and obikderep, the two algorithm crates, are also gone — but unlike +obikpartition/obilayeredmap above, they were not folded into +obikindex. They were first (mistakenly) merged into obikindex as an +algorithms submodule, then corrected into a new sibling crate, +obikindexer, holding obikindexer::algorithms::{partitionner, +dereplicator} and depending on obikindex — never the reverse, same +dependency direction obikpartitionner/obikderep already had. Read +obikpartitionner::X as obikindexer::algorithms::partitionner::X and +obikderep::X as obikindexer::algorithms::dereplicator::X throughout +what follows. The distinction the mistake surfaced, worth keeping: data +crates (obikindex, holding the index/partition/layer model) merge +naturally into one crate as submodules; algorithm crates that operate on +that model from outside stay separate, so the dependency only ever runs +one way.

      +

      Status (2026-08-20, latest pass): (1) done — obilayeredmap::Layer +exists, Mat is gone. (1b) done — Layer::Empty, the first non-ready +state, added (panics on every read method). (2a) done — the +obikpartition crate and KmerPartition itself exist (open/n_layers/ +layer/layers/find). (2b) — migrating PartitionCache/QueryLayer +onto it — not started, deliberately deferred. (3) done — the +obikindex ↔ obikpartitionner dependency inverted: PartitionRouter now +takes &mut KmerIndex and produces Layer::Empty shells directly, closing +the gap Layer::Empty was built for in (1b) — see "(3) done" below. (4) +done — dereplication split out into its own crate, obikderep, first step +of an incremental "one algorithm at a time" split of obikpartitionner's +remaining bundle (count_kmer/build_layers not yet moved) — see "(4) +done" below. (5) — full design agreed, not yet implemented — +KmerPartition was found to be wired into nothing (KmerIndex never +calls it; every path is still computed via free functions), and the fix +turned out to be bigger than KmerPartition alone: Layer's own +constructors don't self-name either. Full redesign of both, agreed in +detail, session ended (budget) before implementation — see "(5) design +agreed" below; read it before touching KmerPartition/Layer +signatures, the shape is fully specified. (6) done — Counter, a third +algorithm, extracted from PartitionRouter the same way Dereplicator +was in (4). (7) done — LayerBuilder, the fourth and last pipeline +algorithm; the indexing pipeline is now fully decomposed into +obikindexer::algorithms::{partitionner, dereplicator, counter, +layer_builder}. (8) design agreed, item 1 done in (9) — +obikindexer::extensions::PrivateBuilder, private, six construction-only +KmerIndex methods moved out. (10) done, same session — obikindex:: +IndexBuilder, public, the four maintenance methods +(clear_output_for_create/create_skeleton/finalize_indexed/state) +shared with merge/select/rebuild/reindex. Item 2 from (8) +(obikalgorithm::Algorithm) done in (12) — new crate, type Output + +fn run(&mut self) -> SKResult<Self::Output>, on_progress moved off +run()'s signature entirely into a per-algorithm .on_progress(...) +setter. Note: Layer renamed +KmerLayer (2026-08-21, outside this conversation). (5) itself still not +implemented, still first on the "order of remaining work" list. Earlier +mix-up, for +context: an earlier +version of this doc used the name KmerPartition (singular) for what was +actually the collection type (later renamed KmerPartitions, later +merged into KmerIndex — see "Major restructuring" below), and never +retracted that usage before this section was rewritten. An agent working +from that stale wording built the wrong thing. If you are about to +implement (2), read "Definitions: obikpartition and KmerPartition" +below — it is the current, authoritative naming — before touching any +other section of this file, some of which still describe superseded +states of the code and are kept only as dated history.

      +

      Definitions: obikpartition and KmerPartition (not yet created)

      +

      obikpartition — a new workspace crate, not created yet. Holds the +Partition tier of the Index { Partition { Layer } } model, the same +way obilayeredmap already holds the Layer tier as its own crate +rather than living inside obikindex. Depends only on obilayeredmap +(for Layer) and lower (obikseq, obiskio). Does not depend on +obikindex, obikpartitionner, or obikphylo. Dependency direction: +obikindex → obikpartition → obilayeredmap; obikphylo → obikindex +(and/or obikpartition directly if it ends up needing it without going +through KmerIndex).

      +

      KmerPartition (singular) — the one type this crate exists for. +Represents one partition's already-open layers — a read cache, built +once per partition and held for the run, not rebuilt per lookup. Shape:

      +
      pub struct KmerPartition {
+    layers: Vec<obilayeredmap::Layer>,
+}
+
      +

      Nothing else. In particular: +- No path computation. KmerPartition::open takes an already-resolved + index_dir: &Path (plus mode: &IndexMode, n_layers: usize, + with_counts: bool — whatever it needs, as plain arguments), the same + discipline obikpartitionner::PartitionRouter::open already follows. + Computing index_dir/layer_dir from a partition number is + KmerIndex's job (obikindex, which owns that already — see "Major + restructuring" below); KmerPartition never reaches back into + KmerIndex to get it (would require obikpartition → obikindex, the + wrong direction). +- No routing/write state. Writing raw superkmers, dereplicate, + count_kmer stay in obikpartitionner::PartitionRouter — a completely + different crate, a completely different phase (pre-layer, whereas + KmerPartition only makes sense once layers exist). +- No multi-partition collection baked in. KmerPartition is one + partition. Whatever ends up caching several of them (replacing + obikphylo::siblings::cache::PartitionCache's Vec<Vec<Layer>> and + obikindex::query_layer's per-call reopen) holds Vec<KmerPartition> — + that collection can live in obikpartition too, or in obikindex + alongside KmerIndex; not yet decided, secondary to getting + KmerPartition itself right first.

      +

      Do not confuse with KmerPartitions (plural — note the s): that +type is gone. It used to be obikpartitionner's (nee obikpartition, +briefly — see the crate-rename history below, itself a separate rename +from this one) do-everything struct — routing, dereplication, and path +lookups all in one. It was deleted on 2026-08-20; its read-side (paths, +n_layers, partition_meta) was absorbed into KmerIndex, its +write-side became PartitionRouter. KmerPartition (this section, +singular, no final s) is a brand-new type with a different job, in a +crate that doesn't exist yet — not a revival, not a renaming, of +KmerPartitions.

      +

      Type-to-concept mapping: Index / Partition / Layer

      +

      The conceptual nesting Index { Partition { Layer { MPHF, Evidence, Matrix +} } } }, current state:

      +
        +
      • Index = obikindex::KmerIndex — { root_path, meta: IndexMeta }. + Also directly exposes the partition-path/metadata accessors + (partition_dir(i), index_dir(i), layer_dir(i, l), + partition_meta(i), n_layers(i), partition_mode(i), + n_partitions()) since KmerPartitions merged into it (see "Major + restructuring" below) — KmerIndex today is "index + collection of + partitions' paths & metadata," just without a Vec of open layers.
        • +
      • Partition, the collection = no dedicated type today; the closest + thing is KmerIndex itself (previous bullet). Once KmerPartition + (singular, see Definitions above) exists, a Vec<KmerPartition> + somewhere would be this — still open, see "Direction agreed" below.
        • +
      • Partition, one of them = obikpartition::KmerPartition — to be + built, see Definitions above. Nothing plays this role today; + obikphylo::siblings::cache::PartitionCache and + obikindex::query_layer::QueryLayer each independently reinvent a + fragment of it.
        • +
      • Layer = obilayeredmap::Layer (format-erased: Count/Presence, + each wrapping a TypedLayer<D>) — see "(1) done" below for how this + came to be; TypedLayer<D> ({ mphf: MphfLayer, data: D }, monomorphic) + is the lower-level, D-fixed building block Layer is built on, not + what other crates should reach for directly.
        • +
      • MPHF = MphfLayer.mphf: MemCase<MphfEps> — kmer → slot.
        • +
      • Evidence = MphfLayer.ev: LayerEvidence (Exact/Approx/ + Hybrid — evidence.bin/fingerprint.bin; see EvidenceKind).
        • +
      • Matrix = TypedLayer<D>.data: D — PersistentBitMatrix / + PersistentCompactIntMatrix.
        • +
      +

      Target nesting once KmerPartition exists:

      +
      KmerIndex                                    (obikindex)
+ └─ (opened on demand, per i)  KmerPartition  (obikpartition — not yet built)
+     └─ layers: Vec<Layer>                    (obilayeredmap)
+         └─ Layer::Count/Presence(TypedLayer<D>)
+             └─ TypedLayer<D> { mphf: MphfLayer, data: D }
+                 ├─ mphf.mphf   → MPHF
+                 ├─ mphf.ev     → Evidence
+                 └─ data        → Matrix
+
      +

      Major restructuring (2026-08-20): KmerPartitions merged into KmerIndex

      +

      Prompted by a direct question: why keep KmerIndex/KmerPartitions split +when, one level down, KmerPartitions is going to directly hold +Vec<KmerPartition> rather than being split again into +"collection-holder" + "collection"? Investigating the actual justification +("KmerPartitions has an independent lifecycle, used before an index +exists") turned out to be false — KmerPartitions::create was called +in exactly one place, inside KmerIndex::create, and every +open_with_config reopen outside KmerIndex's own constructors was a +redundant re-derivation of a KmerPartitions already reachable via +index.partition() (the exact kind of duplication this whole doc has been +tracking). Once that was gone, so was the reason to keep them separate.

      +

      Second correction, from the same conversation: obikpartitionner had +accumulated query/merge/select/rebuild/dump/distance logic that has +nothing to do with partitioning super-kmers — it operates on layers, +which don't exist yet at the phase obikpartitionner is actually +responsible for (scatter → dereplicate → count, all pre-layer). That +logic moved to obikindex, which already depends on obilayeredmap and +never needed obikpartitionner for it. No crate-dependency inversion was +needed — obikindex → obikpartitionner stays the same direction as before.

      +

      Result: +- obikpartitionner (renamed back from obikpartition) now contains only + PartitionRouter (superkmer routing: write/write_batch/flush/ + close, dereplicate, count_kmer, KmerSpectrum) and the + partition_dir(root, i) naming primitive both PartitionRouter and + KmerIndex build on. KmerPartitions no longer exists as a type. +- KmerIndex (obikindex) absorbed KmerPartitions's read-side entirely: + partition_dir/index_dir/layer_dir/partition_meta/n_layers/ + partition_mode/n_partitions (the last now derived from + 2^config.n_bits, no longer a stored, independently-set duplicate field + — kmer_size/minimizer_size used to be double-stored, in both + KmerPartitions and IndexMeta.config, a latent-drift risk flagged + earlier in this doc; now single-sourced from IndexMeta.config). Seven + whole files moved from obikpartitionner into obikindex verbatim as + impl KmerIndex blocks, kept as separate files (not merged into + existing same-topic files): index_layer.rs, query_layer.rs, + merge_layer/, select_layer.rs, rebuild_layer.rs, dump_layer.rs, + plus distance.rs's count_store/presence_store (renamed + matrix_store.rs to avoid colliding with obikindex's own pre-existing + distance.rs), and their shared support (common.rs's load_meta/ + olm_to_sk, filter.rs, graph_pipeline.rs). +- obikphylo::siblings::cache::PartitionCache::build now takes &KmerIndex + directly instead of a separately-opened &KmerPartitions — this deleted + the redundant-reopen pattern at all 8 call sites + (alignment/build/cardinality/distance/entropy×2/ + sankoff_bundle/stats), the same bug flagged earlier in this + conversation as a side effect of investigating the false "independent + lifecycle" claim. +- KmerIndex::partition()/partition_mut() are gone; scatter() + (obikmer) and any write-side code get a transient PartitionRouter via + KmerIndex::partition_router(). +- A real bug caught by the test suite during this move: + PartitionRouter::open initially defaulted to closed: true (inherited + from KmerPartitions::open_with_config's old read-only-reopen + semantics), which broke every write through a router obtained via + partition_router(). Fixed — PartitionRouter is exclusively a + write/processing tool now, so open always starts open.

      +

      Full workspace test suite green (0 failed) after, including all 27 +obikphylo::siblings tests.

      +

      (1) done (2026-08-20): Layer is now the heterogeneous handle, Mat is gone

      +

      Resolved the naming question left open above. Layer<D> (the old +generic/monomorphic type) renamed to TypedLayer<D> throughout +(obilayeredmap, obikindex, obikphylo — 12 files, mechanical) to free +Layer for the type that's actually meant to be everyone's default +handle. obilayeredmap::content_layer::Layer (re-exported at the crate +root) is that type — Count(TypedLayer<PersistentCompactIntMatrix>)/ +Presence(TypedLayer<PersistentBitMatrix>), Layer::open doing the same +disk probe Mat::open used to, find_slot/index_batch/n_cols/ +fill_sub_matrix_carries dispatching per variant exactly as Mat did.

      +

      obikphylo::siblings::cache::Mat deleted outright — PartitionCache now +holds Vec<Vec<obilayeredmap::Layer>> directly. The one sibling-specific +method Mat carried (iter_minorants_batch) is not on obilayeredmap:: +Layer (phylo concepts don't belong in obilayeredmap) — it's an +impl SiblingLayerExt for obilayeredmap::Layer in iter.rs, dispatching +to each variant's existing impl<D: LayerData> SiblingLayerExt for +TypedLayer<D>.

      +

      Full workspace suite green (0 failed) after, including all 27 +obikphylo::siblings tests.

      +

      Still not built: (2) — KmerPartition (singular, one partition's open +Vec<Layer>) and a multi-partition cache in obikpartitionner to replace +obikphylo::siblings::cache::PartitionCache and obikindex::query_layer's +still-separate QueryLayer (which still independently bundles MPHF+matrix, +2-way not using Layer at all). Both remaining consumers now sit one +Layer::open call away from unifying onto (2) once it exists.

      +

      (1b) done (2026-08-20): Layer::Empty — the first non-ready-to-read state

      +

      First step toward Layer representing a layer's whole life, not just the +open-for-reading end of it (see "Definitions" above: KmerPartition will +hold Vec<Layer> regardless of each layer's state, states in between +included). Added one variant:

      +
      pub enum Layer {
+    Empty { dir: PathBuf },
+    Count(TypedLayer<PersistentCompactIntMatrix>),
+    Presence(TypedLayer<PersistentBitMatrix>),
+}
+
      +

      Layer::create(dir) makes the directory and returns Empty { dir } — +nothing else; no MPHF/unitigs/evidence construction yet (that's the +deferred next step: build_mphf()/build_unitigs()/build_evidence() +methods to progress Empty → eventually Count/Presence). Empty +carries path accessors so builder code has one place to get +mphf_path()/unitigs_path()/evidence_path()/fingerprint_path()/ +counts_dir()/presence_dir() from, instead of redeclaring the +mphf.bin/unitigs.bin/… filenames at each write site — reusing the +constants layer.rs/mphf_layer.rs already own (COUNTS_DIR/ +PRESENCE_DIR widened from private to pub(crate), file-name constants +already were).

      +

      Every read method (content/evidence_kind/n/find_slot/ +index_batch/n_cols/fill_sub_matrix_carries) panics on Empty with a +one-line message naming the method — confirmed as the right behaviour: +calling any of them on an Empty layer means the caller assumed a layer +was ready when it wasn't, an implementation error to surface loudly, not +a case to design around (Option/Result would let it silently +propagate instead of failing at the actual mistake). Same panic added to +obikphylo::siblings::iter.rs's impl SiblingLayerExt for +obilayeredmap::Layer (4 methods), the one other place that exhaustively +matched Layer's variants.

      +

      Full workspace suite green (cargo check --workspace --all-targets then +cargo test --workspace, exit code 0) after.

      +

      Still deferred, per explicit instruction: build_mphf()/ +build_unitigs()/build_evidence() to progress Empty further, and (2) +— KmerPartition itself — unchanged from above (see "(2a) done" below, +added next).

      +

      (2a) done (2026-08-20): obikpartition crate + KmerPartition

      +

      Built exactly the shape "Definitions" (top of file) specifies, nothing +more — deliberately scoped down from the full "Direction agreed" plan +below: only steps 1–2 (open/n_layers/layer/layers/find), not 3–4 +(migrating PartitionCache/QueryLayer onto it), per explicit +instruction to implement KmerPartition first and decide the wiring +("comment on branche tout ça dans la construction") separately, later.

      +
      pub struct KmerPartition {
+    layers: Vec<obilayeredmap::Layer>,
+}
+
+impl KmerPartition {
+    pub fn open(index_dir: &Path, mode: &IndexMode, n_layers: usize, with_counts: bool) -> OLMResult<Self>;
+    pub fn n_layers(&self) -> usize;
+    pub fn layer(&self, i: usize) -> &Layer;
+    pub fn layers(&self) -> &[Layer];
+    pub fn find(&self, kmer: CanonicalKmer) -> Option<usize>;
+}
+
      +

      open takes index_dir/mode/n_layers/with_counts as plain +arguments — no reach-back into KmerIndex (would need obikpartition → +obikindex, the wrong direction) — and builds each layer's path via +obilayeredmap::layer_dir(index_dir, l), the same shared naming +primitive KmerIndex::layer_dir itself delegates to, not a second copy of +the layer_N convention. find mirrors PartitionCache::find's +semantics (first layer that carries the kmer wins) but doesn't yet cover +find_presence_batch/find_presence_batch_fast — those exist only to +serve PartitionCache, so they're part of the (2b) migration, not this +step; building them now against the current sibling-specific tuple shape +(CanonicalKmer, usize, u8, u8) would either bake phylo vocabulary +(family_idx, base) into obikpartition or require deciding a generic +payload shape — a real design fork, deferred to when (2b) is actually +tackled rather than guessed at here.

      +

      Crate deps: obikseq, obilayeredmap only (dev-deps add obiskio, +obicompactvec, tempfile for tests) — matches the "Definitions" +constraint (obikpartition depends on obilayeredmap and below, never +obikindex/obikpartitionner/obikphylo). Registered as a new workspace +member (src/Cargo.toml). 3 new tests (open_reads_every_layer_in_order, +find_reports_the_first_layer_that_carries_the_kmer, +find_returns_none_for_an_absent_kmer). Full workspace suite green +(cargo check --workspace --all-targets then cargo test --workspace) +after.

      +

      Still not done: (2b) — migrating obikphylo::siblings::cache:: +PartitionCache (currently Vec<Vec<Layer>>) and +obikindex::query_layer::QueryLayer (currently uncached, bypasses Layer +entirely) onto KmerPartition/Vec<KmerPartition>; deciding whether that +collection lives in obikpartition or obikindex; deciding the +batch-lookup surface's exact shape (generic payload vs. as-is sibling +tuple moved in wholesale); scan_layer_families's still-independent +PartitionMeta::load (see "Remaining instance…" below) — all explicitly +deferred to whenever wiring is tackled next.

      +

      (3) done (2026-08-20): obikindex ↔ obikpartitionner dependency inverted, PartitionRouter now fills Layer::Empty shells

      +

      Resolved a question left implicit since "Major restructuring": that pass +set the direction obikindex → obikpartitionner (so KmerIndex could +delegate partition_dir to it) without questioning whether that was the +right direction at all. Challenged directly: obikpartitionner is an +algorithm (superkmer routing/dereplication/counting) operating on an +index (KmerIndex, the data structure) — algorithms depend on the data +types they need, not the other way around. [[feedback_no_precedent_defense]] +applied here: "that's the direction we already picked" was not treated as +a justification for keeping it.

      +

      New direction: obikpartitionner → obikindex (+ obilayeredmap, +obipipeline, obiread directly, for what run's pipeline itself needs). +obikindex → obikpartitionner is gone entirely — KmerIndex no longer +imports PartitionRouter/KmerSpectrum in any form. Two path-naming +primitives that used to make this edge necessary moved down a tier instead +of staying put: +- partition_dir/PARTITIONS_SUBDIR moved from obikpartitionner into + obikpartition (the Partition-tier crate KmerPartition already lives + in), alongside a new index_dir(root, i) — both free functions, + mirroring obilayeredmap::layer_dir one tier down. KmerIndex:: + partition_dir/index_dir now delegate here instead of to + obikpartitionner/an inline .join("index"). +- KmerIndex::create/create_skeleton no longer call + PartitionRouter::create to lay out an empty partitions/ skeleton + upfront — turned out to be dead weight once traced: select_layer.rs/ + rebuild_layer.rs already create_dir_all their own partition/layer + directories on demand, and Layer::create's directory-creation covers + the scatter path the same way. Partitions and their layer-0 shells now + come into existence lazily, on first write, with nothing to pre-create. + KmerIndex::create's now-unused force: bool parameter was dropped + (4 call sites updated) rather than left as a dead parameter.

      +

      PartitionRouter reshaped (obikpartitionner/src/partition/router.rs) +around the "création, paramétrage, run()" shape agreed on: new(index: +&mut KmerIndex) -> Self (no disk access), chainable setters +(level_max/theta/workers/max_open, defaults matching the CLI's old +hardcoded values), then run(path_source, on_progress). write/ +write_batch/flush/close/dereplicate/count_kmer stay public, +unconsumed (&self/&mut self, not self) — callers needing fine-grained +control (tests, obikphylo's test harness) still get it, run is a +convenience layered on top, not the only way in.

      +

      run absorbs the entire body of what used to be the free function +obikmer::steps::scatter (now deleted, along with the steps module +entirely) — the obipipeline::make_pipe! two-stage pipeline +(file→pages→superkmers), throttling, per-file logging. What changed: +- Every ensure_writer(partition) call now does Layer::create(&layer0_dir) + (layer0_dir = obilayeredmap::layer_dir(&index.index_dir(i), 0)) before + opening raw.{ext} inside it — raw/dereplicated superkmer files and the + provisional mphf1.bin/counts1.bin/kmer_spectrum_raw.json now live + under <partition>/index/layer_0/, not flat under <partition>/ as + before. This is Layer::Empty actually being used as the "builder code + holding an Empty layer" its own (1b) docs anticipated, not just a shell + with no consumer. + - Caught by an end-to-end smoke test, not by cargo test: this path + move broke obikindex::index_layer::build_index_layer and + remove_build_artifacts, both of which still read/deleted + dereplicated.skmer.zst/mphf1.bin/counts1.bin from + self.partition_dir(i) (the old flat location) — no test in the + workspace suite exercises the real CLI's file-reading scatter path + end-to-end (obikphylo's test harness and obikpartitionner's own + tests both call write_batch directly, bypassing run/file discovery + entirely), so the whole suite stayed green while obikmer index on + real FASTA silently indexed 0 kmers. Found by running the actual CLI + against a small FASTA and noticing count.json's f0 (870, correct) + didn't match "0 total kmers indexed" at the final stage. Fixed by + retargeting both functions to self.layer_dir(i, 0). Lesson, + consistent with the retracted-claim lesson above: a green test suite + is not proof a refactor is correct when no test in it exercises the + specific path that changed — for anything touching the CLI's own + file-driven entry point, running the CLI for real is not optional + verification. +- The internal obisys::spinner("scatter") + hand-rolled EMA-rate display + is gone from the library entirely, replaced by an Option<impl + FnMut(obisys::Progress)> parameter — a new, deliberately generic + progress-reporting type (obisys::Progress { position: u64, total: + Option<u64> }, alongside the existing TracedBar/spinner/ + progress_bar) added specifically so every future algo crate's run() + reports progress the same shape, once, rather than each inventing its + own. total: None here (bases processed isn't knowable without + pre-scanning every input file) — deliberately simpler than the old + in-library rate/file-count/thread-count message; the caller can + recompute a Mbp/s rate from consecutive position values + + wall-clock time itself, which is exactly what cmd/index/mod.rs now + does to reproduce the old spinner message. This is a real, intentional + restriction of the library's job: it reports raw ticks, the CLI decides + what a human sees — same "generic vs. domain-specific" split applied + again, this time to progress reporting rather than to Layer content. + Explicitly not the same mechanism as Stage/Reporter (per + [[feedback_stage_reporter_in_cmd_layer]]): Stage/Reporter measures a + whole call's wall time from outside it; a progress callback has to fire + from inside a loop mid-call, which wrapping from outside cannot + express — two different needs, not the same rule reapplied under a new + name. Stage::start("scatter")/rep.push(...) stayed in + cmd/index/mod.rs, wrapping the whole run() call, unchanged in kind. +- dereplicate/count_kmer keep their existing internal + obisys::progress_bar(...) calls as-is (unconverted to the callback) — + explicitly out of scope for this pass, by agreement.

      +

      Forced, not optional, consequence of the dependency inversion: +KmerIndex::dereplicate_and_count/partition_router/write_spectrum(& +KmerSpectrum) could not stay on KmerIndex at all once obikindex can no +longer name obikpartitionner::{PartitionRouter, KmerSpectrum} in any +position — not a design choice, a mechanical requirement of severing the +edge. Replaced by: KmerIndex::write_spectrum(f0: u64, f1: u64, counts: +&BTreeMap<u32, u64>) (plain values, no KmerSpectrum dependency) and a +new KmerIndex::mark_counted() (symmetric to the already-existing +mark_scattered), with the orchestration itself (router.dereplicate() → +router.count_kmer() → write_spectrum → mark_counted()) now living in +cmd/index/mod.rs, not obikindex.

      +

      Every PartitionRouter::new(&mut index) call in this codebase runs into +the same NLL trap once: PartitionRouter has a Drop impl (auto-close +on scope exit), which extends its &mut KmerIndex borrow to the end of +the enclosing scope even after its last real use — idx.mark_scattered() +right after router.run(...) (or idx.write_spectrum(...) right after +router.count_kmer(...)) fails to borrow-check unless the router is +drop()-ed explicitly first. Hit and fixed identically at all three call +sites that needed it (cmd/index/mod.rs ×2, obikphylo's test harness, +obikpartitionner's own tests).

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0) both before and after the +index_layer.rs fix above — the smoke test is what actually caught the +regression the suite missed.

      +

      (4) done (2026-08-20): obikderep — dereplication split out of obikpartitionner, one algorithm at a time

      +

      Follow-on question after (3): the indexing pipeline has 4 stages (scatter, +dereplicate, count_kmer, index-build — see the CLI's own Reporter output, +one line per stage), but obikpartitionner — a name that says +partitioning — owned three of them (routing, dereplication, counting). +Challenged directly, same as (3)'s dependency-direction question: a crate +should hold what its name says, not accumulate unrelated stages just +because they happened to land there first. Two ways to fix it — one crate +renamed to hold all remaining stages, or one crate per stage — decided in +favour of the latter, explicitly incremental: build the second algo +crate first (obikderep, dereplication only), only then look at what it +and PartitionRouter actually have in common, and factor a shared +Algorithm trait (future obikalgorithm crate) from that real overlap — +not guessed at from a single example. count_kmer and build_layers +(currently KmerIndex inherent methods — itself flagged as inconsistent +with "KmerIndex is a data structure, not a compute structure") are left +alone this round, on purpose — one stage moves at a time.

      +

      obikderep (new crate): Dereplicator<'a> { index: &'a KmerIndex, n_partitions, level } +— new(index: &KmerIndex) (shared borrow, not &mut: dereplication never +writes index metadata), no setters yet (nothing to configure), run(on_progress) +does the two-phase split+merge dereplication in parallel across partitions, +ported unchanged from PartitionRouter::dereplicate (moved wholesale: +optimal_buckets/dereplicate_partition/load_bucket/flush_map/ +remove_skmer_file, now private to this crate in dereplicate.rs). +obikpartitionner::PartitionRouter::dereplicate is gone; count_kmer +stays.

      +

      A real signature difference from PartitionRouter::run, not an +inconsistency: Dereplicator::run takes Option<impl Fn(Progress) + +Sync>, not FnMut. PartitionRouter::run's callback is invoked from one +sequential loop (FnMut is fine); Dereplicator::run's work is +rayon::par_iter, so the callback can be invoked concurrently from +multiple worker threads — same reason obisys::TracedBar's own methods +take &self, not &mut self. Progress position is tracked with an +AtomicU64, incremented from inside the parallel closure so each +completed partition reports immediately — collecting all results first and +reporting after (the first draft of this) would have delivered every tick +in one burst at the very end, defeating the point of a live progress bar. +total: Some(n_partitions) (known up front, unlike scatter's bases count) +— cmd/index/mod.rs renders a real progress_bar, not a spinner, driven +by the callback exactly like scatter's spinner is.

      +

      A second, pre-existing instance of the exact bug (3) fixed, caught +before it shipped: dereplicated.skmer.zst was hand-built as a string +literal independently in three places — obikpartitionner's +dereplicate.rs/count.rs and obikindex's index_layer.rs (a literal +that already predated this session, never caught until now). Splitting +dereplication into its own crate turns this from "two places, still +matching by luck" into "three independent crates that must agree on a +filename with no shared dependency forcing them to" — no longer +deferrable. Fixed by adding obilayeredmap::{raw_superkmers_path, +dereplicated_superkmers_path} (free functions, layer_dir: &Path -> +PathBuf, mirroring layer_dir itself) — the filename lives in one place, +in the Layer-tier crate every consumer here already depends on +(obikpartitionner, obikderep, obikindex all reach it without a new +edge), and no external crate ever sees the literal "skmer.zst" again. +This reverses (3)'s own earlier call to keep SK_EXT private to +obikpartitionner — that call assumed a single owner; a second owner +appearing (obikderep) removed the assumption it rested on, so the +decision changed with it, not out of inconsistency.

      +

      Every count_kmer call site that used to run after router.dereplicate() +on the same PartitionRouter now runs after a separate +Dereplicator::new(&idx).run(...) call, on a freshly-constructed +PartitionRouter — PartitionRouter no longer offers a combined +"dereplicate then count" path. Updated at all three call sites that had +one: cmd/index/mod.rs, obikphylo's test harness, obikpartitionner's +own tests.

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0), plus an end-to-end CLI smoke test +against real FASTA data (scatter → dereplicate → count → index-build → +query, same numbers as (3)'s smoke test: 870 kmers) — required this time +too, per (3)'s own lesson: no test in the suite exercises obikmer index's +real file-driven path.

      +

      Still not done: count_kmer/build_layers staying where they are, the +obikalgorithm shared-trait extraction (deliberately deferred until a +third data point exists), and everything already listed under (2b).

      +

      (5) design agreed, not yet implemented (2026-08-20): KmerPartition rewritten, Layer gains self-naming, a future cache crate over KmerIndex

      +

      Session ended (out of budget) before any of this was coded. Everything +below is a fully specified plan, agreed sentence by sentence with the +user — not a sketch to re-derive, not a proposal to re-litigate. Implement +it as written; if something here turns out to be wrong once coded, fix it +and update this section, don't restart the design conversation.

      +

      How this was found

      +

      Direct question from the user: "tu as bien créé une structure +KmerPartition ?" — yes (2a), but investigating exposed that it is +wired into nothing. KmerIndex has no partition(i) method at all; +partition_dir/index_dir/layer_dir still call obikpartition:: +partition_dir/index_dir and obilayeredmap::layer_dir as bare free +functions directly, never touching a KmerPartition/Layer object to get +there. The end result on disk is identical (same paths), which is exactly +why no test caught it — but the responsibility is in the wrong place: +one function (on KmerIndex) knows the whole three-tier naming +convention, instead of each tier asking the one below it for its own +path. User's framing, verbatim, now saved as [[feedback_no_spaghetti_petits_pois]]: +"spaghetti" (logic untraceable, split across too many unrelated crates) +and "petits pois" (small bits of naming logic dispersed with no owning +object) are strictly forbidden — this was a live example of both.

      +

      Pushed further, twice: +1. First correction: Layer::create(&obilayeredmap::layer_dir(&dir, 0)) — + still a free-function call from outside Layer to compute where it + should live. "Le layer n'est pas con, c'est lui qui dit où est-ce qu'il + doit être sauvé" (the layer isn't stupid, it says itself where it + should be saved). +2. Second correction, the general principle: "une partition est juste + identifiée par un numéro, tout se calcule à partir du numéro, et un + layer est identifié à partir d'un numéro et tout se calcule à partir de + ce numéro." Concretely: each object stores its own local identifying + number plus its immediate parent's path (captured once, at + construction) — never a path handed in again later by a caller, and + never a free function outside the object that can compute that path + independently. Explicitly rejected along the way: making users pass + "the partition's path that contains the layer" to open a layer — the + parent path is captured once, at the child's construction, not + re-supplied at every call.

      +

      The agreed shape

      +

      Layer (obilayeredmap) — identified by l + its parent partition's +directory, both captured at construction, never received again:

      +
      pub enum Layer {
+    Empty { partition_dir: PathBuf, l: usize },   // pure identification, no disk I/O
+    Count(TypedLayer<PersistentCompactIntMatrix>),
+    Presence(TypedLayer<PersistentBitMatrix>),
+}
+
+impl Layer {
+    pub fn at(partition_dir: &Path, l: usize) -> Self;                 // identify only
+    fn dir(&self) -> PathBuf;                                          // private — layer_dir() no longer a public free function, folded in here
+    pub fn create(self) -> io::Result<Self>;                           // creates the directory if needed; no path parameter anymore
+    pub fn open(self, mode: &IndexMode, with_counts: bool) -> OLMResult<Self>;  // no path parameter anymore
+    // mphf_path()/unitigs_path()/evidence_path()/fingerprint_path()/counts_dir()/presence_dir()
+    // unchanged in spirit, implemented via self.dir() instead of a stored `dir` field read directly
+}
+
      +

      Note this replaces Layer::Empty { dir: PathBuf } from (1b) — dir +becomes a computed value (partition_dir.join(format!("layer_{l}"))), not +a stored field. obilayeredmap::layer_dir/raw_superkmers_path/ +dereplicated_superkmers_path (currently public free functions, +introduced in (3)/(4)) stop being called from outside obilayeredmap +entirely once this lands — they were the right fix for their moment (a +second crate, obikderep, needed to agree on a filename with no owner), +but the real fix, now visible with a third data point, is that Layer +itself should be the only thing anyone asks.

      +

      KmerPartition (obikpartition) — same principle, one tier up:

      +
      pub struct KmerPartition {
+    index_root: PathBuf,   // the parent KmerIndex's root, captured once
+    i: usize,
+}
+
+impl KmerPartition {
+    pub fn new(index_root: PathBuf, i: usize) -> Self;                 // identify only, no disk I/O
+    pub fn create(index_root: PathBuf, i: usize) -> io::Result<Self>;  // creates this partition's directory + an empty layer 0 (a partition is never born without one — that knowledge lives here, not in whoever calls create)
+    pub fn partition_dir(&self) -> PathBuf;                            // part_{i:05}
+    pub fn index_dir(&self) -> PathBuf;                                // part_{i:05}/index
+    pub fn layer(&self, l: usize) -> Layer;                            // Layer::at(&self.index_dir(), l) — caller never touches a path
+    pub fn meta(&self) -> SKResult<PartitionMeta>;                     // n_layers + mode; must absorb the recovery-on-missing-file logic
+                                                                         // currently private in obikindex::common::load_meta (obikpartition
+                                                                         // can't depend on obikindex to reuse it — this logic moves down)
+    pub fn n_layers(&self) -> SKResult<usize>;                         // meta()?.n_layers
+    pub fn mode(&self) -> SKResult<IndexMode>;                         // meta()?.mode — "exact/approximatif"
+    pub fn is_filled(&self) -> bool;                                   // does this partition's directory exist at all
+    pub fn n_kmers(&self) -> io::Result<usize>;                        // LayerMeta::load(&self.layer(0).dir()).n — reads layer 0's count as
+                                                                         // a representative figure, same "read the first one" trick
+                                                                         // n_layers_per_partition() already uses at the KmerIndex level
+}
+
      +

      This replaces (2a)'s KmerPartition { layers: Vec<Layer> } entirely +— no eagerly-opened Vec<Layer>, no find() (both belong to the future +cache, see below, which is the thing that actually holds opened layers +alive across many lookups). (2a)'s version is safe to delete outright: it +was never wired into anything (confirmed above), so nothing depends on +its current shape. New dependencies needed: obikpartition gains +obiskio (for SKResult) and obicompactvec (for LayerMeta).

      +

      Deliberately not built this round: cross-level consistency checks +("verify everything below me is in the same state") — a real idea, raised +by the user, but nothing concrete needs it yet; building it speculatively +would be exactly the premature-abstraction pattern this project avoids.

      +

      KmerIndex (obikindex) — becomes the sole entry point:

      +
      pub fn partition(&self, i: usize) -> KmerPartition;   // KmerPartition::new(self.root_path.clone(), i)
+
      +

      partition_dir(i)/index_dir(i)/layer_dir(i, l) stay as public +methods (≈30 existing call sites across obikindex/obikphylo — see (3)'s +option A, applied identically here) but become pure delegations: +self.partition(i).partition_dir(), self.partition(i).index_dir(), +self.partition(i).layer(l).dir() (needs Layer::dir() to be visible +enough for this — likely pub(crate) in obilayeredmap plus a thin +public wrapper, or a public accessor on Layer itself; not fully nailed +down, decide while implementing). No caller outside obikindex changes.

      +

      Known blast radius (why this wasn't done in the same session)

      +
        +
      • 14 files call Layer::open/Layer::create directly today + (obikphylo/siblings/{cache,build,family_scan,tests}.rs, + obikpartitionner/partition/router.rs, obikpartition/src/lib.rs, + obikindex/{rebuild_layer,dump_layer,index,query_layer}.rs, + obilayeredmap/{mphf_layer,layer,map,content_layer}.rs) — every one + loses its path parameter and gains a (partition_dir, l) or an + already-identified Layer to call .create()/.open() on instead.
        • +
      • ≈30 files call KmerIndex::partition_dir/index_dir/layer_dir — + unaffected in their own code (same public signatures), but worth + re-checking once (5) lands that none of them were relying on the old + free-function-based implementation in a way the new delegation breaks.
        • +
      • obikpartitionner::PartitionRouter::ensure_writer and obikderep's + run both currently call obilayeredmap::{layer_dir, raw_superkmers_path, + dereplicated_superkmers_path} directly (from (3)/(4)) — both need to + switch to going through index.partition(i).layer(0) instead.
        • +
      +

      Also agreed, separately: PartitionRouter::new never needed &mut KmerIndex

      +

      Verified by reading the code: every call PartitionRouter makes on +index is &self (index.kmer_size(), index.index_dir(i)). The &mut +in its current signature (from (3)) was inherited from the original +"the router writes to the partitions" reasoning, never actually required +by any method call. This is exactly what caused every drop(router) +workaround needed throughout (3)/(4) (cmd/index/mod.rs ×2, obikphylo's +test harness, obikpartitionner's own tests) — PartitionRouter holds a +Drop impl, which extends a &mut borrow to the end of its scope even +past its last real use. Fix alongside (5): change +PartitionRouter::new(index: &'a mut KmerIndex) to &'a KmerIndex, and +remove the now-unnecessary drop(router) calls at all four sites.

      +

      Also discussed: a future cache crate, not part of (5), not obikalgorithm either

      +

      Separate idea, explicitly not part of this design and not started: +a new crate whose only job is to cache open KmerPartitions (and their +opened Layers) across one run — replacing both obikphylo::siblings:: +cache::PartitionCache (today, sibling-specific, holds Vec<Vec<Layer>>) +and obikindex::query_layer::QueryLayer (today, uncached, bypasses +Layer entirely) — the two consumers (2b) already identified as each +reinventing a fragment of the same thing.

      +

      User's framing: this is not a third obikalgorithm data point — an +algorithm has a new → run → done shape; a cache has a fundamentally +different one (open, stay alive for a whole run, serve lookups, maybe +evict) — "on crée un cache sur un index, ça consomme un index." Two +distinct crate roles in this ecosystem (data crates: obikpartition/ +obilayeredmap; algorithm crates: obikpartitionner/obikderep/future +obikalgorithm implementors; and now a cache/service crate), not one +unified shape to force everything into.

      +

      Depends on (5) being done first: the cache crate's whole job is holding +Vec<KmerPartition>/opened Layers alive, built via KmerIndex:: +partition(i) as its factory — nothing to build it on top of until (5) +lands. Still open once (5) is done: eviction policy vs. holding everything +open for the process lifetime (the never-measured mmap/VM-mapping-count +question from earlier in this doc), and whether it lives in obikpartition +itself or a new crate.

      +

      Order of remaining work, as currently understood

      +
        +
      1. (5) — Layer/KmerPartition/KmerIndex rewrite described above, + plus the PartitionRouter &mut → & fix (same root cause, same + session, do together).
        2. +
      2. The future cache crate (name not chosen), consuming KmerIndex:: + partition(i) — unblocks migrating PartitionCache/QueryLayer (2b).
        3. +
      3. obikalgorithm — still deliberately waiting for a third run()-shaped + data point (count_kmer or build_layers migrating out of + KmerIndex/PartitionRouter) before extracting a shared trait; two + examples were judged not enough to be sure of the shape (Fn+Sync vs + FnMut callback bound already diverged between the two that exist).
        4. +
      +

      (6) done (2026-08-21): Counter — third algorithm, extracted the same way as Dereplicator

      +

      Between (5) and this, the user did a session of their own crate +restructuring (see the two "Superseded" notes at the top of this file): +obikpartition/obilayeredmap folded into obikindex as submodules +(obikindex::partition, obikindex::layer), and obikpartitionner/ +obikderep merged into one sibling crate, obikindexer, holding +obikindexer::algorithms::{partitionner, dereplicator}. (5)'s design +(Layer/KmerPartition self-naming by number, PartitionRouter's +&mut → & fix) was not part of that — pure crate/module packaging, +confirmed by reading the actual code (Layer::open/create still take an +external dir: &Path, KmerPartition still eagerly opens all layers, +PartitionRouter still holds &mut KmerIndex). (5) remains exactly as +specified, not yet implemented.

      +

      This step: count_kmer (still living on PartitionRouter, per (4)'s own +"still not done" note) extracted into obikindexer::algorithms::counter:: +Counter, mirroring Dereplicator exactly — third data point for the +eventual obikalgorithm trait, still not extracted (still only 3 examples +with 2 different callback bounds; holding off per (5)'s "order of +remaining work").

      +
      pub struct Counter<'a> {
+    index: &'a KmerIndex,
+    n_partitions: usize,
+    keep_partial: bool,
+}
+
+impl<'a> Counter<'a> {
+    pub fn new(index: &'a KmerIndex) -> Self;
+    pub fn keep_partial(mut self, v: bool) -> Self;   // setter, mirrors PartitionRouter's style; defaults to false
+    pub fn run(&self, on_progress: Option<impl Fn(Progress) + Sync>) -> SKResult<KmerSpectrum>;
+}
+
      +

      Same shape as Dereplicator throughout: Fn(Progress) + Sync (not +FnMut) since counting is also a parallel par_iter over partitions, an +AtomicU64 position counter incremented from inside the parallel closure +so progress reports arrive in real time rather than bursting at the end +once .collect() finishes, total: Some(n_partitions) (known up front). +KmerSpectrum (the {f0, f1, counts} aggregate) moved from +partitionner::router to counter, since it's Counter::run's return +value now, not PartitionRouter's. count.rs/kmer_sort.rs moved +verbatim from partitionner/ to counter/ (unchanged bodies — only +count_kmer itself, KmerSpectrum, and the imports they pulled in were +removed from router.rs).

      +

      One divergence from Dereplicator: a keep_partial setter exists (no +equivalent on Dereplicator, which has no setters at all) — a real, +already-present parameter (keep_intermediate at the CLI), not a +speculative addition.

      +

      count_kmer's three former callers (obikmer::cmd::index, obikphylo's +test harness, obikindexer::algorithms::partitionner's own +pipeline_counts test helper) all updated to Counter::new(&idx). +run(...) — the last one simplified further: it used to read back +kmer_spectrum_raw.json from disk after calling count_partition +directly (white-box), now it just uses the KmerSpectrum Counter::run +already returns.

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0), plus an end-to-end CLI smoke test +against real FASTA data (scatter → dereplicate → count → index-build → +query) — required every time per (3)'s lesson, and it earned its keep +again: the very first smoke-test query returned zero matches, which +looked like a regression until traced to the query sequence itself being +low-complexity ("GGCCCCCCACG", six same-base runs) and rejected by +query's own default entropy threshold — nothing to do with this change. +Re-tested with a different substring, confirmed working (kmer found, +count matched the index).

      +

      Still not done: (5) (Layer/KmerPartition redesign, PartitionRouter's +&mut→&), the future cache crate, build_layers (still a KmerIndex +inherent method, not an algorithm), and obikalgorithm itself.

      +

      (7) done (2026-08-21): LayerBuilder — fourth and last pipeline algorithm

      +

      Closes out the indexing pipeline: build_layers/build_index_layer +(the last stage still living as KmerIndex inherent methods, flagged as +inconsistent since (6)) extracted into obikindexer::algorithms:: +layer_builder::LayerBuilder, same two-phase shape as the other three.

      +
      pub struct LayerBuilder<'a> {
+    index: &'a KmerIndex,
+    n_partitions: usize,
+    min_abundance: u32,
+    max_abundance: Option<u32>,
+    keep_intermediate: bool,
+}
+
+impl<'a> LayerBuilder<'a> {
+    pub fn new(index: &'a KmerIndex) -> Self;
+    pub fn min_abundance(mut self, v: u32) -> Self;
+    pub fn max_abundance(mut self, v: Option<u32>) -> Self;
+    pub fn keep_intermediate(mut self, v: bool) -> Self;
+    pub fn run(&self, on_progress: Option<impl FnMut(Progress) + Send>) -> SKResult<usize>;  // returns total kmers built
+}
+
      +

      Different from all three prior extractions in one respect, deliberately +not "fixed" to match them: the actual per-partition construction logic +(De Bruijn graph from dereplicated superkmers + provisional counts → +unitigs → MPHF → matrix) stayed put as KmerIndex::build_index_layer/ +remove_build_artifacts (both already pub) — not moved into +obikindexer. Checked first: unlike dereplicate_partition/ +count_partition (which only ever had one caller), build_index_layer +depends on several obikindex-internal helpers (graph_pipeline:: +{write_graph_as_unitigs, materialize_layer}, common::olm_to_sk) that +are pub(crate) and shared with merge/select/rebuild's own +layer-construction paths — moving build_index_layer out would have +meant either exporting that internal surface just for this one algorithm +or duplicating it. Neither was needed: build_index_layer/ +remove_build_artifacts were already public KmerIndex methods, so +LayerBuilder's job is purely the orchestration around them (scheduling, +config, progress) — the exact same "algorithm calls already-public +KmerIndex primitives" shape PartitionRouter/Dereplicator/Counter +already have, just at a coarser grain for this one stage. This is the +"is the producer's API actually deficient?" check from +[[feedback_no_spaghetti_petits_pois]] applied and answered "no" — not +skipped.

      +

      Two more real divergences, both forced by PartitionRunner, not +arbitrary: +- Uses obikindex::PartitionRunner (NUMA-aware scheduler, already + pub used from obikindex) instead of plain rayon::into_par_iter + like Dereplicator/Counter — matches what build_layers already used + before extraction; this stage is more CPU/memory-intensive per partition + (graph construction) than scatter/dereplicate/count. +- Callback bound is FnMut(Progress) + Send — a third variant, not + matching either prior shape. PartitionRunner::run's on_done is + invoked from its own single controller thread (never concurrently, so + no Sync needed, unlike Dereplicator/Counter's Fn + Sync), but + that controller thread is itself std::thread::scope-spawned, so the + closure still has to be Send to cross into it — caught immediately by + the compiler (cannot be sent between threads safely) when Send was + first omitted, not a design guess. obikalgorithm's eventual shared + trait now has three real callback-bound data points to reconcile + (FnMut alone for PartitionRouter::run's sequential loop, FnMut + + Send here, Fn + Sync for Dereplicator/Counter's rayon + par_iter), not two.

      +

      KmerIndex::build_layers deleted outright (KmerIndex stays a pure data +structure — no compute orchestration methods, consistent with dereplicate/ +count_kmer's removal in (4)/(6)). New KmerIndex::mark_indexed() added, +symmetric to mark_scattered/mark_counted, replacing the inline +touch(SENTINEL_INDEXED) that used to live inside build_layers. +Stage::start("index")/rep.push(...) and the progress_bar/ +"{n} total kmers indexed" log line both moved to cmd/index/mod.rs, +same pattern as (3)/(4)/(6) — LayerBuilder renders nothing itself, just +reports Progress.

      +

      All callers updated: cmd/index/mod.rs (Stage 3), obikphylo's test +harness (also gained a mark_indexed() call it was missing — harmless +before since nothing checked IndexState::Indexed in that test, but now +correct).

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0), plus the end-to-end CLI smoke test +(scripts/smoke_test_index.sh, built earlier specifically so this +verification step is a one-liner from now on) — 870 kmers indexed, query +round-trip confirmed, same numbers as (6).

      +

      The indexing pipeline is now fully decomposed: obikindexer:: +algorithms::{partitionner, dereplicator, counter, layer_builder}, each a +new/(setters)/run algorithm operating on a &KmerIndex (or &mut for +PartitionRouter, not yet fixed — see (5)), KmerIndex itself holding no +pipeline-orchestration logic anymore. Still not done: (5), the future +cache crate, obikalgorithm (now unblocked — three real callback-bound +variants observed, worth revisiting whether a single trait can express +all three or whether that's itself the answer: it can't, and the trait +should not force it).

      +

      (8) design agreed, not yet implemented (2026-08-21): obikalgorithm::Algorithm trait + obikindexer::extensions — private/public extension-trait split, KmerLayer rename

      +

      Session note: Layer was renamed KmerLayer (user, outside this +conversation, alongside other naming homogenisation with KmerIndex/ +KmerPartition) — every reference to Layer in this doc from before +2026-08-21 means today's obikindex::layer::KmerLayer.

      +

      Why this came up

      +

      Verifying "does cmd/index now rest entirely on the algorithm structs" +(it doesn't quite — see below) led to sorting KmerIndex's own methods by +a criterion the user was explicit is semantic, not mechanical: "les +méthodes qui, sémantiquement, n'ont pas d'intérêt hors de la construction +de l'index" (methods that have no semantic interest outside index +construction) — not "methods only called from cmd/index today," which +a grep could answer but would miss methods construction-adjacent code +elsewhere (merge/select/rebuild/reindex) also depends on for the +same reason.

      +

      Checked, not assumed (grepped every call site before classifying):

      +
        +
      • Construction-only, real candidates for a private extension trait: + KmerIndex::{mark_scattered, mark_counted, mark_indexed, write_spectrum, + build_index_layer, remove_build_artifacts, clear_output_for_create, + create_skeleton, finalize_indexed, state}. The last four are called + from merge.rs/select.rs/rebuild.rs/reindex.rs too (as + precondition checks — "is my source Indexed?" — or shared + skeleton/finalize machinery), not just from the 4-stage pipeline — so + this extension trait's scope is "construction of any kind," not + narrowly "the initial build pipeline."
        • +
      • Looked construction-only by name, checked, and kept on KmerIndex: + layer_unitigs_path (unitigs are the only way to recover a built + index's kmer sequences — read by rebuild_layer.rs and others, well + beyond construction — see [[project_unitigs_always_kept]]), + pack_matrices (re-runnable maintenance on an already-finished index + via obikmer pack, not just a pipeline step), upgrade_layer_meta + (migration, runnable on any existing index at any time).
        • +
      +

      The general pattern (not obikindexer-specific)

      +

      KmerIndex/KmerPartition/KmerLayer stay generic, in obikindex — +every domain-specific consumer crate gets to attach its own extension +trait(s), of two kinds:

      +
        +
      • Private (pub(crate), invisible outside the defining crate) — for + plumbing only that crate's own algorithms need. obikindexer gets + exactly one of these (see below); no public counterpart makes sense for + it — "l'index est tellement central que le second trait n'a pas + vraiment d'intérêt" for construction specifically: nothing external + should ever want to call mark_scattered or build_index_layer.
        • +
      • Public — for a genuinely reusable domain extension. The user's own + example, found while discussing this, not hypothetical: obikindex/src/ + index/distance.rs (phylogenetic distance metrics) is currently an + impl KmerIndex block inside obikindex itself — under this + principle it should be a public extension trait owned by obikphylo + instead (distance metrics are a phylo concept, obikindex has no more + business defining them than obikindex::layer has defining + "family"/"minorant", the reasoning SiblingLayerExt already followed + for KmerLayer — see obikphylo/src/siblings/iter.rs). Explicitly + deferred — noted here so it isn't lost, not part of this round.
        • +
      • The future cache-manager crate (still blocked on (5), see above) will + add its own public extension trait mirroring part of KmerIndex's/ + KmerPartition's own read API in cached form (e.g. a cached + .partition(i) that doesn't re-touch disk) — same pattern, third data + point once built.
        • +
      +

      Concretely, next to implement (two items, in order)

      +
        +
      1. obikindexer::extensions — a private (pub(crate)) extension + trait, most likely named something like IndexBuildExt (final name + not yet chosen), implemented for KmerIndex, carrying the ten methods + listed above, moved out of obikindex::index::{kmer_index, + index_layer}. Every algorithm in obikindexer::algorithms::* that + currently calls idx.mark_scattered()/etc. keeps the same call syntax + (extension trait methods are called the same way as inherent ones, + just need the trait in scope) — cmd/index/mod.rs itself would need + use obikindexer::extensions::IndexBuildExt; (or the module re-exports + it) to keep compiling, since it's the one place outside obikindexer's + own algorithms that currently calls mark_scattered/write_spectrum/ + mark_counted/mark_indexed directly. Not yet decided: exact + trait name, whether it's one trait or split further (e.g. sentinel + marking vs. skeleton/finalize machinery), and whether merge/select/ + rebuild/reindex (not yet extracted into algorithms themselves) move + onto it now too or keep calling the soon-to-be-inherent-no-longer + methods some other way in the meantime — ask before implementing, + this changes the blast radius significantly (4 more obikindex + internal files depend on clear_output_for_create/create_skeleton/ + finalize_indexed/state).
        2. +
      2. obikalgorithm::Algorithm trait — new crate, the shared trait + obikpartitionner→obikindexer merge (session start of 2026-08-21) + and (6)/(7) were deliberately building toward, now with four real + new/(setters)/run examples and three distinct callback-bound + shapes to reconcile (plain FnMut for PartitionRouter, FnMut + + Send for LayerBuilder, Fn + Sync for Dereplicator/Counter — + see (7)). Exact shape not yet drafted in this doc — do that as its own + design pass before coding, same discipline as everything above.
        3. +
      +

      Both items: design only, nothing implemented yet — this section is +the record to resume from, not a plan already executed.

      +

      (9) done (2026-08-21): obikindexer::extensions::PrivateBuilder — item 1 above, implemented

      +

      Renamed from IndexBuilder to PrivateBuilder immediately after (same +session), freeing the name IndexBuilder for (10)'s public trait — read +IndexBuilder below as PrivateBuilder throughout this section.

      +

      Scoped down from (8)'s six-method list to the concrete set that's +genuinely movable without further ripple — checked, not assumed, before +writing anything:

      +
      pub(crate) trait PrivateBuilder {
+    fn mark_scattered(&mut self) -> OKIResult<()>;
+    fn mark_counted(&self) -> OKIResult<()>;
+    fn mark_indexed(&self) -> OKIResult<()>;
+    fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap<u32, u64>) -> OKIResult<()>;
+    fn build_index_layer(&self, i: usize, min_ab: u32, max_ab: Option<u32>, with_counts: bool, mode: &IndexMode, block_bits: u8) -> Result<usize, SKError>;
+    fn remove_build_artifacts(&self, i: usize);
+}
+impl PrivateBuilder for KmerIndex { ... }
+
      +

      All six moved bodily out of obikindex::index::{kmer_index, index_layer} +into obikindexer::extensions (new module, pub(crate)) — index_layer.rs +is now empty and deleted outright. +clear_output_for_create/create_skeleton/finalize_indexed/state +stayed inherent on KmerIndex, per (8)'s reasoning: merge/select/ +rebuild/reindex — living inside obikindex itself — call them too, +and obikindex can never depend on obikindexer to reach a trait defined +there. Moving those four is real future work (extract +merge/select/rebuild/reindex into algorithms first), not part of this +step.

      +

      One new, small, deliberate API widening in obikindex: build_index_layer +depends on three helpers that were pub(crate) to obikindex +(graph_pipeline::{write_graph_as_unitigs, materialize_layer}, +common::olm_to_sk) — widened to pub (re-exported from obikindex's +crate root) so obikindexer could reach them. This is exactly the +"enrich shared/lower-level APIs instead of ad hoc local code" call the +project's own rules ask for, made explicitly rather than routed around: +three functions, already generically written (no rewrite needed), now +serve a second caller instead of being duplicated.

      +

      Why the trait had to be defined in obikindexer, not obikindex: +Rust's orphan rule — implementing a trait for a foreign type requires +either the trait or the type to be local to the current crate. KmerIndex +is foreign to obikindexer, so the trait must be the local half; if it +were defined in obikindex instead, pub(crate) there would make it +invisible to obikindexer too (crate-private means private to that +crate, not "private except to one named dependent") — the opposite of +what was wanted.

      +

      A real design decision made while wiring callers up, not a mechanical +rename: PrivateBuilder being genuinely pub(crate) to obikindexer +means obikmer::cmd::index (a different crate) can no longer call +mark_scattered/mark_counted/mark_indexed/write_spectrum directly — +it never could have, once privacy was real rather than aspirational. Each +algorithm now marks its own completion as part of run()/close() +instead of leaving it to the caller: +- PartitionRouter::close() (not run()) calls mark_scattered() — + close(), not run(), is the actual shared completion point between + the file-driven run() path and the manual write/write_batch+ + close() path low-level callers (tests) use; putting it in run() + alone would have silently skipped marking for every caller that never + calls run(). run() already calls self.close() at its own end, so + this covers both paths through one line, not two. +- Counter::run calls write_spectrum then mark_counted before + returning. +- LayerBuilder::run calls mark_indexed before returning.

      +

      cmd/index/mod.rs lost all four direct calls (mark_scattered/ +write_spectrum/mark_counted/mark_indexed) — each stage's if +idx.state() < IndexState::X { ... } block is now purely "run the +algorithm," no separate bookkeeping call after it. Confirms, precisely +this time (checked by re-reading the whole file, not assumed): cmd/index +now rests on the four algorithms for every read/write of pipeline state +except KmerIndex::{exists, create, state, n_partitions}, which are +genuinely index-identity concerns, not construction bookkeeping — the +original question this whole design pass started from.

      +

      Same fix applied to obikphylo's test harness (its four explicit +mark_*/write_spectrum calls removed, relying on the algorithms now +doing it themselves) — obikindexer::algorithms::partitionner's own +pipeline_counts test needed no change (never called mark_* directly).

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0), plus the CLI smoke test — 870 +kmers, same as (6)/(7).

      +

      Still not done at the time of writing: item 2 from (8) (obikalgorithm:: +Algorithm), (5), the future cache crate, the distance.rs → +obikphylo relocation (noted in (8), explicitly deferred), and +extracting merge/select/rebuild/reindex into algorithms.

      +

      (10) done (2026-08-21): obikindex::IndexBuilder — the public counterpart, same session

      +

      Immediate correction to (9): the private trait built there was renamed +PrivateBuilder (freeing the name), and the four methods (9) had left +inherent on KmerIndex — clear_output_for_create/create_skeleton/ +finalize_indexed/state — got their own trait after all: IndexBuilder, +public, defined in obikindex itself (not obikindexer):

      +
      pub trait IndexBuilder: Sized {
+    fn clear_output_for_create<P: AsRef<Path>>(output: P, force: bool) -> OKIResult<()>;
+    fn create_skeleton<P: AsRef<Path>>(output: P, meta: &IndexMeta) -> OKIResult<Self>;
+    fn finalize_indexed<P: AsRef<Path>>(output: P, rep: &mut Reporter) -> OKIResult<Self>;
+    fn state(&self) -> IndexState;
+}
+impl IndexBuilder for KmerIndex { ... }
+
      +

      User's framing: these four are "maintenance", not "scientific computation +on an index" — a different kind of non-generic-ness than (9)'s six +(mark_*/write_spectrum/build_index_layer/remove_build_artifacts, +exclusive to the 4-stage pipeline). Maintenance is used more broadly +(merge/select/rebuild/reindex), so it gets a real, public trait — +not folded back into KmerIndex's inherent surface, and not private +either.

      +

      Where it lives, and why that's not arbitrary: (9) needed the orphan +rule to force its trait into obikindexer, to achieve genuine +crate-private visibility. Here the requirement is the opposite: +merge.rs/select.rs/rebuild.rs/reindex.rs — the trait's own +heaviest users — live inside obikindex. A trait they need to reach +must be local to obikindex (or a crate obikindex itself depends on, +which doesn't exist for this). So IndexBuilder lives in a new +obikindex/src/index/builder.rs, pub trait (no orphan-rule tension at +all here — both trait and type are local to the same crate), re-exported +from obikindex's crate root alongside PrivateBuilder's sibling +obikindexer::extensions::PrivateBuilder staying where it is. Two +traits, two crates, two different reasons, not a contradiction.

      +

      Blast radius, all inside obikindex plus one external crate: every +internal caller of these four methods needs the trait imported now that +they're no longer inherent — merge.rs, select.rs, rebuild.rs, +reindex.rs (use crate::index::builder::IndexBuilder;) and, externally, +obikmer::cmd::index::mod (use obikindex::IndexBuilder;, for the three +idx.state() < IndexState::X resumability checks). Call syntax at every +site is unchanged (KmerIndex::create_skeleton(...), +self.state()) — only trait-in-scope requirements are new, which is +exactly the point: same ergonomics, less surface baked into KmerIndex +itself.

      +

      Verification went one step further than (9): beyond +cargo check --workspace --all-targets + cargo test --workspace + +scripts/smoke_test_index.sh (all green, 870 kmers again), ran +obikmer merge end to end on two freshly built indexes (exercises +clear_output_for_create/finalize_indexed directly, the two methods +scripts/smoke_test_index.sh itself never touches) — exit 0, pack +stage completed. Test suite alone would not have caught a regression +here: no existing test builds two real indexes and merges them through +the CLI.

      +

      KmerIndex itself now carries only: identity/config accessors +(root_path/meta/kmer_size/n_bits/evidence_mode/genomes/...), +path resolution (partition_dir/index_dir/layer_dir/ +partition_meta/n_layers), and a few index-maintenance operations not +yet sorted into either trait (layer_unitigs_path, pack_matrices, +upgrade_layer_meta — see (8)'s "tested and discarded" list; still +correctly inherent, not construction-only by the semantic criterion) — +create/open/exists (identity, can't be trait methods needing Self +before one exists) round that out.

      +

      (11) done (2026-08-21): KmerIndex/IndexMeta made fully stateless, IndexState moved off sentinel files

      +

      Triggered mid-discussion of obikalgorithm::Algorithm (still not started — +see "Still not done" below): user asked why PartitionRouter::new still +took &mut KmerIndex at all, and questioned whether mark_scattered +belonged in the algorithm or in cmd/index. Investigation found &mut +had become newly necessary since (9) — mark_scattered was mutating +self.meta.genomes in memory so Counter's later write_spectrum call +(same idx instance) would see the derived label. User's resolution: the +"disk is truth, stateless" principle already agreed for KmerPartition/ +Layer in (5) (still unimplemented for those two) should extend to +KmerIndex itself — move IndexState (Empty/Scattered/Counted/ +Indexed) off the three sentinel files (scatter.done/count.done/ +index.done, detected by existence) and into a field of index.meta's +own JSON, so the mark_* calls become plain disk writes an algorithm can +legitimately make on &self — no in-memory mutation left to protect.

      +

      Shape of IndexMeta, per the user's explicit spec: one JSON file per +index (index.meta), one IndexMeta instance per index, held and +returned as Arc<IndexMeta> (not &IndexMeta) by KmerIndex::meta(). +config (kmer_size/minimizer_size/n_bits/with_counts/evidence/ +block_bits) is fixed at construction, cached as a pub field (getter +kept alongside, for symmetry) — "les champs constants restent des champs +de la structure", read once, never re-read from disk. genomes and +state are the opposite: no in-memory cache at all, every accessor +(genomes(), state()) re-reads index.meta from disk, every mutator +(push_genome/rename_genome/set_genomes/set_state/mark_scattered/ +mark_counted/mark_indexed) does a full read-modify-write of the same +file. An internal std::sync::RwLock<()> is held across each +read-modify-write sequence (not just the write) so two callers sharing +the same Arc<IndexMeta> can't lose an update to each other — this is +not a cross-process lock (that's obisys::DirLock, already held by +cmd/index for the whole build); it only serialises access through one +shared in-process instance.

      +

      Construction, chain-of-responsibility style, matching (5)'s pattern: +IndexMeta::create(&KmerIndex, config, genomes) / IndexMeta::open(&KmerIndex) +ask the index for its own root path rather than taking one directly. Since +KmerIndex::create doesn't have a complete KmerIndex yet to hand in +(it's what's being built), added lower-level pub(crate) path-based +primitives create_at(&Path, ...) / open_at(&Path) that KmerIndex::create/ +open and builder.rs's create_skeleton call directly, bypassing the +convenience wrappers for that one bootstrap case.

      +

      The one deliberate exception: select_in_place and reindex +genuinely rewrite config after an index already exists (output +type/evidence mode changes in place) — contradicting "config never +changes" for the general case. Resolved with a separate, explicitly +rare-labelled IndexMeta::rewrite_config(config, genomes) (preserves +state, overwrites everything else); callers refresh their own cached +Arc<IndexMeta> afterward (self.meta = Arc::new(IndexMeta::open(self)?)) +since IndexMeta has no way to reach back into whichever KmerIndex +holds it.

      +

      Consequence confirmed, not just hoped for: with mark_scattered no +longer touching anything in memory, PartitionRouter genuinely never +needs &mut KmerIndex — PartitionRouter<'a> { index: &'a KmerIndex }, +new(&'a KmerIndex). This is effectively the PartitionRouter half of +(5)'s "order of remaining work" item done as a side effect; KmerPartition/ +Layer themselves are still unimplemented for (5).

      +

      Blast radius — much larger than (9)/(10), touched nearly every crate: +every .meta().genomes/.meta.genomes field access became a fallible +.genomes()? method call (genomes reads io::Result<Vec<GenomeInfo>> +now, not a field), and .meta_mut() was removed outright (no more direct +field mutation from outside IndexMeta). Fixed across: +- obikindex internals: meta.rs/state.rs/kmer_index.rs/builder.rs + (full rewrites), reindex.rs/select.rs (switched to rewrite_config), + merge.rs (heaviest single file — genome counts precomputed once per + source into a Vec<Vec<GenomeInfo>> up front rather than re-reading + index.meta from disk repeatedly through the function, sentinel write + replaced with dst2.meta.mark_indexed()), stats.rs, distance.rs, + dump.rs, predicate.rs (its IndexMeta-inherent matching_genome_indices/ + build_group_filter now read genomes fresh internally), mod.rs/lib.rs + (sentinel constant re-exports removed — IndexState no longer has + SENTINEL_*/detect() at all). +- obikindexer::extensions::PrivateBuilder: mark_scattered signature + dropped &mut self → &self; the four mark_*/write_spectrum bodies + became one-line delegations to self.meta().mark_*(). +- obikphylo::siblings: alignment.rs/cardinality.rs/distance.rs/ + entropy.rs/sankoff_bundle.rs/stats.rs/tests.rs — all had + self.meta().genomes.len()-shaped reads, mechanically fixed to + .genomes().map_err(OKIError::Io)?.len() (tests: .unwrap()). +- obikmer::cmd::*: annotate (rewrote its rename path to load genomes + once, mutate the in-memory Vec, then idx.meta().set_genomes(...) + instead of meta_mut()), filter/pack/dump/unitig/merge/select/ + phylo (fetch-once-then-use pattern for genome counts/labels), + utils/maintenance.rs (run_rename now calls the pre-existing + IndexMeta::rename_genome, dropping its own hand-rolled field mutation + entirely), index/mod.rs (three idx.state() < IndexState::X + resumability checks needed a fallible read — factored into a small + current_state(&KmerIndex) -> IndexState helper rather than repeating + the same unwrap_or_else three times), query/* (emit_batch's + signature changed from &IndexMeta to &[GenomeInfo], and genomes is + now fetched once in run() and threaded down through process_chunk + as Arc<Vec<GenomeInfo>> rather than re-reading index.meta from disk + on every chunk — a deliberate deviation from the "always re-read" + default, justified because this is a genuine per-chunk hot path, unlike + every other call site touched in this pass). +- One &IndexMeta-vs-Arc<IndexMeta> argument-type mismatch pattern + recurred at several CLI call sites (build_filters/build_specs/ + emit_batch's original signature) — resolved via Arc's deref + coercion (&idx.meta() coerces to &IndexMeta) rather than changing + every downstream signature to accept Arc<IndexMeta>.

      +

      Verification: cargo check --workspace --all-targets and +cargo test --workspace both green (0 failures) after the full +propagation, scripts/smoke_test_index.sh green (870 kmers, same as every +prior round), plus a manual CLI run of index (×2) → merge → select +→ reindex → utils --new-label (rename) → utils --stats, all exit 0, +confirming the four most-affected commands (the ones (10)'s verification +already flagged as under-covered by the automated test suite) still work +end to end against the new Arc<IndexMeta>/on-disk-IndexState shape.

      +

      Still not done: (5)'s KmerPartition/Layer self-naming redesign itself +(only the PartitionRouter-&mut-removal piece landed, as a side +effect); the distance.rs → obikphylo relocation ((9), explicitly +deferred); the future cache-manager crate. (8)'s obikalgorithm:: +Algorithm trait, resumed and closed in (12) below.

      +

      (12) done (2026-08-21): obikalgorithm::Algorithm — the shared trait, resumed and closed in one session

      +

      Resumed (8)'s point 2 through a point-by-point discussion of what's +actually common across the four pipeline algorithms, now that (11) made +KmerIndex itself immutable everywhere. Four sub-points, each closed +before moving to the next:

      +

      1. Receiver (&self vs &mut self) — investigated whether (11)'s +removal of &mut KmerIndex also removed the need for PartitionRouter:: +run to take &mut self. It didn't: PartitionRouter holds real +per-run state of its own (writers: Vec<Option<SKFileWriter>>, open file +handles, purely in-process RAM — confirmed by checking where writers is +stored, nothing to do with KmerIndex/disk truth), unrelated to the +index. First proposal (wrap writers in RefCell so all four could +share a uniform &self) was retracted on pushback: manufacturing +interior mutability with runtime borrow checks to satisfy a cosmetic +uniformity that Rust doesn't even require is over-engineering — a trait +method's receiver must match exactly across implementors, but nothing +stops that shared receiver from being &mut self with three of the four +implementations simply not using the mutability. Settled: trait declares +&mut self; Dereplicator/Counter/LayerBuilder (previously &self) +now also take &mut self, unused.

      +

      2. path_source as a PartitionRouter setter, not a run() param — +added a files: Option<Box<dyn Iterator<Item = PathBuf> + Send>> field + +.files(impl Iterator<Item = PathBuf> + Send + 'static) -> Self setter +(boxed rather than a generic type parameter on PartitionRouter<'a>: +negligible cost — one PathBuf per input file, not per k-mer — for a +much more usable type when passing the builder around). run now does +self.files.take().ok_or_else(...), erroring if .files(...) was never +called, instead of taking path_source as a parameter.

      +

      3. Unifying the three progress-callback bound shapes — reopened, then +resolved differently than (8) originally framed it. First proposal +(force everything to FnMut(Progress) + Send) was rejected on the same +principle as point 1: Dereplicator/Counter's Fn(Progress) + Sync +isn't arbitrary — their callback is invoked concurrently from multiple +rayon worker threads, and FnMut requires exclusive access, so forcing +it would mean wrapping the callback in a Mutex for zero benefit at the +one real call site (pb.inc(1), already thread-safe). The actual +resolution: move on_progress off run()'s signature entirely, onto a +per-algorithm .on_progress(...) setter — same treatment as point 2's +path_source — so each algorithm keeps its own bound (PartitionRouter: +FnMut(Progress) + 'a, sequential, no Send needed; LayerBuilder: +FnMut(Progress) + Send + 'a, crosses into PartitionRunner's +thread::scope-spawned controller thread once; Dereplicator/Counter: +Fn(Progress) + Sync + 'a, invoked concurrently from rayon workers). +This also dissolves the original problem run() had: once the +callback isn't part of run's signature at all, there's nothing left to +unify there, and point 4 (below) becomes trivial.

      +

      4. Output as an associated type, Error fixed — trivial once (3) +moved the callback out: Error was already uniform (all four return +obiskio::SKResult<T> = Result<T, SKError>), only Output varied +(()/()/KmerSpectrum/usize). First cut reused obiskio::SKResult +directly as the trait's return type — caught and corrected the same +session: SKError enumerates I/O-specific cases (BadMagic/ +Truncated/Compression/...), meaningless at the level of a generic +"algorithm" abstraction, and borrowing it made obikalgorithm — meant to +be minimal and neutral — depend on a low-level I/O crate purely to reuse +its error type. Textbook instance of the "petits pois" failure mode +(patch around a convenient existing type instead of asking what this +crate should actually own). Fixed to a genuinely generic, boxed error +type owned by obikalgorithm itself:

      +
      // obikalgorithm — no dependency on obiskio or any other crate
+pub type Error = Box<dyn std::error::Error + Send + Sync>;
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub trait Algorithm {
+    type Output;
+    fn run(&mut self) -> Result<Self::Output>;
+}
+
      +

      Any concrete error (SKError, std::io::Error, ...) converts +automatically via ?, through std's own blanket impl<E: Error + Send ++ Sync> From<E> for Box<dyn Error + Send + Sync> — no custom From impl +needed, no dependency on the crate that defines the concrete error type. +The four algorithms' run bodies needed no change beyond the signature's +return type (every existing ? on an SKError-returning subcall keeps +compiling, converting through the same blanket impl at the boundary).

      +

      PartitionRouter/Dereplicator/Counter/LayerBuilder each impl +Algorithm for X<'_> { type Output = ...; fn run(&mut self) -> obikalgorithm::Result<...> { ... } } +— the old inherent run methods were removed outright (not kept as +duplicates), so callers now use obikalgorithm::Algorithm; to call +.run(). Every field-lifetime-bound boxed callback (Box<dyn +FnMut(Progress) + 'a> etc.) is tied to the algorithm's own 'a (the +&'a KmerIndex lifetime already on the struct), not 'static — avoids +forcing callers' progress closures to move-capture (and therefore clone +or Arc-wrap) local state like TracedBar/EMA-rate accumulators that +they'd otherwise want to keep using by reference after run() returns.

      +

      Why a new crate, not a submodule of obikindexer: obikmer::cmd:: +index::mod and obikphylo's own test helpers both need to call .run() +on these algorithms, so the trait has to be reachable from outside +obikindexer — putting it in obikindexer itself would work file-wise +but conflates "the trait every algorithm implements" with "one crate's +particular four implementations of it", the same reasoning that already +separated obikindex (data model) from obikindexer (algorithms +operating on it). obikalgorithm has no dependencies at all (see +above); obikindexer, obikmer, and obikphylo (dev-dependency, for +its test helper) all depend on it.

      +

      Blast radius: obikindexer's four algorithm modules (struct field + +setter + trait impl each); obikmer::cmd::index::mod (three call sites: +.on_progress(cb) before .run(), unqualified now that the trait is in +scope); obikindexer::algorithms::partitionner::tests and obikphylo:: +siblings::tests (both had direct .run(None::<fn(Progress)>)-shaped +calls needing the same treatment). New obikalgorithm crate registered +in the workspace Cargo.toml, depended on by obikindexer/obikmer +(regular) and obikphylo (dev).

      +

      Verification: cargo check --workspace --all-targets and cargo test +--workspace both green (0 failures), scripts/smoke_test_index.sh green +(870 kmers, same as every prior round) — this round didn't repeat the +manual merge/select/reindex CLI exercise from (11), since nothing +in this pass touched those commands' code paths (only the four pipeline +algorithms and cmd::index, already covered by the smoke test). Reverified +after the obiskio-dependency fix above (same three checks, still green, +obikalgorithm/Cargo.toml now has zero [dependencies]).

      +

      Still not done: (5)'s KmerPartition/Layer self-naming redesign; the +distance.rs → obikphylo relocation ((9), explicitly deferred); the +future cache-manager crate (mentioned in (8) as a later, mirrored +extension-trait exercise, not started).

      +

      The problem

      +

      Reading a layer's data (MPHF + matrix) is not free: MphfLayer::open mmaps +mphf.bin plus (evidence.bin/fingerprint.bin + unitigs.bin), and the +matrix side mmaps matrix.pbmx/matrix.pcmx (or one file per genome column +if not yet packed). Any code path that reopens a layer per lookup instead of +once per run pays this cost repeatedly.

      +

      obikphylo::siblings::cache::PartitionCache was built to avoid exactly this +for build_sibling_annex/sibling_annex_stats: those commands probe many +partitions, once per source layer, over the whole run. Profiling a real run +showed wall-clock time dominated by repeated open()/mmap syscalls, not +computation — parallelising the naive per-lookup opens spread the cost +across cores without reducing it. PartitionCache::build opens every +partition's every layer once, up front, in parallel, and keeps the handles +alive for the run.

      +

      Three independent implementations of the same bundle (historical — (1) fixed this)

      +

      As of 2026-08-20 this table describes the pre-(1) state. Mat no +longer exists (deleted when obilayeredmap::Layer replaced it — see "(1) +done" above); Layer<D> in the table below is what's now called +TypedLayer<D>. QueryLayer is unaffected and still stands as described — +still uncached, still not using Layer at all — which is exactly what (2) +needs to fix. Kept for the original motivation, not as current fact:

      +

      Searching the codebase for "who bundles MPHF + matrix, with per-layer format +auto-detection" turned up three unrelated implementations:

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      lives inscopecached?
      Layer<D> (now TypedLayer<D>)obilayeredmapone layer, D fixed at compile timeheld alive by whoever owns the Layer, no policy of its own
      Mat (now deleted; superseded by obilayeredmap::Layer)obikphylo::siblings::cacheone layer, format resolved per instance from an enum of 3 Layer<D> variantsyes, via PartitionCache
      QueryLayer (unchanged, still current)obikindex::query_layer (moved crates since this was written — see "Major restructuring")one layer, (MphfLayer, PersistentBitMatrix\|PersistentCompactIntMatrix) pair, bypasses TypedLayer<D>/Layer entirelyno — opened fresh inside query_partition_with on every call
      +

      query_partition_with is obikmer query's normal query path — the one +most exposed to repeated cross-partition lookups — and it is still the one +with no cache at all. obikphylo built a cache first only because +sibling-annex construction hits the cost hardest, not because the need is +sibling-specific.

      +

      The gap in obilayeredmap's existing cache (historical — (1) fixed this)

      +

      obilayeredmap::LayeredMap<D> already caches correctly at the granularity +of one partition: open(root) opens every layer once, keeps +Vec<TypedLayer<D>> alive for the LayeredMap's lifetime. But it is +monomorphic — every layer in the Vec must share the same concrete D. +In practice this was false: layers in the same partition are packed +independently over time (pack --sparse converts one layer's presence +matrix at a time). This motivated (1) — obilayeredmap::Layer, done — but +note the specific PersistentSparseBitMatrix-mixing scenario described +here turned out to be moot: PersistentBitMatrix itself absorbed sparse +storage as a 4th internal variant before (1) was built (see "One bug found +… one earlier claim retracted" below), so the only heterogeneity Layer +actually needs to represent is Count vs. Presence, not dense-vs-sparse +presence. LayeredMap<D> itself is unaffected by any of this — it's still +monomorphic, still not used by Layer/KmerPartition (which bypass it +entirely, opening each TypedLayer<D> directly, the same way Mat did).

      +

      Resource cost: mmap does not hold a file descriptor

      +

      Before deciding how many layers/partitions a cache may hold open +simultaneously, the binding constraint needs to be identified correctly.

      +

      Confirmed against upstream documentation, not inferred from behaviour:

      +
      +

      "After the mmap() call has returned, the file descriptor, fd, can be +closed immediately without invalidating the mapping." +— mmap(2), man7.org

      +

      "The close(2) function does not unmap pages" +— mmap(2), Apple Developer

      +

      "A file backed Mmap ... will remain valid even after the File is dropped. +... the Mmap handle is completely independent of the File used to create +it." +— memmap2::Mmap, docs.rs

      +
      +

      Every read-only mmap in this codebase already follows this: Mmap::map(&File::open(path)?)? +— the File is a temporary, dropped (fd closed) immediately after the +mapping is established; every persistent struct (PersistentBitVec, +PersistentCompactIntVec, PackedBitMatrix, Evidence, FingerprintVec, +...) stores only the Mmap, never the File. So a cache built on these +types does not consume the process's open-file-descriptor budget +(ulimit -n, notoriously low by default on macOS) proportionally to how +many mmapped files it holds.

      +

      It does consume a different resource — the process's virtual-memory mapping +table (one entry per active mmap() region). Linux exposes this as +vm.max_map_count (default 65530). No documented macOS equivalent (fixed +numeric ceiling) was found; the constraint there appears to be virtual +address space rather than an explicit mapping counter, but this is not +sourced and should not be assumed. This is the resource actually worth +measuring before deciding on cache size, not fd count — and it is why +packing (matrix.pbmx/matrix.pcmx, one mmap for all columns) matters +independently of any caching decision: an unpacked Columnar matrix opens +one mmap per genome column, multiplying the mapping count a cache would +have to hold by n_genomes.

      +

      Layering: who owns what (superseded — see Definitions above)

      +

      This section used to argue nobody owned "the collection of partitions." +That's resolved: KmerIndex (obikindex) owns it now, directly (see +"Major restructuring" below). What's still genuinely unowned is one +partition's open layers — KmerPartition, in the not-yet-created +obikpartition — see "Definitions" at the top of this file for the +current, authoritative answer. Left here only so old links/references to +this heading don't 404; don't read this section for current facts.

      +

      Direction agreed, not yet implemented

      +

      Only (2) remains — (1) shipped as obilayeredmap::Layer (see "(1) done" +above). Concretely, in order:

      +
        +
      1. Create the obikpartition crate (obikindex → obikpartition → + obilayeredmap, no other edges — see "Definitions" above for the exact + constraint and why).
        2. +
      2. KmerPartition { layers: Vec<obilayeredmap::Layer> } — open, + n_layers, layer(i), find, plus whatever batch-lookup surface + obikphylo::siblings::cache::PartitionCache currently needs + (find_presence_batch/find_presence_batch_fast; fast_mode is + sibling-specific bookkeeping and should probably stay in obikphylo, + wrapping a KmerPartition/Vec<KmerPartition> rather than living + inside it — same "generic vs. domain-specific" split iter_minorants_batch + already went through for Layer in (1)).
        3. +
      3. Migrate obikphylo::siblings::cache::PartitionCache to hold + Vec<KmerPartition> instead of Vec<Vec<Layer>>.
        4. +
      4. Migrate obikindex::query_layer::QueryLayer/query_partition_with to + use KmerPartition too, closing the "no cache at all" gap on + obikmer's normal query path (see "Three independent implementations," + historical, above).
        5. +
      +

      Open before implementing: exact API shape of KmerPartition (propose, +confirm before coding — non-trivial), and whether the multi-partition +Vec<KmerPartition> needs an eviction policy or can simply hold every +partition open for the process lifetime (revisit once the +VM-mapping-count question above has a real number behind it for this +codebase's scale — still not measured).

      +

      Preparatory work done (2026-08-20)

      +

      Groundwork for (1)/(2), landed ahead of the design itself. Note: at +the time this was written, the collection type these bullets describe was +named KmerPartition (singular) in this doc; it was renamed +KmerPartitions (plural) shortly after, then deleted entirely and merged +into KmerIndex (see "Major restructuring" above). The bullets below are +edited to say KmerPartitions throughout, to not collide with the +unrelated, brand-new singular KmerPartition defined at the top of this +file — the accessors described here live on KmerIndex today, not on +any type called KmerPartition.

      +
        +
      • KmerPartitions (obikpartitionner, at the time) gained + partition_dir/index_dir/layer_dir as the single source of truth + for a partition's on-disk layout, replacing per-module duplicated + const INDEX_SUBDIR: &str = "index" (7 copies) and ad hoc path joins — + including one found duplicated inside the struct itself + (ensure_writer rebuilt part_dir's own logic by hand).
        • +
      • Same struct gained partition_meta/n_layers/index_mode, wrapping + obilayeredmap::meta::PartitionMeta::load (via the existing + common::load_meta, which also recovers indexes built before + meta.json existed). Before this, obikphylo and obikindex imported + obilayeredmap::meta::PartitionMeta directly and called ::load() + themselves at 21 call sites, each redoing its own error-mapping — + every one of those crates knew the on-disk metadata format instead of + going through an interface. Fixed everywhere except one remaining spot + (below). Caught as a side effect: dump_layer.rs/query_layer.rs had + been calling PartitionMeta::load directly, bypassing load_meta + entirely — they never got the missing-meta.json recovery the other + callers did.
        • +
      • Layer introspection API discussed but not yet implemented — three + axes, deliberately kept separate after an initial draft conflated them:
        • +
      • LayerContent { Count, Presence } — what the layer stores; a const + on LayerData (compile-time, zero-cost), not a runtime field.
        • +
      • StorageKind { Implicit, Columnar, Packed, Sparse } — how it's + stored; only meaningful for D that actually carry data (Layer<()> + has neither this nor LayerContent — it's a write-time-only state, + never a queryable content: once a layer is closed, "no matrix file" + reads back as Presence/Implicit via PersistentBitMatrix::open's + own fallback, not as some third "empty" content).
        • +
      • EvidenceKind { Exact, Approx, Hybrid } — from MphfLayer's own + already-in-memory LayerEvidence discriminant.
        • +
      • Not all (LayerContent, StorageKind) pairs are legal: Count never + has Implicit or Sparse.
        • +
      +

      Implemented (2026-08-20). LayerContent/StorageKind/EvidenceKind +now exist, each with two forms: +- A runtime accessor on an already-open value (Layer<D>::content()/ + storage_kind()/evidence_kind(), PersistentBitMatrix::storage_kind(), + PersistentCompactIntMatrix::storage_kind(), MphfLayer::evidence_kind()) + — reads a discriminant already in memory, zero disk access. +- A lightweight detect()/detect_storage() disk probe that mirrors the + corresponding open()'s own priority order by hand (file-existence + checks only, no mmap) — usable before committing to a D, unlike the + runtime accessors. Exposed per-layer on LayeredMap<D> as + detect_layer_content/detect_layer_storage/detect_layer_evidence + (work regardless of D, since they only use self.root + the layer + index).

      +

      StorageKind lives in obicompactvec (owner of PersistentBitMatrix/ +PersistentCompactIntMatrix); LayerContent/EvidenceKind live in +obilayeredmap. HasLayerContent/HasStorageKind gate Layer<()> out of +content()/storage_kind() (no matrix, nothing to report), matching the +"empty is transitional" conclusion above. 42 new tests across +obilayeredmap's tests/layer.rs and tests/map.rs; full workspace +suite green (0 failed) after.

      +

      Not done: these detect() probes don't yet replace Mat::open's or +QueryLayer::open's own hand-rolled equivalents (still duplicated content/ +storage decisions, now a third copy of the same logic to keep in sync) +— that consolidation is (1)/(2)'s job, not this prep step's.

      +

      One bug found while reading around this (signalled, not fixed); one earlier claim retracted

      +
        +
      • obicompactvec::bitmatrix::sparse.rs's module doc says + "Not used by any production code path yet" — false since obikmer pack + --sparse (cmd/pack/mod.rs) is wired to pack_sparse_bit_matrix and + Mat::open already reads the result back in the sibling-annex path. + Stale comment, not corrected.
        • +
      • Retracted (2026-08-20): an earlier pass through this doc claimed + obikpartition::query_layer::QueryLayer::open had no sparse-format + detection and would silently corrupt reads on a pack --sparsed layer. + False — PersistentBitMatrix (obicompactvec::bitmatrix::persistent) + is a 4-way enum (Columnar/Packed/Sparse/Implicit), not 3-way as + first read; its open() already detects Sparse via + presence/sparse_meta.json, and every method on the type (row, + fill_row, nonzero_iter, …) already dispatches all 4 arms. + QueryLayer::open's PersistentBitMatrix::open(layer_dir) call was + never the bug. Root cause of the false claim: a grep -n + "Implicit\|Columnar\|Packed" used to read the enum definition silently + skipped the Sparse(...) line because it matched none of those three + words — a self-inflicted blind spot from a filtered read, not a fact + about the code. Lesson: for a pub enum whose variant list matters, + read the definition unfiltered, don't grep for the variant names you + expect to find.
        • +
      • One real consequence of that same correction, fixed (2026-08-20): + obikphylo::siblings::cache::Mat::SparsePresence(Layer< + PersistentSparseBitMatrix>) was redundant — Mat::Presence(Layer< + PersistentBitMatrix>) alone already handles sparse layers + transparently, since PersistentBitMatrix absorbs Sparse internally. + Removed the variant, the presence/is_multi.prsb probe in Mat::open + (now just opens Layer::<PersistentBitMatrix> unconditionally for the + non-count case — sparse-vs-dense is PersistentBitMatrix::open's own + concern), and every now-single-armed match in find_slot/index_batch/ + iter_minorants_batch/n_cols/fill_sub_matrix_carries. Full + workspace test suite green after, including the 27 obikphylo::siblings + tests that exercise pack_matrices(true)/sparse through Mat.
        • +
      +

      Remaining instance of the PartitionMeta-encapsulation problem

      +

      obikphylo::siblings::family_scan::scan_layer_families still re-derives +index_dir from layer_dir.parent() and calls PartitionMeta::load +itself, purely to get .mode for obilayeredmap::Layer::open (was +Mat::open, same gap, survived the Mat → Layer swap in (1) unchanged). +Fixing it the way the 21 other call sites were fixed needs more than a 1:1 +swap: scan_layer_families only receives a bare layer_dir: &Path, not a +(partition, part, layer) triple, and its single upstream source of layer +paths, sibling_layer_dirs, returns a flat Vec<PathBuf> with the +partition/layer indices already discarded. Fixing it properly means either +having sibling_layer_dirs return (PathBuf, IndexMode) (or (part, +layer)) pairs, or threading &KmerIndex + indices through instead of +paths (not &KmerPartition — that type doesn't exist yet, and once it +does it still won't know IndexMode, which lives on KmerIndex/ +PartitionMeta) — and touching every one of scan_layer_families's 8 +callers (distance.rs, alignment.rs, cardinality.rs, entropy.rs ×2, +sankoff_bundle.rs ×2, stats.rs). Left alone this round; worth doing as +part of the same pass that builds KmerPartition, since those callers are +exactly the sibling-annex consumers it's meant to serve.

      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/DevDoc/implementation/persistent_bit_vec.refs/index.html b/DevDoc/implementation/persistent_bit_vec.refs/index.html index ed24d31a..452c7db1 100644 --- a/DevDoc/implementation/persistent_bit_vec.refs/index.html +++ b/DevDoc/implementation/persistent_bit_vec.refs/index.html @@ -727,14 +727,14 @@
    9. - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
    10. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      11. + + + + + + + + + + +
    11. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      12. + + + + diff --git a/DevDoc/implementation/persistent_bit_vec/index.html b/DevDoc/implementation/persistent_bit_vec/index.html index ff48156b..3e40d825 100644 --- a/DevDoc/implementation/persistent_bit_vec/index.html +++ b/DevDoc/implementation/persistent_bit_vec/index.html @@ -733,14 +733,14 @@
    12. - + - obilayeredmap crate + obikindex layer tier @@ -1225,6 +1225,62 @@ + + + + + + +
    13. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      14. + + + + + + + + + + +
    14. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      15. + + + + diff --git a/DevDoc/implementation/persistent_compact_int_vec.refs/index.html b/DevDoc/implementation/persistent_compact_int_vec.refs/index.html index 031ff570..9e306acb 100644 --- a/DevDoc/implementation/persistent_compact_int_vec.refs/index.html +++ b/DevDoc/implementation/persistent_compact_int_vec.refs/index.html @@ -727,14 +727,14 @@
    15. - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
    16. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      17. + + + + + + + + + + +
    17. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      18. + + + + diff --git a/DevDoc/implementation/persistent_compact_int_vec/index.html b/DevDoc/implementation/persistent_compact_int_vec/index.html index 274ec9bf..05ea4d9f 100644 --- a/DevDoc/implementation/persistent_compact_int_vec/index.html +++ b/DevDoc/implementation/persistent_compact_int_vec/index.html @@ -9,7 +9,7 @@ - + @@ -733,14 +733,14 @@
    18. - + - obilayeredmap crate + obikindex layer tier @@ -1208,6 +1208,62 @@ + + + + + + +
    19. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      20. + + + + + + + + + + +
    20. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      21. + + + + diff --git a/DevDoc/implementation/pipeline.refs/index.html b/DevDoc/implementation/pipeline.refs/index.html index fec369f9..3ab74967 100644 --- a/DevDoc/implementation/pipeline.refs/index.html +++ b/DevDoc/implementation/pipeline.refs/index.html @@ -727,14 +727,14 @@
    21. - + - obilayeredmap crate + obikindex layer tier @@ -944,6 +944,62 @@ + + + + + + +
    22. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      23. + + + + + + + + + + +
    23. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      24. + + + + @@ -1214,12 +1270,12 @@

      Coverage: implementation/pipeline.md

      Code couvert

      Notes

      diff --git a/DevDoc/implementation/pipeline/index.html b/DevDoc/implementation/pipeline/index.html index 211b01f0..a697eee7 100644 --- a/DevDoc/implementation/pipeline/index.html +++ b/DevDoc/implementation/pipeline/index.html @@ -857,14 +857,14 @@
    24. - + - obilayeredmap crate + obikindex layer tier @@ -1074,6 +1074,62 @@ + + + + + + +
    25. + + + + + + + + Benchmark: query-path testing + + + + + + + + +
      26. + + + + + + + + + + +
    26. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
      27. + + + + @@ -1546,7 +1602,7 @@ branching / dead-end → unitig start or end counts/ ← PersistentCompactIntMatrix (with_counts only)

      Cleanup: unless --keep-intermediate is set, remove_build_artifacts deletes dereplicated.skmer.zst, mphf1.bin, and counts1.bin after all partitions are indexed.

      -

      See obilayeredmap and MPHF selection for data structure details.

      +

      See obikindex::layer and MPHF selection for data structure details.

      Query path (exact evidence):

      query kmer q
   → canonical_minimizer(q) → hash → PART → part_XXXXX/
diff --git a/DevDoc/implementation/select/index.html b/DevDoc/implementation/select/index.html
index d1351204..c98f84e1 100644
--- a/DevDoc/implementation/select/index.html
+++ b/DevDoc/implementation/select/index.html
@@ -733,14 +733,14 @@
   
   
     
    27. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1119,17 +1119,6 @@
     
   
   
-
      28. 
-        
-          
    28. 
-  
-    
-      
-        In-place: keep only group A
-      
-    
-  
-  
 
      29. 
         
           
    29. 
@@ -1157,6 +1146,23 @@
     
   
   
+    
+  
 
      30. 
       
     
@@ -1196,6 +1202,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    30. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      31. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    31. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      32. 
+  
+
+              
+            
           
         
       
@@ -1582,17 +1644,6 @@
     
   
   
-
-        
-          
    32. 
-  
-    
-      
-        In-place: keep only group A
-      
-    
-  
-  
 
      33. 
         
           
    33. 
@@ -1620,6 +1671,23 @@
     
   
   
+    
+  
 
      34. 
       
     
@@ -1672,23 +1740,32 @@ are preserved unchanged; only the data matrices are rewritten.
      
 
      
 
      Synopsis
      
 
      obikmer select <input-index>
-        { --output <dir> | --in-place }
+        --output <dir>
         [--group    <name>:<pred>  ...]
         [--group-op <name>:<op>    ...]
         [--aggregate-by <key>          ]
         [--aggregate-op <op>           ]
         [--select   <col1,col2,...>    ]
         [--presence-threshold <N>      ]
+        [--force-copy                  ]
 
      
 
      
 
      Output destination
      
-
      Exactly one of --output or --in-place must be specified.
      
-
      --output <dir> — writes a new index to <dir>. The source index is
-unchanged. The MPHF and unitig files are copied; only the data matrices are
-rewritten with the new column layout.
      
-
      --in-place — rewrites the data matrices of the source index directly.
-Removed or replaced columns are lost. The operation writes to temporary files
-first, then renames atomically, so an interrupted run leaves the index intact.
      
+
      --output <dir> is required — select always writes a new index; there is no
+--in-place mode (2026-08-28: never implemented, removed from the design). The
+source index is unchanged.
      
+
      Each layer's kmer-identity files (mphf.bin/unitigs.bin/evidence.bin/
+unitigs.bin.idx/fingerprint.bin/layer_meta.json) are never rewritten by a
+column projection/aggregation, so they are hard-linked into the output rather
+than copied — no extra disk for them even on a large index. Falls back to a
+real copy automatically if linking fails (different filesystems); --force-copy
+forces a real copy always, for an output that must survive independently of the
+source on disk (a hard link shares the same inode — rewriting one path outside
+select itself would affect the other). Only the presence/counts
+subdirectory is ever a genuinely new, independent file.
      
+
      To replace an index with a selected version of itself, select to a temporary
+directory and swap it in (rm -rf INDEX && mv INDEX.tmp INDEX) — the case
+--in-place used to cover.
      
 
      
 
      Defining output columns
      
 
      Named groups — --group
      
@@ -1852,9 +1929,6 @@ is > N.
      
 
      obikmer select myindex --output out \
   --select "Betula_nana--TROM-V-149986,Betula_nana--AG-P04-25-01"
 
      
-
      In-place: keep only group A
      
-
      obikmer select myindex --in-place --group "A:group=A" --select "A"
-
      
 
      Compose with filter
      
 
      # Step 1: keep only B. nana-specific k-mers
 obikmer filter myindex --output filtered \
@@ -1865,32 +1939,46 @@ obikmer select 
      
 
      
 
      Implementation notes
      
-
      select does not rebuild the MPHF. The 256 partitions are processed in parallel
-(rayon), each writing its output independently; results require no synchronisation
-because every partition owns a distinct set of files.
      
-
      For each layer in each partition:
      
+
      select does not rebuild the MPHF. Every partition is processed independently
+(PartitionRunner), each writing its own output layers; no cross-partition
+synchronisation is needed.
      
+
      For each layer in each partition (obikselect::select_layer::select_partition):
      
 
        
-
      1. The slot count n is read by opening the source data matrix.
        2. 
-
      2. A new data matrix is built with M columns (M = number of output columns).
        3. 
-
      3. For each slot s in 0..n:
        4. 
-
      4. old_row = matrix.fill_row(s) — reads the original N-column row without allocating.
        5. 
-
      5. For each output column j:
          
-
        • new_row[j] = aggregate(op, old_row[group_indices]).
          • 
-
        • Pass-through columns are represented as single-element groups with the
-   default operator (any for presence, sum for count) — same code path.
          • 
-
        
-
        6. 
-
      6. The new row is written slot by slot into each column builder.
        7. 
-
      7. All plain files in the source layer directory (mphf.bin, unitigs.bin,
-   evidence files, layer_meta.json) are copied verbatim; only the presence/
-   or counts/ subdirectory is rewritten.
        8. 
+
      8. copy_layer_files hard-links the source layer's kmer-identity files
+   (mphf.bin/unitigs.bin/evidence.bin/unitigs.bin.idx/
+   fingerprint.bin/layer_meta.json) into the destination — never a real
+   copy unless linking fails or --force-copy is given.
        9. 
+
      9. A new data matrix is built with M columns (M = number of output columns),
+   under a fresh presence//counts/ subdirectory (never touching the
+   source's own).
        10. 
+
      10. Presence source (2026-08-28: batch_presence_counts): one shared pass
+   over the source bit matrix computes every output group's presence count at
+   once — row-major native for a Sparse source (for_each_genome_in_row,
+   which has no column representation to read a col_view from at all — the
+   reason this replaced the old per-group loop, not just an optimisation of
+   it), deduplicated column-major (one col_view per distinct referenced
+   column, not per group) for Columnar/Packed. Every AggOp for a bit
+   matrix is then a cheap derivation of that one count vector (sum = the
+   count itself, any/max = count ≥ 1, all/min = count == group
+   size, none = count == 0) — see
+   obikselect::select_layer::agg_result_from_count.
        11. 
+
      11. Count source: unchanged, one col_view-driven pass per output column
+   via MatrixGroupOps — sum/min/max are genuine per-value reductions
+   for a count matrix, not derivable from a single presence count the way
+   they are for a bit matrix.
        12. 
 
      12. index.meta is rewritten with the new genome list and updated with_counts.
        13. 
 
      
-
      --in-place write strategy: new data is written to a temporary sibling
-directory (presence_new/ or counts_new/); on success the old directory is
-removed and the temporary one is renamed into place. An interrupted run leaves
-at most one stale *_new/ directory; the original data is intact until the
-rename step.
      
+
      Known gap (not yet fixed, 2026-08-28)
      
+
      Step 4 above still panics (col_view() not available on Sparse
+PersistentCompactIntMatrix) if the source is a count index packed
+sparse — batch_presence_counts' row-major treatment was only ported to the
+bit-matrix (Presence) case, since that was the one actually blocking a real
+benchmark run. select/filter on a sparse-packed count index still hits
+this; the fix would follow the same shape (a PersistentSparseCompactIntMatrix
+row-major decode, analogous to for_each_genome_in_row), just not done. Since
+obisys::numa::runner::PartitionRunner's panic-propagation fix (see
+architecture/numa_partition_runner.md), this at least fails fast (process
+panic, exit 101) instead of hanging.
      
 
 
 
diff --git a/DevDoc/implementation/storage.refs/index.html b/DevDoc/implementation/storage.refs/index.html
index be053b27..03bfe3b3 100644
--- a/DevDoc/implementation/storage.refs/index.html
+++ b/DevDoc/implementation/storage.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    34. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    35. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      36. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    36. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      37. 
+  
+
+              
+            
           
         
       
@@ -1216,7 +1272,7 @@
 
        
 
      • obikindex/src/meta.rs — IndexMeta, IndexConfig (version, config, genomes)
        • 
 
      • obikindex/src/index.rs — layout sur disque : partitions/, index.meta
        • 
-
      • obilayeredmap/src/meta.rs — LayerMeta (evidence kind), PartitionMeta (n_layers)
        • 
+
      • obikindex/src/layer/meta.rs — LayerMeta (evidence kind), PartitionMeta (n_layers)
        • 
 
      • obiskio/src/unitig_index.rs — fichiers unitigs.bin + unitigs.bin.idx
        • 
 
      
 
      Notes
      
diff --git a/DevDoc/implementation/storage/index.html b/DevDoc/implementation/storage/index.html
index c463fba5..7fd77097 100644
--- a/DevDoc/implementation/storage/index.html
+++ b/DevDoc/implementation/storage/index.html
@@ -929,14 +929,14 @@
   
   
     
    37. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1146,6 +1146,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    38. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      39. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    39. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      40. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/implementation/superkmer.refs/index.html b/DevDoc/implementation/superkmer.refs/index.html
index 62a1ac84..946e4da1 100644
--- a/DevDoc/implementation/superkmer.refs/index.html
+++ b/DevDoc/implementation/superkmer.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    40. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    41. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      42. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    42. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      43. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/implementation/superkmer/index.html b/DevDoc/implementation/superkmer/index.html
index f7a43cec..d5267a43 100644
--- a/DevDoc/implementation/superkmer/index.html
+++ b/DevDoc/implementation/superkmer/index.html
@@ -835,14 +835,14 @@
   
   
     
    43. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1052,6 +1052,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    44. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      45. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    45. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      46. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/implementation/unitig_evidence.refs/index.html b/DevDoc/implementation/unitig_evidence.refs/index.html
index eb9e7a42..1dfbaa9c 100644
--- a/DevDoc/implementation/unitig_evidence.refs/index.html
+++ b/DevDoc/implementation/unitig_evidence.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    46. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    47. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      48. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    48. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      49. 
+  
+
+              
+            
           
         
       
@@ -1215,7 +1271,7 @@
 
      Code couvert
      
 
        
 
      • obiskio/src/unitig_index.rs — format unitigs.bin + unitigs.bin.idx, UnitigFileWriter, UnitigFileReader, build_unitig_idx(), DEFAULT_BLOCK_BITS=0, chemin chaud block_bits=0 dans chunk_start()
        • 
-
      • obilayeredmap/src/evidence.rs — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
        • 
+
      • obikindex/src/layer/evidence.rs — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
        • 
 
      • obidebruinj/src/debruijn.rs — extraction unitigs, chunking à MAX_KMERS_PER_CHUNK
        • 
 
      
 
      Notes
      
diff --git a/DevDoc/implementation/unitig_evidence/index.html b/DevDoc/implementation/unitig_evidence/index.html
index 17064faf..a66adbd3 100644
--- a/DevDoc/implementation/unitig_evidence/index.html
+++ b/DevDoc/implementation/unitig_evidence/index.html
@@ -957,14 +957,14 @@
   
   
     
    49. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1174,6 +1174,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    50. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      51. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    51. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      52. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/index.html b/DevDoc/index.html
index 6b85aa94..e6629e0a 100644
--- a/DevDoc/index.html
+++ b/DevDoc/index.html
@@ -831,14 +831,14 @@
   
   
     
    52. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1048,6 +1048,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    53. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      54. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    54. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      55. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/index.refs/index.html b/DevDoc/index.refs/index.html
index 9eae697e..5f0a7f61 100644
--- a/DevDoc/index.refs/index.html
+++ b/DevDoc/index.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    55. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    56. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      57. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    57. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      58. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/installation/index.html b/DevDoc/installation/index.html
index 768aaa36..42e8a762 100644
--- a/DevDoc/installation/index.html
+++ b/DevDoc/installation/index.html
@@ -917,14 +917,14 @@
   
   
     
    58. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1134,6 +1134,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    59. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      60. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    60. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      61. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/kmers.refs/index.html b/DevDoc/kmers.refs/index.html
index 7ccf378e..b12faff2 100644
--- a/DevDoc/kmers.refs/index.html
+++ b/DevDoc/kmers.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    61. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    62. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      63. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    63. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      64. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/kmers/index.html b/DevDoc/kmers/index.html
index f0f09bbe..a707331c 100644
--- a/DevDoc/kmers/index.html
+++ b/DevDoc/kmers/index.html
@@ -830,14 +830,14 @@
   
   
     
    64. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1047,6 +1047,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    65. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      66. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    66. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      67. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/sitemap.xml.gz b/DevDoc/sitemap.xml.gz
index d240d53a..03a78e93 100644
Binary files a/DevDoc/sitemap.xml.gz and b/DevDoc/sitemap.xml.gz differ
diff --git a/DevDoc/theory/encoding.refs/index.html b/DevDoc/theory/encoding.refs/index.html
index 26b92a90..fe650220 100644
--- a/DevDoc/theory/encoding.refs/index.html
+++ b/DevDoc/theory/encoding.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    67. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    68. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      69. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    69. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      70. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/theory/encoding/index.html b/DevDoc/theory/encoding/index.html
index fdb70802..e8a2dbb1 100644
--- a/DevDoc/theory/encoding/index.html
+++ b/DevDoc/theory/encoding/index.html
@@ -802,14 +802,14 @@
   
   
     
    70. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1019,6 +1019,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    71. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      72. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    72. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      73. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/theory/entropy.refs/index.html b/DevDoc/theory/entropy.refs/index.html
index c046f97e..49c1b70d 100644
--- a/DevDoc/theory/entropy.refs/index.html
+++ b/DevDoc/theory/entropy.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    73. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    74. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      75. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    75. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      76. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/theory/entropy/index.html b/DevDoc/theory/entropy/index.html
index 0f66af11..5e201572 100644
--- a/DevDoc/theory/entropy/index.html
+++ b/DevDoc/theory/entropy/index.html
@@ -868,14 +868,14 @@
   
   
     
    76. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1085,6 +1085,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    77. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      78. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    78. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      79. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/theory/evolutionary_distances/index.html b/DevDoc/theory/evolutionary_distances/index.html
index e7627d12..77c3d6cf 100644
--- a/DevDoc/theory/evolutionary_distances/index.html
+++ b/DevDoc/theory/evolutionary_distances/index.html
@@ -909,6 +909,56 @@
     
   
   
+
+      
+        
    79. 
+  
+    
+      
+        --distance unification: SNP corrections as first-class metrics (implemented, 2026-08-28)
+      
+    
+  
+  
+    
+  
 
      80. 
       
         
    80. 
@@ -1244,14 +1294,14 @@
   
   
     
    81. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1461,6 +1511,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    82. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      83. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    83. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      84. 
+  
+
+              
+            
           
         
       
@@ -2139,6 +2245,56 @@
     
   
   
+
+      
+        
    84. 
+  
+    
+      
+        --distance unification: SNP corrections as first-class metrics (implemented, 2026-08-28)
+      
+    
+  
+  
+    
+  
 
      85. 
       
         
    85. 
@@ -3554,7 +3710,7 @@ diagonal entry directly, at no extra lookup cost.
      
 
      Mirror distance.rs: open the presence or count store per partition. But
 instead of a per-partition partial_*, run the sequential source sweep:
      
 
      for p in 0..n_partitions:                       # OUTER — sequential
-    open source partition p's layers (QueryLayer-style, obikpartitionner)
+    open source partition p's layers (QueryLayer-style, obikindex::partition)
     enumerate distinct canonical k-mers of p (one per MPHF slot) with their
         presence/count vectors                   # column-major, as query stage 2
     par_iter over these source k-mers:           # INNER — rayon, thread-local tally
@@ -3577,7 +3733,7 @@ instead of a per-partition partial_*, run the sequential source swe
     merge thread-local tallies into global SnpTally
 
      
 
      The inner lookup is precisely QueryLayer::find_slot +
-col_value(g, slot) (obikpartitionner/src/query_layer.rs) — reuse or factor
+col_value(g, slot) (obikindex/src/partition/query_layer.rs) — reuse or factor
 out that path rather than reimplementing MPHF access. Enumerating "all distinct
 k-mers of a partition with their vectors" is the dump/query stage-2
 column-major scan already implemented in dump_layer.rs /
@@ -4205,6 +4361,249 @@ among the survivors) — a single extra pass is sufficient.
      
 M call at ~1/62 frequency, --iqtree-min-freq 0.05; asserts M absent
 from the written _iqtree_states.csv and A/C still present). Full
 workspace cargo test green.
      
+
      --distance unification: SNP corrections as first-class metrics (implemented, 2026-08-28)
      
+
      Implemented. --metric (renamed --distance — several of
+its existing values, e.g. Bray-Curtis, aren't metrics in the strict sense,
+--metric was a misnomer) gains a family of snp-* values computed from the
+central-position SNP pipeline, routed internally to the sibling-annex
+machinery (PairwiseTally, obikphylo::siblings::algorithms::pairwise)
+instead of cache.distance(...)'s existing per-layer traversal — a different
+code path behind the same CLI surface, not just another branch of one
+formula function.
      
+
      Why unify at the CLI level despite the implementation split: phylogenetically
+a SNP-corrected distance is a distance like any other — NJ/UPGMA are agnostic
+to how the matrix was produced, so exposing it as a special-cased subcommand
+instead of a --distance value would misrepresent its role. The
+implementation divergence (sibling-annex-based vs. plain index scan) is real
+but belongs at the routing layer, invisible to the CLI's own vocabulary.
      
+
      --subsample becomes optional for snp-* distances (it stays mandatory
+for --sankoff/--pseudo-alignment, unrelated commands): absent means
+exhaustive, achieved for free by reusing sample_index's existing
+proportional-per-layer-quota mechanism with n set to the index-wide total
+non-monomorphic-minorant count (already available from the sibling-annex
+stats) — every layer's quota then equals its own full count, giving Bernoulli
+p = 1 everywhere, i.e. every eligible family is drawn. No second,
+exhaustive-only driver needed. Present means sampled, exactly as --sankoff
+already behaves.
      
+
      One shared tally, many derived formulas. PairwiseTally's subst[4][4]
+per-pair substitution counts (plus marginal base frequencies derived from it)
+are the sufficient statistic for every closed-form correction below — each
+is a small pure function PairwiseTally -> Array2<f64>, at the same level as
+the already-implemented raw_snp_distance/base_pair_tally/
+cardinality_tally. No new full scan per formula, whether the tally itself
+was built exhaustively or from a subsample.
      
+
      --raw-snp-counts stays a separate, unrelated flag — same underlying
+tally, but a diagnostic (n_snp/n_shared/n_eligible per genome pair, one
+row per pair) rather than a distance value, and its long-table shape doesn't
+fold into a single N×N matrix the way a distance does. No change to its
+existing CSV format.
      
+
      snp-* distance catalog
      
+
      All closed-form (method-of-moments / direct formula), none requiring
+per-pair or per-tree maximum-likelihood fitting — that excludes HKY85's
+tree-ML usage but not its pairwise estimator, which is closed-form like
+F84/TN93 and is included below. snp- prefix on every CLI value.
      
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
      valuecorrects forinputs beyond raw counts
      snp-rawnothing (uncorrected p-distance)—
      snp-jc (Jukes-Cantor, JC69)multiple substitutions per site—
      snp-k2p (Kimura 2-parameter, K80)+ transition/transversion rate biasts/tv split
      snp-k81 (Kimura 3-parameter, K3ST)+ splits transversions into 2 categoriests/tv split, by category
      snp-f81 (Felsenstein 81)+ unequal base frequencies (no ts/tv split)empirical base freqs
      snp-tajima-nei (Tajima-Nei 1984)same goal as F81 (equal-input model), different formula, better small-sample behaviorempirical base freqs
      snp-t92 (Tamura 3-parameter)K2P + GC-content biasts/tv split, GC content
      snp-f84 (Felsenstein 84)full empirical base freqs + single ts/tv rateempirical base freqs, ts/tv split
      snp-hky85 (Hasegawa-Kishino-Yano, pairwise estimator)same inputs as F84, different formulaempirical base freqs, ts/tv split
      snp-tn93 (Tamura-Nei)full empirical base freqs + separate purine/pyrimidine transition rates + transversion rateempirical base freqs, purine-ts/pyrimidine-ts/tv split
      snp-logdet (LogDet / paralinear)no shared-model or stationarity assumption at all — general divergence-matrix determinantfull empirical 4×4 divergence matrix (already subst[4][4])
      snp-tv (transversions-only p-distance)diagnostic/deep-divergence variant — drops transitions entirely (they saturate first)tv-only counts
      
+
      +Γ rate-heterogeneity modifier, applicable to snp-jc, snp-k2p,
+snp-k81, snp-t92, snp-f84, snp-hky85, snp-tn93 (not snp-raw,
+nothing to correct; not snp-logdet, no standard gamma formulation) — same
+formula as the base correction, weighted by a shape parameter α supplied
+by the user (--gamma-shape <alpha>), not estimated by ML. A modifier on
+existing values, not a separate enum arm per distance.
      
+
      Implemented now: snp-raw, snp-jc, snp-k2p, snp-k81, snp-f81,
+snp-t92, snp-tn93, snp-tv, all with +Γ except raw/tv — see
+"Exact formulas" below. snp-tajima-nei, snp-f84, snp-hky85,
+snp-logdet are catalogued above but not implemented: snp-logdet
+needs the true directional per-pair base co-occurrence matrix
+(PairwiseTally only keeps the symmetrised substitution counts
+BasePairTally itself wants — see snp_distance.rs's own module docs for
+why that loses exactly the compositional-asymmetry information LogDet
+exists to detect), snp-tajima-nei needs each genome's own base
+composition (not the pair-pooled estimate the formulas below use), and
+snp-f84/snp-hky85 had no formula independently verified against a
+primary source at implementation time (unlike every formula below, checked
+line-by-line against ape's own
+src/dist_dna.c, not re-derived from memory). Adding any of these later is
+a new function in obikphylo::siblings::algorithms::snp_distance, plus for
+snp-logdet/snp-tajima-nei a new field on PairStats/a per-genome
+accumulator — not an architecture change.
      
+
      Exact formulas (implemented, 2026-08-28)
      
+
      Sufficient statistic, per genome pair (i, j), from
+PairwiseTally::categories/PairwiseTally::base_freq (base order always
+0=A, 1=C, 2=G, 3=T, matching FamilyMask/STATE_SYMBOL):
      
+
        
+
      • \(n_{ts1}\): A↔G substitutions (purine transitions), \(n_{ts2}\): C↔T
+  (pyrimidine transitions)
        • 
+
      • \(n_{tv1}\): A↔C and G↔T substitutions, \(n_{tv2}\): A↔T and C↔G
+  (Kimura's two transversion categories)
        • 
+
      • \(n_{shared}\): loci where both genomes agree
        • 
+
      • \(L = n_{ts1} + n_{ts2} + n_{tv1} + n_{tv2} + n_{shared}\) (total eligible
+  loci for the pair)
        • 
+
      • \(\pi_A, \pi_C, \pi_G, \pi_T\): pair-pooled base frequencies,
+  \(\pi_a = \dfrac{2 \cdot (\text{agreements on } a) + \sum_b n_{a \leftrightarrow b}}{2L}\)
+  (both genomes' calls at this pair's eligible loci, pooled — Nei & Kumar's
+  standard pairwise estimator, not a whole-index average)
        • 
+
      
+
      Derived proportions used below:
      
+
      \[
+p = \frac{n_{ts1}+n_{ts2}+n_{tv1}+n_{tv2}}{L}, \quad
+P = \frac{n_{ts1}+n_{ts2}}{L}, \quad
+Q = \frac{n_{tv1}+n_{tv2}}{L}, \quad
+Q_1 = \frac{n_{tv1}}{L}, \quad
+Q_2 = \frac{n_{tv2}}{L}, \quad
+P_1 = \frac{n_{ts1}}{L}, \quad
+P_2 = \frac{n_{ts2}}{L}
+\]
      
+
      Every formula below was checked term-by-term against ape's own
+src/dist_dna.c (not re-derived from memory) before being ported to
+obikphylo::siblings::algorithms::snp_distance.
      
+
      snp-raw — uncorrected p-distance:
      
+
      \[
+d_{raw} = p
+\]
      
+
      snp-tv — transversions-only p-distance (deliberately uncorrected —
+dropping transitions, which saturate first, is the correction):
      
+
      \[
+d_{tv} = Q
+\]
      
+
      snp-jc (Jukes-Cantor, JC69):
      
+
      \[
+d_{JC} = -\frac{3}{4} \ln\!\left(1 - \frac{4p}{3}\right)
+\]
      
+
      snp-k2p (Kimura 2-parameter, K80), with \(a_1 = 1-2P-Q\), \(a_2 = 1-2Q\):
      
+
      \[
+d_{K2P} = -\frac{1}{2}\ln a_1 - \frac{1}{4}\ln a_2
+\]
      
+
      snp-k81 (Kimura 3-parameter, K3ST), with \(a_1 = 1-2P-2Q_1\),
+\(a_2 = 1-2P-2Q_2\), \(a_3 = 1-2Q_1-2Q_2\):
      
+
      \[
+d_{K81} = -\frac{1}{4}\left(\ln a_1 + \ln a_2 + \ln a_3\right)
+\]
      
+
      snp-f81 (Felsenstein 81), with \(E = 1 - \left(\pi_A^2+\pi_C^2+\pi_G^2+\pi_T^2\right)\):
      
+
      \[
+d_{F81} = -E \ln\!\left(1 - \frac{p}{E}\right)
+\]
      
+
      snp-t92 (Tamura 3-parameter), with GC content
+\(g = \pi_C+\pi_G\), \(w = 2g(1-g)\), \(a_1 = 1 - \dfrac{P}{w} - Q\),
+\(a_2 = 1-2Q\):
      
+
      \[
+d_{T92} = -w \ln a_1 - \frac{1}{2}(1-w)\ln a_2
+\]
      
+
      snp-tn93 (Tamura-Nei), with purine/pyrimidine pooled frequencies
+\(g_R = \pi_A+\pi_G\), \(g_Y = \pi_C+\pi_T\), and
      
+
      \[
+k_1 = \frac{2\pi_A\pi_G}{g_R}, \quad
+k_2 = \frac{2\pi_C\pi_T}{g_Y}, \quad
+k_3 = 2\left(g_R g_Y - \frac{\pi_A\pi_G\, g_Y}{g_R} - \frac{\pi_C\pi_T\, g_R}{g_Y}\right)
+\]
      
+
      \[
+w_1 = 1 - \frac{P_1}{k_1} - \frac{Q}{2g_R}, \quad
+w_2 = 1 - \frac{P_2}{k_2} - \frac{Q}{2g_Y}, \quad
+w_3 = 1 - \frac{Q}{2g_R g_Y}
+\]
      
+
      \[
+d_{TN93} = -k_1 \ln w_1 - k_2 \ln w_2 - k_3 \ln w_3
+\]
      
+
      +Γ gamma correction (Jin & Nei 1990): every formula above is a
+weighted sum of \(-\ln(x)\) terms; the gamma-corrected version replaces
+each such term with the same weight applied to
+\(\alpha\left(x^{-1/\alpha} - 1\right)\) instead — the standard mechanical
+substitution (as \(\alpha \to \infty\), this expression → \(-\ln(x)\),
+recovering the uncorrected formula exactly). E.g. for JC:
      
+
      \[
+d_{JC,\Gamma} = \frac{3}{4}\,\alpha\left[\left(1-\frac{4p}{3}\right)^{-1/\alpha} - 1\right]
+\]
      
+
      Verified term-by-term against ape's own gamma branches for JC69/K80/F81
+(including K80's two-term form — algebraically identical to the generic
+substitution applied to snp-k2p's own \(a_1\)/\(a_2\) terms above, checked
+both symbolically and numerically before simplifying the implementation to
+share one corrected_log helper across every model rather than
+special-casing K80). K81/T92/TN93's gamma branches follow the same
+mechanical substitution but weren't independently checked against an
+ape-equivalent reference for those three specifically — flagged here, not
+silently assumed correct.
      
+
      Output format: PHYLIP-relaxed by default for the distance matrix
      
+
      Implemented. The primary distance-matrix output
+(_dist.csv today) gains multiple formats: PHYLIP-relaxed becomes the
+default (widely read by external NJ tools — PHYLIP neighbor, FastME,
+T-REX, SplitsTree — relaxed rather than strict to avoid the 10-character
+label truncation, since genome labels here routinely exceed it), a --csv
+flag opts back into the current CSV format, PHYLIP-strict is a possible
+future addition (not now). This changes the default output of every
+existing --distance value (jaccard, hamming, bray-curtis, ...), not just
+the new snp-* ones — accepted explicitly (pre-release, single developer
+user, no external consumers to break). Scoped to the distance matrix only:
+--shared-kmers and --raw-snp-counts are counts, not distances, and keep
+their existing CSV-only format.
      
 
      References
      
 
      The Mash mutation-rate model this discussion contrasts with:
 (Fan et al. 2015; Marbl Lab 2026)1 2.
      
diff --git a/DevDoc/theory/indexing.refs/index.html b/DevDoc/theory/indexing.refs/index.html
index ee81d98c..890844ab 100644
--- a/DevDoc/theory/indexing.refs/index.html
+++ b/DevDoc/theory/indexing.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    86. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    87. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      88. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    88. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      89. 
+  
+
+              
+            
           
         
       
@@ -1214,8 +1270,8 @@
 
      Coverage: theory/indexing.md
      
 
      Code couvert
      
 
        
-
      • obikpartitionner/src/partition.rs — routage par hash de minimiseur, choix des paramètres
        • 
-
      • obikpartitionner/src/lib.rs — structure KmerPartition, nombre de partitions
        • 
+
      • obikindex/src/partition/partition.rs — routage par hash de minimiseur, choix des paramètres
        • 
+
      • obikindex/src/partition/mod.rs — structure KmerPartition, nombre de partitions
        • 
 
      
 
      Notes
      
 
      Vérifier que la doc mentionne bien que le nombre de partitions est une puissance de 2
diff --git a/DevDoc/theory/indexing/index.html b/DevDoc/theory/indexing/index.html
index 3de3a3e9..afbd21e6 100644
--- a/DevDoc/theory/indexing/index.html
+++ b/DevDoc/theory/indexing/index.html
@@ -802,14 +802,14 @@
   
   
     
    89. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1019,6 +1019,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    90. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      91. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    91. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      92. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/theory/minimizer.refs/index.html b/DevDoc/theory/minimizer.refs/index.html
index 046fa357..efa795a8 100644
--- a/DevDoc/theory/minimizer.refs/index.html
+++ b/DevDoc/theory/minimizer.refs/index.html
@@ -727,14 +727,14 @@
   
   
     
    92. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -944,6 +944,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    93. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      94. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    94. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      95. 
+  
+
+              
+            
           
         
       
diff --git a/DevDoc/theory/minimizer/index.html b/DevDoc/theory/minimizer/index.html
index cf959bac..f7d4cdc0 100644
--- a/DevDoc/theory/minimizer/index.html
+++ b/DevDoc/theory/minimizer/index.html
@@ -846,14 +846,14 @@
   
   
     
    95. 
-      
+      
         
   
   
   
     
   
-    obilayeredmap crate
+    obikindex layer tier
   
 
     
@@ -1063,6 +1063,62 @@
 
               
             
+              
+                
+  
+  
+  
+  
+    
    96. 
+      
+        
+  
+  
+  
+    
+  
+    Benchmark: query-path testing
+  
+
+    
+  
+  
+  
+
+      
+    
      97. 
+  
+
+              
+            
+              
+                
+  
+  
+  
+  
+    
    97. 
+      
+        
+  
+  
+  
+    
+  
+    Partition and layer caching (discussion)
+  
+
+    
+  
+  
+  
+
+      
+    
      98. 
+  
+
+              
+            
           
         
       
diff --git a/DevDocMD/architecture/index_architecture.refs.md b/DevDocMD/architecture/index_architecture.refs.md
index ff9e3c00..4ca5528e 100644
--- a/DevDocMD/architecture/index_architecture.refs.md
+++ b/DevDocMD/architecture/index_architecture.refs.md
@@ -3,10 +3,10 @@
 
 ## Code couvert
 
-- `obilayeredmap/src/layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix
-- `obilayeredmap/src/mphf_layer.rs` — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum
-- `obilayeredmap/src/map.rs` — LayeredMap
-- `obilayeredmap/src/meta.rs` — LayerMeta, PartitionMeta
+- `obikindex/src/layer/typed_layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix
+- `obikindex/src/layer/mphf_layer.rs` — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum
+- `obikindex/src/layer/map.rs` — LayeredMap
+- `obikindex/src/layer/meta.rs` — LayerMeta, PartitionMeta
 - `obikindex/src/meta.rs` — IndexConfig (kmer_size, n_bits, with_counts, evidence, block_bits), IndexMeta
 - `obikindex/src/index.rs` — KmerIndex, build_layers
 - `obicompactvec/src/` — PersistentCompactIntMatrix, PersistentBitMatrix (DataStore implementations)
diff --git a/DevDocMD/architecture/numa_partition_runner.md b/DevDocMD/architecture/numa_partition_runner.md
index 06b7765c..994e8c76 100644
--- a/DevDocMD/architecture/numa_partition_runner.md
+++ b/DevDocMD/architecture/numa_partition_runner.md
@@ -304,10 +304,29 @@ step, capped by that node's remaining dormant workers and by the run's total
 budget (`n_total`) — balance across nodes is now guaranteed by construction,
 not incidental to channel implementation details.
 
+## Panic propagation (fixed, 2026-08-28)
+
+A worker whose `f(i)` panicked used to never send its `WorkerEvent::Completed`
+— the controller's `while completed < n_total` loop then waited forever for
+an event that partition could no longer produce, since other live
+workers/timer threads kept `event_rx` open. Discovered via `obikselect`
+panicking on an unimplemented `Sparse` matrix case (see `implementation/select.md`):
+the process hung indefinitely instead of erroring out.
+
+Fixed: each worker wraps `f(i)` in `std::panic::catch_unwind` and sends a new
+`WorkerEvent::Panicked(i, payload)` instead of silently dropping the
+partition. The controller counts it toward `completed` (unblocking the loop)
+and keeps the first payload seen; once `run` returns, `panic::resume_unwind`
+re-raises it on the caller's thread — the original message/backtrace still
+surfaces, from the right place, instead of a silent deadlock. Takes priority
+over a plain `Err` (a panic means a bug, not a normal typed failure).
+
 ## Open questions
 
 - **Error handling**: `run` currently returns the first error; remaining errors
-  are dropped.  A `Vec` return would give complete diagnostics.
+  are dropped.  A `Vec` return would give complete diagnostics. (Panics are
+  now caught and re-raised individually — see above — this only concerns
+  plain `Err` results.)
 
 - **`INITIAL_DIVISOR` / `GROWTH_DIVISOR` tuning**: currently `4` and `8`
   (start at 1/4 of a node's cores, grow by 1/8 per step), chosen to fix an
diff --git a/DevDocMD/architecture/numa_worker_pools.md b/DevDocMD/architecture/numa_worker_pools.md
index 434d5135..673a4ba9 100644
--- a/DevDocMD/architecture/numa_worker_pools.md
+++ b/DevDocMD/architecture/numa_worker_pools.md
@@ -72,9 +72,9 @@ Each NUMA group runs its own independent adaptive pool. Workers are distributed
 |------|--------|
 | `obikindex/src/merge.rs` | Detect NUMA topology; build N `ThreadPool`s with pinned threads; assign each pre-spawned worker to a pool; wrap `merge_partition` in `pool.install()` |
 | `obikindex/src/merge.rs` | Replace `available_parallelism()` with per-NUMA core count for spawn criterion |
-| `obikpartitionner/src/merge_layer.rs` | No change — `merge_partition` already works inside any Rayon context |
+| `obikindex/src/partition/merge_layer.rs` | No change — `merge_partition` already works inside any Rayon context |
 | `obidebruinj/src/debruijn.rs` | No change — `par_iter` and `current_num_threads` are pool-context-aware |
-| `obikpartitionner/src/partition.rs` | No change — same reason |
+| `obikindex/src/partition/partition.rs` | No change — same reason |
 
 ## Platform guard
 
diff --git a/DevDocMD/architecture/query.md b/DevDocMD/architecture/query.md
index 79ca8a25..98a1ebe1 100644
--- a/DevDocMD/architecture/query.md
+++ b/DevDocMD/architecture/query.md
@@ -16,7 +16,7 @@ Given a set of query sequences, determine for each sequence how many of its k-me
 
 ## Algorithm
 
-The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikpartitionner::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.
+The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikindex::partition::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.
 
 ```
 for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
@@ -118,11 +118,11 @@ The `-z` CLI option overrides the index metadata value. A higher z increases str
 
 ## Layer lookup: `MphfLayer::find`
 
-`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obilayeredmap](../implementation/obilayeredmap.md) for the full `find` / `find_strict` API.
+`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obikindex::layer](../implementation/layer_tier.md) for the full `find` / `find_strict` API.
 
 ### `QueryLayer` variant selection
 
-`QueryLayer::open` (`obikpartitionner/src/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order:
+`QueryLayer::open` (`obikindex/src/partition/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order:
 
 | Order | Condition | Variant | Data returned per k-mer |
 |---|---|---|---|
@@ -244,7 +244,7 @@ Fix direction: restructure `query`'s pipe with an initial `Flat` stage analogous
 `chunk_bytes = available_memory_bytes() / (n_workers * 16)` (`query.rs:407-414`) assumes a fixed ~8–16× overhead per raw input byte. But `KmerResults::new` (`query.rs:165-179`) allocates `data: Vec` sized `total_kmers_in_chunk × n_genomes` — dense, **for every k-mer position in the chunk, hit or not** — plus `win_min` and (with `--detail`) `cov`, same scaling. Real per-chunk memory is `O(n_genomes)`, not constant; the formula doesn't know `n_genomes` at all. This is the direct cause of the OOM kill on indexes with many reference genomes.
 
 **4. MPHF lookup and matrix-row fetch are fused, not staged.**
-`QueryLayer::find_into` (`obikpartitionner/src/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part).
+`QueryLayer::find_into` (`obikindex/src/partition/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part).
 
 **5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.**
 `QueryBatch::from_records` currently dereplicates at the *superkmer* level (`HashMap>`, `query.rs:112`). This misses redundancy between k-mers shared by *different* superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer *construction* (`SuperKmerIter`) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on `CanonicalKmer`, in the same pass: `HashMap>`. This also means the MPHF `find` itself runs once per **distinct** k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.
@@ -285,7 +285,7 @@ None of this is implemented yet — parked here as a coherent roadmap while the
 
 ## Implementation plan
 
-Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikpartitionner`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)).
+Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikindex::partition`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)).
 
 Instrumentation is deliberately sequenced *before* the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.
 
@@ -295,7 +295,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
 
 **Debug logging.** Every phase that changes an algorithmic choice (not phase 0, which *is* the logging) adds `tracing::debug!`/`trace!` at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF `find` calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured `debug!` per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.
 
-**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obilayeredmap`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/.rs`. Neither `obikmer` nor `obikpartitionner` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
+**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obikindex::layer`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/.rs`. Neither `obikmer` nor `obikindex::partition` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.
 
 **Workflow (`jj`).** Work happens in a fresh `jj` commit, easy to abandon. `jj new` between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then `query_layer.rs` split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.
 
@@ -345,13 +345,13 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
 - `obikmer/src/cmd/query.rs`:
   - Replace `QueryBatch::from_records`'s dedup map (`HashMap>`, current `query.rs:112`) with a per-partition `HashMap>`, built in the same `SuperKmerIter` pass: superkmer construction and partition routing (`part_idx` from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each `CanonicalKmer` within a superkmer is inserted individually instead of the whole superkmer being the dedup key.
   - **Verified**: `CanonicalKmer` (`obikseq/src/kmer.rs:390`, `pub type CanonicalKmer = CanonicalKmerOf`) — the underlying `CanonicalKmerOf` derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash` (`kmer.rs:269`). Usable as a `HashMap`/`HashSet` key as-is, no change needed.
-- `obikpartitionner/src/query_layer.rs`:
+- `obikindex/src/partition/query_layer.rs`:
   - Split `QueryLayer::find_into` (`query_layer.rs:48-67`) into two methods: `find_slot(&self, kmer: CanonicalKmer) -> Option` (MPHF only, no matrix touch) and keep `fill_row` as-is for phase 4 to call later.
   - Replace `query_partition_with`'s inner loop (`query_layer.rs:103-113`) with a version that, for each unique `CanonicalKmer`, calls `find_slot` across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records `(layer_idx, slot)`.
   - New return shape for the partition-level query, replacing today's `on_hit(sk_idx, kmer_idx, row)` callback: `HashMap>>` (roadmap point 6) — built directly from the k-mer dedup map's `Vec<(seq_idx,pos)>` values, keyed by the resolved slot instead of the k-mer.
 - **This phase alone has no throughput benefit yet** (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.
 - Log, per chunk: total k-mer occurrences vs. unique `CanonicalKmer` count (the dedup ratio — the entire justification for this phase) and the resulting MPHF `find` call count. If the dedup ratio is close to `1.0` on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.
-- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikpartitionner/src/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced.
+- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikindex/src/partition/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced.
 
 ### Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred
 
@@ -359,7 +359,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
 
 **What shipped:**
 - `obicompactvec`: the per-column accessors this phase needed **already existed** — `PersistentCompactIntMatrix::col_view(c)` and `PersistentBitMatrix::col_view(c)` are public, and `IntSliceView::get(slot)`/`BitSliceView::get(slot)` are public — the original plan underestimated how much of this plumbing the pairwise-distance code (`dump`/`select`/`stats`) had already required. The one real gap: `PersistentBitMatrix::col_view()` panics on the `Implicit` variant (the documented mono-genome fast path, `bitmatrix.rs`). Added `PersistentBitMatrix::get(c, slot) -> u32` (`bitmatrix.rs`), a non-panicking column-major point lookup that returns `1` for `Implicit` regardless of `c` — the smallest surface needed, not a new `col_get` API from scratch.
-- `obikpartitionner/src/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call.
+- `obikindex/src/partition/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call.
 - `obikmer/src/cmd/query.rs`: `KmerResults::set` (row-major, whole-row-at-once) replaced by `mark_found` (stage 1: flag a position as indexed, independent of any genome's value) and `set_one` (stage 2: write one genome's value at one position). `QueryStats` extended with `n_columns_scanned`/`n_col_get_calls`, logged per chunk.
 - Total `get()`-equivalent calls are unchanged from the row-major version (`n_hits × n_cols` in the worst case, confirmed by `n_col_get_calls` in the debug log) — the win is locality (sequential access within one layer's column at a time, across `mmap`'d regions, instead of jumping across all columns per hit), exactly as predicted.
 
@@ -367,7 +367,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the
 Reading `obikindex/src/numa.rs`'s actual `run()` body (not just its doc comments) shows every call spawns a timer thread **plus one OS thread per worker slot on every NUMA node** (`std::thread::scope` + one `s.spawn()` per node per `max_workers`) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned **per call**. This is fine for its actual, established usage in this codebase (`merge.rs`, `index.rs`'s `build_layers`): one `PartitionRunner::new()` + one `run()` call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for `query`'s call pattern: `query_partition_with` runs once per `(chunk, partition)`, potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact.
 The column-major loop in stage 2 is therefore a **plain sequential loop** for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) `rayon`'s already-warm global pool (`into_par_iter()`) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern `numa_worker_pools.md` warns about for a *different* workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one `mmap`'d region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so `PartitionRunner` is invoked once per whole `query` run (or per large batch of chunks) rather than per `(chunk, partition)`, amortising its spawn cost the way `merge`/`build_layers` do — a bigger structural change than this phase's scope.
 - Log (implemented): `QueryStats::n_columns_scanned`/`n_col_get_calls`, folded into the existing per-chunk `debug!("k-mer dedup + column-major fetch", ...)` line (`query.rs`) alongside phase 3's dedup counters.
-- **Unit tests**: extended `obikpartitionner/src/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature.
+- **Unit tests**: extended `obikindex/src/partition/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature.
 - **Validation performed**: full workspace build + `cargo test --workspace`, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same `kmer_count`/`kmer_strict_matches` on every record); (2) the existing 20-genome `benchmark/global_index_presence` index — runs correctly, `n_hits=0` for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the `Implicit`/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) **the critical correctness case**: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from `genomeA` matched **only** `genomeA` (`kmer_count` identical to the pre-dedup occurrence count, zero leakage into `genomeB`'s column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in `n_cols`, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly.
 - **Not yet done**: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-`n_genomes` layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.
 
diff --git a/DevDocMD/architecture/query.refs.md b/DevDocMD/architecture/query.refs.md
index 2dcbc50b..d281c78c 100644
--- a/DevDocMD/architecture/query.refs.md
+++ b/DevDocMD/architecture/query.refs.md
@@ -4,7 +4,7 @@
 ## Code couvert
 
 - `obikmer/src/cmd/query.rs` — commande query, format de sortie
-- `obikpartitionner/src/query_layer.rs` — routage de la requête à travers les partitions
+- `obikindex/src/partition/query_layer.rs` — routage de la requête à travers les partitions
 - `obiread/src/lib.rs` — lecture des séquences d'entrée pour la requête
 
 ## Notes
diff --git a/DevDocMD/architecture/rebuild_filter.md b/DevDocMD/architecture/rebuild_filter.md
index 443aa753..6c0cd940 100644
--- a/DevDocMD/architecture/rebuild_filter.md
+++ b/DevDocMD/architecture/rebuild_filter.md
@@ -94,9 +94,9 @@ Option B avoids storing kmer values and works uniformly regardless of filter sel
 
 ## Files to modify
 
-- `src/obikpartitionner/src/rebuild_layer.rs` — `rebuild_partition` and `iter_src_layers`
+- `src/obikindex/src/partition/rebuild_layer.rs` — `rebuild_partition` and `iter_src_layers`
 - Possibly `src/obicompactvec/` — add column iterator API if not already present
-- `src/obilayeredmap/` — check if per-column sequential access is exposed on `SrcLayerData`
+- `src/obikindex/src/layer/` — check if per-column sequential access is exposed on `SrcLayerData`
 
 ## Open questions
 
diff --git a/DevDocMD/architecture/siblings.md b/DevDocMD/architecture/siblings.md
index 1247cbdb..352956f4 100644
--- a/DevDocMD/architecture/siblings.md
+++ b/DevDocMD/architecture/siblings.md
@@ -17,7 +17,7 @@ one-way function, not an invertible bijection with a stored inverse. Any
 method that reconstructs a kmer from a bare slot number is wrong by
 construction, regardless of the mechanism used (MPHF re-hash, or evidence
 decode + direct unitig read). See `MphfLayer::kmer_at`
-(`obilayeredmap/src/mphf_layer.rs`) — flagged for removal, currently called
+(`obikindex/src/layer/mphf_layer.rs`) — flagged for removal, currently called
 from `obikphylo/siblings/build.rs` and `family_scan.rs` (since removed — see
 "Pending work" status below).
 
@@ -70,7 +70,7 @@ partition is unknown) and must keep going through
 ## Pending work — done
 
 The plan above shipped: `obikphylo` (a new crate — phylo-domain extension
-traits over `obikindex::KmerIndex`/`obilayeredmap::Layer`, replacing the
+traits over `obikindex::KmerIndex`/`obikindex::layer::Layer`, replacing the
 old `obikindex::siblings` module) builds and reads the annex purely in
 iteration order (`SiblingLayerExt::iter_siblings`/`iter_minorants`, both with
 batch variants, mirroring `Layer`'s own `KmerIter`/`KmerBatchIter`
@@ -652,13 +652,13 @@ BRWT-style column-correlation exploitation.
 `PersistentSparseBitMatrix` went from a validated but unused type to a
 real, selectable on-disk format:
 
-- **Generic `Layer`**: `obilayeredmap::Layer`'s presence-only methods
+- **Generic `Layer`**: `obikindex::layer::Layer`'s presence-only methods
   (`n_cols`, `sub_matrix`, `fill_sub_matrix`) are generic over any
   `D: LayerData> + BinaryMatrix`, not hardcoded to
   `PersistentBitMatrix` — `PersistentSparseBitMatrix` implements
   `LayerData` (`open`/`read`) the same way. `find_slot`/`index_batch` were
   already generic over any `D: LayerData`, so they needed no change.
-  Verified by `obilayeredmap`'s
+  Verified by `obikindex::layer`'s
   `presence_layer_generic_over_sparse_matches_dense` test: build a dense
   presence layer, convert it to sparse via `build_from_dense`, open both
   as `Layer`/`Layer` on
@@ -668,7 +668,7 @@ real, selectable on-disk format:
   `AtomicUsize`s in test builds, not thread-local, so a test using a
   different `k` races every other test in the same crate binary; a k=11
   version of this test passed alone but failed under the full
-  `obilayeredmap` suite for exactly that reason before being fixed.)
+  `obikindex::layer` suite for exactly that reason before being fixed.)
 - **`obikphylo::siblings::cache::Mat`** gained a third variant,
   `SparsePresence(Layer)`, alongside `Count`
   and `Presence` — every method (`find_slot`, `index_batch`,
@@ -698,7 +698,7 @@ real, selectable on-disk format:
   dense-path test (`sibling_annex_one_sibling_each`) exactly — proves the
   sparse format round-trips through the real build pipeline
   (`PartitionCache` sparse-detection included), not just the
-  `obicompactvec`/`obilayeredmap` unit layers below it.
+  `obicompactvec`/`obikindex::layer` unit layers below it.
 
 Full workspace `cargo test` (all crates, unit + doc tests) green after
 this change.
@@ -746,7 +746,7 @@ Replaces the "four independent scans" problem above and implements
   (activation: either given; defaults `1.0`/`0.5` for whichever is unset).
 
 Full workspace `cargo test` green after this change (167 unit tests in
-`obicompactvec`+`obilayeredmap`+`obikphylo` alone, plus every other
+`obicompactvec`+`obikindex::layer`+`obikphylo` alone, plus every other
 crate's suite, no regressions).
 
 **Still open, not part of this change** (per "Correction to the 'single
@@ -756,3 +756,269 @@ scan the full unsampled index — never threaded `--subsample`/`--entropy`,
 out of scope here since the reported problem was specifically about the
 `--sankoff`/`--tnt` pipeline's redundant/inconsistent scans, not these
 two standalone flags.
+
+## `query` never benefits from sparse row-major access (found 2026-08-19, not implemented)
+
+Benchmarked `obikmer query` against `global_index_presence` (dense-packed)
+vs. `global_index_presence_sparse` (`pack --sparse`), 100k simulated reads
+× 2 specimens (`benchmark/`, see
+[benchmark_query_testing.md](../implementation/benchmark_query_testing.md)).
+Correctness: 0 mismatches — sparse and dense return bit-identical query
+results. Performance: sparse consistently *slower* than dense (~30-50%,
+reproducible across two runs with warm disk cache), the opposite of
+`pack --sparse`'s stated intent ("faster for single-row access... like
+query").
+
+**Root cause, read from source, not measured in isolation:**
+`KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs:155-220`)
+is architecturally column-major: stage 2 walks `for g in 0..n_cols { for
+slot in hit_slots { layer.col_value(g, slot) } }`, documented (correctly)
+as the right locality strategy for the packed/columnar formats, where
+`col_value` → `PersistentBitMatrix::get` is a genuine O(1) mmap'd column
+read (`persistent.rs:110-113`).
+
+For `Self::Sparse`, that same `get(c, slot)` (`persistent.rs:114-118`)
+allocates a full `n_cols`-wide buffer and calls `fill_row` — materializing
+the *entire row* — just to return one cell. Called from inside the
+column-major double loop, this reconstructs the same row once per genome
+column touched: O(hits × n_cols) full-row rebuilds instead of O(hits).
+`PersistentSparseBitMatrix`'s own native row-major decode
+(`for_each_genome_in_row`, `sparse.rs:164-177`, used correctly by its own
+`row`/`fill_row`/`fill_sub_matrix`) is never reached from the query path
+at all.
+
+**`fill_sub_matrix` (the existing `BinaryMatrix` trait primitive,
+`traits.rs:13-37`) is not the right replacement for `query` either**, even
+once its own dispatch bug is fixed (see next section) — its output shape
+is inherently column-dense: `out[col]` gets an entry for every column,
+including columns with zero hits among the requested slots. On real
+sparse data (a hit typically touching a handful of genomes out of dozens)
+that's still O(n_cols) output regardless of true sparsity. What `query`
+actually wants is the sparse triple stream `(slot, col, value)` it already
+consumes as `QueryHit::Value` — not a materialized sub-matrix.
+
+**Proposed primitive** (design only, not implemented — explicit ask: keep
+count matrices *not excluded*, even though effort right now is
+presence/absence only):
+
+Not a closure-driven `for_each` — a real `Iterator`, one concrete struct
+per matrix format, so the traversal state (current position in the sorted
+slot list, current column, permutation, sparse-row decode cursor…) lives
+in named struct fields instead of being threaded implicitly through
+recursion or a captured closure. RPITIT (stable since 1.75, and this
+workspace is edition 2024) means the trait method can return it without
+naming or boxing the concrete type:
+
+```rust
+/// Yields every nonzero cell among `slots`, in implementation-defined order.
+fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator + 'a;
+// item: (idx into `slots`, col, value)
+```
+
+This is the one foundational primitive per format — both `fill_sub_matrix`
+and a `for_each`-style callback become trivial, free consumers of it
+(`.for_each(f)` is already `Iterator::for_each` from std; `fill_sub_matrix`
+becomes "drain the iterator, scatter into `out[][]`"), instead of two
+independently-maintained traversals that can silently diverge (see the bug
+below — this is exactly how it happened).
+
+- **On `PersistentSparseBitMatrix`**: the struct is nearly free to write —
+  it wraps the existing (currently private) `for_each_genome_in_row`
+  per-row decode, advancing to the next `slots` entry on exhaustion. O(Σ
+  row nnz), zero `n_cols`-wide allocation.
+- **On `PersistentBitMatrix::{Packed,Columnar}`**: revised — cheaper than
+  first thought, by reusing the same split already used for
+  `fill_matrix`'s own implementation instead of hand-writing a resumable
+  state machine at the matrix level. The base-vector layer
+  (`BitSliceView`, `views.rs`) already separates the two concerns:
+  `fill_batch_sorted` (`views.rs:55-60`, sorted-slot batch lookup) and a
+  genuine per-bit `Iterator` (`BitSliceIter`, `views.rs:94+`) sit side by
+  side there, one level below the matrix. Adding a
+  "positions among `sorted_slots` where the bit is set" iterator at that
+  same vector level is a `filter` over the existing `get()` — no new
+  state machine, since `std::iter::Filter` already *is* one, generated by
+  the compiler:
+  ```rust
+  fn nonzero_among_sorted<'s>(&'s self, sorted_slots: &'s [usize]) -> impl Iterator + 's {
+      sorted_slots.iter().copied().filter(move |&slot| self.get(slot))
+  }
+  ```
+  The matrix-level `nonzero_iter` then composes these per column with
+  `flat_map` over `0..n_cols` (each column's hits, tagged with `c`,
+  slot mapped back through the sort permutation `fill_batch`/
+  `fill_batch_sorted` already carry) — again a combinator chain, not a
+  hand-rolled struct. Same algorithm, same mmap/sort locality as today's
+  `fill_sub_matrix`; just assembled from `std` iterator adaptors instead
+  of a loop body writing into a buffer, mirroring the vector/matrix split
+  the codebase already uses for `fill_batch_sorted` rather than
+  introducing a new shape.
+- **`Implicit`**: trivial (`slots.iter().map(|&i| (i, 0, 1))`, one column,
+  always present).
+- **On `PersistentCompactIntMatrix` (counts)**: same treatment as
+  `Packed`/`Columnar` — no sparse count format exists yet ("Explicitly
+  deferred" per `traits.rs:9-12`), so no native low-effort case the way
+  `Sparse` has one, but not excluded either: the iterator's `Item` is
+  already `(usize, usize, u32)`, not `bool`, specifically so presence
+  (`0`/`1`) and counts (arbitrary `u32`) share one primitive instead of a
+  bool/u32 split forcing counts out of the design. Ready for a native
+  sparse-count struct later without a signature change.
+
+Would let `query_partition_with`'s stage 2 collapse to one
+`for (i, g, v) in layer.matrix().nonzero_iter(&hit_slots) { on_event(...) }`
+per layer, format-agnostic, each backend's struct deciding the actual
+traversal.
+
+**This also closes the existing dispatch bug for free, by construction**:
+`PersistentBitMatrix::fill_sub_matrix` (`persistent.rs:190-215`, the enum
+wrapper backing `BinaryMatrix`'s trait impl) today does *not* delegate to
+`PersistentSparseBitMatrix::fill_sub_matrix` for `Self::Sparse` — it
+reimplements the same naive per-(column, slot) `fill_row_bool` loop
+instead, bypassing the efficient native method one file over
+(`sparse.rs:249-258`). `obikphylo::siblings::cache::Mat`
+(`cache.rs:138-145`) independently built its own parallel enum wrapper
+that dispatches correctly — evidence this was worked around rather than
+fixed at the source: two hand-written traversals for the same format,
+free to drift apart, and they did. If `fill_sub_matrix` itself is
+rewritten as "drain `nonzero_iter`, scatter into `out[][]`", there is only
+one traversal per format left to get right — the bug class doesn't just
+get fixed once, it stops being possible to reintroduce.
+
+## Implemented (2026-08-20)
+
+Built as designed above, with one deviation from the original sketch:
+`nonzero_iter` ended up `Box>`, not a bare `impl
+Iterator`, because `Columnar`/`Packed`/`Sparse`/`Implicit` are genuinely
+different concrete types and this method isn't on a trait (kept off
+`BinaryMatrix` deliberately — that trait is used as `dyn BinaryMatrix` in
+`tests/sparse.rs`, and RPITIT methods aren't dyn-compatible). One `Box`
+per `nonzero_iter` call, not per cell — negligible next to what it
+replaces.
+
+- `BitSliceView::nonzero_among_sorted` / `IntSliceView::nonzero_among_sorted`
+  (`obicompactvec/src/views.rs`): the vector-level `filter`/`filter_map`
+  primitive, exactly as sketched — no new state machine, `std`'s own.
+- `PersistentSparseBitMatrix::nonzero_iter` (`bitmatrix/sparse.rs`): native,
+  `std::iter::from_fn` over one buffered row at a time via the existing
+  `for_each_genome_in_row` — no `n_cols`-wide allocation, ever.
+- `PersistentBitMatrix::nonzero_iter` (`bitmatrix/persistent.rs`): dispatches
+  to the above for `Sparse`; for `Columnar`/`Packed`, loops columns,
+  collects each column's `nonzero_among_sorted` hits via `.extend()` (not
+  `flat_map` — a `flat_map` closure can't lazily return something
+  borrowing its own captured sort permutation across separate calls
+  without either boxing per-column or fighting the borrow checker; eager
+  collection into one `Vec` sidesteps it, at zero cost since
+  `fill_sub_matrix` already fully materialized anyway). `Implicit` trivial.
+- `PersistentBitMatrix::fill_sub_matrix` and `sub_matrix` rewritten to
+  drain `nonzero_iter` — the dispatch bug is gone because there is now
+  only one traversal per format, not because the old one was patched.
+  `PersistentCompactIntMatrix::nonzero_iter` added the same way (counts
+  not excluded, per the earlier ask) — no native low-effort case, since no
+  sparse count format exists, but on the same primitive, ready for one.
+- `KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs`):
+  stage 2's column-major `for g { for slot { col_value } }` replaced by one
+  `layer.nonzero_iter(&slot_list)` call per layer, format-agnostic.
+- Tests: `nonzero_iter_matches_dense`, `nonzero_iter_matches_row`, and —
+  the one that actually targets the dispatch bug rather than each type's
+  own correctness — `enum_wrapper_dispatches_to_native_sparse` (builds
+  `PersistentBitMatrix::Sparse(...)` directly, not through `open`, since
+  `open` only auto-detects `Sparse` from a `presence/` dir layout).
+  `cargo test --workspace`: green, no regressions.
+
+**Measured**: re-ran the `benchmark/` query branch (100k reads × 2
+specimens, same setup as the original finding). Correctness still 0
+mismatches. The dense/sparse performance gap is gone — previously sparse
+~30-50% slower than dense, reproducibly; now within ~1-3% either way
+(7.42s dense vs 7.60s sparse for `Escherichia_coli--K-12_MG1655`; 5.25s vs
+5.30s for `Saccharolobus_islandicus--M.16.4`) — noise-level, not a
+systematic gap. `pack --sparse`'s claimed query win isn't confirmed
+outright by this (sparse should arguably now *beat* dense on truly sparse
+real data, not just tie), but the pathological regression is fixed.
+
+## `PersistentCompactIntMatrix::Sparse` — implemented (2026-08-26)
+
+Closes the gap flagged throughout this document ("no sparse count format
+exists yet", `traits.rs:9-12`'s "Explicitly deferred"): `obicompactvec`
+already had `PersistentSparseCompactIntMatrix` (row-major, built on top of
+`PersistentSparseBitMatrix` as its "which columns are non-zero" support,
+values *not* deduplicated — see that struct's own doc comment), but it was
+never wired into `PersistentCompactIntMatrix`, the dense-dispatching enum
+every real consumer (`TypedLayer`,
+`KmerLayer::Count`) actually holds. Concretely: `kmer_index.rs::
+pack_matrices(sparse=true)` already called `pack_sparse_compact_int_matrix`
+on every layer's `counts/` — but `PersistentCompactIntMatrix::open` had no
+code path back to what that just wrote, so a `Count` layer became
+unreadable ("no count matrix found ... run 'obikmer upgrade'") the moment
+anyone ran `pack --sparse` on an index with count layers. Root cause, not a
+workaround: add the missing `Sparse` variant.
+
+- **Enum + dispatch** (`intmatrix.rs`): `PersistentCompactIntMatrix::Sparse
+  (PersistentSparseCompactIntMatrix)`, detected in `open`/`detect_storage`
+  via a `singleton_values.pciv` marker (mirrors `PersistentBitMatrix`'s own
+  `sparse_meta.json` check), reported via `storage_kind()`. `col`/
+  `col_view`/`col_persist` panic/`Unsupported` on `Sparse`, same convention
+  as the bit side. `sub_matrix`/`fill_sub_matrix` and `nonzero_iter`
+  unified the same way `PersistentBitMatrix`'s already are (drain
+  `nonzero_iter`, one traversal per format — see "Implemented
+  (2026-08-20)" above); `nonzero_iter` had to become `Box>`
+  for the same reason (`Columnar`/`Packed`/`Sparse` are different concrete
+  types). No change needed in `obikindex` at all — `KmerLayer::Count`
+  already only ever holds `TypedLayer`, so the
+  enum absorbing `Sparse` fixes the unreadable-layer bug for free, same as
+  `PersistentBitMatrix::Sparse` already did on the presence side.
+
+- **`CountPartials`, non-naive** (`sparse_intmatrix.rs`): unlike
+  `PersistentSparseBitMatrix`'s dict-driven `col_weights_and_pair_counts`,
+  values here aren't deduplicated (two rows can share the same non-zero
+  column set via the same `dict_id` while carrying different counts), so
+  the "weight by how many rows share a dict entry" shortcut doesn't carry
+  over. What does: a single row-major pass (`row_major_pairwise`, decodes
+  each row once via `for_each_cell_in_row`, nests over that row's own
+  co-present columns) — `O(Σ k̄²)` over populated rows instead of the naive
+  `O(n_cols² × n)` column-pair rescan, same complexity class as the bit
+  side minus the dict multiplicity discount. Kernels used: `min(a,b)`
+  (bray, relfreq-bray — both vanish when either side is absent, so no
+  correction needed), `a·b` and `√(a·b)` (euclidean/relfreq-euclidean and
+  hellinger — these *do* need a correction, reconstructed from per-column
+  marginals via `Σ(a-b)² = Σa²+Σb²-2Σab`, since `(a-0)² = a² ≠ 0` unlike
+  the `min`-based formulas). `threshold_jaccard(1)` shortcuts straight to
+  `support`'s own `BitPartials::partial_jaccard` (threshold 1 is exactly
+  presence); `threshold_jaccard(0)` is closed-form (every `u32` is `≥ 0`).
+
+- **Two pre-existing bugs found and fixed while wiring the `threshold==1`
+  shortcut** (`bitmatrix/sparse.rs`, `BitPartials for
+  PersistentSparseBitMatrix`, present since the 2026-08-15 implementation
+  above, never caught because no test compared `Sparse`'s raw `partial_*`
+  output against dense on real data — only the diagonal-blind
+  `jaccard_dist_matrix`/`hamming_dist_matrix` finalisations were tested):
+  1. `partial_jaccard`'s diagonal was `(0, 2×col_weights[i])` instead of a
+     genuine self-comparison `(col_weights[i], col_weights[i])` —
+     `col_weights_and_pair_counts`'s `inter` never pairs a column with
+     itself by construction.
+  2. `partial_hamming`'s off-diagonal formula itself was wrong: `total -
+     union` (count of rows where *neither* column is present) instead of
+     the actual Hamming distance `col_weights[i] + col_weights[j] -
+     2×inter[i,j]` (symmetric-difference size). Only coincides with the
+     correct value when `col_weights[i] + col_weights[j] == total`, so
+     small/synthetic test data could easily have hidden it.
+
+  Neither surfaced through `jaccard_dist_matrix`/`hamming_dist_matrix`
+  (both explicitly zero their own diagonal at finalisation, and the
+  off-diagonal `partial_hamming` bug had gone untested against dense
+  entirely) — only visible to a caller of the raw `partial_*` methods
+  directly, which is exactly what `partial_threshold_jaccard(1)`'s new
+  shortcut became. Fixed at the source, not patched around at the call
+  site; regression test added:
+  `tests::sparse::partial_jaccard_and_hamming_match_dense_including_diagonal`.
+
+- **Tests**: `tests::intmatrix::sparse_roundtrip_matches_columnar`/
+  `sparse_roundtrip_from_packed` (the `open`-dispatch fix, both build
+  paths); `tests::intmatrix::sparse_count_partials_match_dense` (all six
+  `CountPartials` formulas, thresholds 0/1/2/3, against `Columnar` on
+  asymmetric-presence data — this is what caught the diagonal gap in the
+  int side's own new code before it shipped, the same way it exposed the
+  two pre-existing bit-side bugs above); `obikindex`'s
+  `count_layer_transparently_reads_sparse_after_pack` — the actual
+  end-to-end regression test for the original "layer unreadable after
+  `pack --sparse`" bug, built → packed sparse → reopened, compared against
+  the pre-pack dense read. `cargo test -p obicompactvec -p obikindex`:
+  green, no regressions (180 + 12 tests).
diff --git a/DevDocMD/implementation/benchmark_query_testing.md b/DevDocMD/implementation/benchmark_query_testing.md
new file mode 100644
index 00000000..e234e984
--- /dev/null
+++ b/DevDocMD/implementation/benchmark_query_testing.md
@@ -0,0 +1,93 @@
+# Benchmark: query-path testing
+
+`benchmark/Makefile` exercises indexing, merge, and phylo distance
+reconstruction against simulated bacterial genomes. It now also covers
+`obikmer query` — the read-matching path — and the sparse packed
+presence-matrix format (`obikmer pack --sparse`), previously untested by
+this pipeline.
+
+## Motivation
+
+- `query` had no end-to-end coverage. A regression there would not be caught
+  by `verify_presence`/`verify_merge_presence`, which only check index
+  *content* against the `.npz` truth, never the query API.
+- `pack --sparse` produces a presence-matrix format documented (see
+  [siblings.md](../architecture/siblings.md)) as faster for single-row
+  access (query) and slower for column-oriented access (phylo `--metric`).
+  `global_index_presence/` built by `merge_presence.sh` is always packed
+  dense (packing is a stage inside `merge`, not a separate `pack`
+  invocation) — there was no dense/sparse regression check.
+
+## Query read source
+
+Query reads are independent of `simulated_data/` (which is folded into the
+index being queried): reusing those reads would test against the exact
+error draw the index was built from. `query_data///` holds
+a *second*, independent `iss generate` run against the same reference
+genome, via `simulate_query_one.sh` — unseeded, so a second draw picks up
+different sequencing errors than `simulate_one.sh`'s draw for the same
+genome. Fixed at 100,000 read pairs per genome (not coverage-proportional
+like the 15x used for `simulated_data/`), so wall/RSS numbers stay
+comparable across genomes of very different sizes.
+
+Two query-source specimens, hardcoded as `QUERY_SPECIMENS` in
+`make_deps.py`: `Escherichia_coli--K-12_MG1655` (common, well-represented
+bacterium) and `Saccharolobus_islandicus--M.16.4` (the only archaeon in
+`SPECIES` — distant lineage, stresses the query path differently from a
+close-relative match). Two is enough to catch a dense/sparse regression
+without duplicating the exhaustive per-specimen coverage
+`verify_merge_presence` already provides across all `SPECIMENS`.
+
+## Sparse global index
+
+`global_index_presence_sparse/` is built by `pack_sparse.sh`: copy
+`global_index_presence/` wholesale, then `obikmer pack --sparse` in place.
+This works directly because `merge`'s pack stage (`merge.rs:252`,
+`pack_matrices(false)`) keeps the per-genome column files on disk after
+dense-packing — `pack_sparse_bit_matrix` (`obicompactvec/src/bitmatrix/sparse.rs:447`)
+reads those, is idempotent, and removes `matrix.pbmx` once the sparse form
+is written, so `Persistent::open` falls through to the sparse format
+afterward. No separate merge run needed.
+
+## Query runs
+
+`query_one.sh dense|sparse SPECIMEN` runs `obikmer query --count-missing`
+against `global_index_presence` or `global_index_presence_sparse`, output
+gzipped to `query_{dense,sparse}/SPECIMEN.fasta.gz`, Reporter wall/RSS
+captured to `stats/query_{dense,sparse}/SPECIMEN.stats` (same
+stderr-parsing convention as `merge_presence.sh`).
+
+Flags: `--count-missing` only. `--mismatch` is a no-op today
+(`query/mod.rs:212-213`, prints "not yet implemented, ignored") — left off
+rather than tested for a feature that doesn't exist yet.
+
+## Dense/sparse regression
+
+`verify_query.py` compares the two query outputs per specimen, matched by
+read id (not stream position — the query pipeline chunks input across
+worker threads and doesn't guarantee output order). Compares `kmer_count`,
+`kmer_missing`, and the full `kmer_strict_matches` map per read. Any
+mismatch is a real regression: dense and sparse must be content-identical,
+only I/O access pattern differs. `.stats` → `stats/verify_query/`,
+aggregated by `aggregate_stats.sh query|verify_query`-style cases
+(`query_dense`, `query_sparse`, `verify_query`).
+
+## Performance comparison
+
+No dedicated script: the wall/RSS columns from the `query_dense` and
+`query_sparse` aggregated `.stats` CSVs are the dense-vs-sparse performance
+comparison — the expected win for query on sparse, per the `pack --sparse`
+help text.
+
+## Scope
+
+`count` track excluded from the sparse branch: `pack --sparse` targets
+presence matrices only (per CLI help); `pack_matrices` leaves count
+matrices untouched regardless of the `sparse` flag
+(`obikindex/src/index.rs:308`).
+
+## New Makefile targets
+
+`simulate_query`, `pack_sparse`, `query_dense`, `query_sparse`,
+`aggregate_query_dense`, `aggregate_query_sparse`, `verify_query`,
+`aggregate_verify_query` — the last three folded into `all`.
diff --git a/DevDocMD/implementation/evidence_elimination.refs.md b/DevDocMD/implementation/evidence_elimination.refs.md
index 0fedcb34..b5a17e28 100644
--- a/DevDocMD/implementation/evidence_elimination.refs.md
+++ b/DevDocMD/implementation/evidence_elimination.refs.md
@@ -3,9 +3,9 @@
 
 ## Code couvert
 
-- `obilayeredmap/src/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
-- `obilayeredmap/src/mphf_layer.rs` — build_approx_evidence(dir, b, z), find_approx()
-- `obilayeredmap/src/meta.rs` — EvidenceKind::Approx { b, z }, LayerMeta
+- `obikindex/src/layer/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
+- `obikindex/src/layer/mphf_layer.rs` — build_approx_evidence(dir, b, z), find_approx()
+- `obikindex/src/layer/meta.rs` — EvidenceKind::Approx { b, z }, LayerMeta
 - `obikindex/src/reindex.rs` — KmerIndex::reindex(), conversion exact↔approx en place
 - `obikmer/src/cmd/reindex.rs` — CLI reindex, options --approx, -z, --evidence-bits, --fp, --block-size
 - `obikmer/src/cmd/index.rs` — resolve_approx_params(), options --approx, -z, --evidence-bits, --fp
diff --git a/DevDocMD/implementation/filtering.md b/DevDocMD/implementation/filtering.md
index 1021cbae..518d1e17 100644
--- a/DevDocMD/implementation/filtering.md
+++ b/DevDocMD/implementation/filtering.md
@@ -303,7 +303,7 @@ This parameter has no effect on presence/absence indexes (where values are alrea
 
 ## Implementation
 
-- **`obikpartitionner::filter::GroupQuorumFilter`** — implements `KmerFilter`
+- **`obikindex::partition::filter::GroupQuorumFilter`** — implements `KmerFilter`
   using pre-computed ingroup and outgroup index vectors. The heavy logic
   (predicate parsing, three-value evaluation, genome classification) happens
   once before any iteration; each k-mer row evaluation is a simple index
@@ -314,7 +314,7 @@ This parameter has no effect on presence/absence indexes (where values are alrea
   `UnitigArgs`. `FilterArgs::build_filters()` returns a ready-to-use filter
   list.
 
-- **`obikpartitionner::KmerPartition::iter_partition_kmers`** — accepts
+- **`obikindex::partition::KmerPartition::iter_partition_kmers`** — accepts
   `filters: &[Box]` and applies them per-kmer before invoking
   the callback. `filter`, `dump`, and `unitig` all go through this single
   entry point.
diff --git a/DevDocMD/implementation/obilayeredmap.md b/DevDocMD/implementation/layer_tier.md
similarity index 81%
rename from DevDocMD/implementation/obilayeredmap.md
rename to DevDocMD/implementation/layer_tier.md
index 9985bebb..57499b99 100644
--- a/DevDocMD/implementation/obilayeredmap.md
+++ b/DevDocMD/implementation/layer_tier.md
@@ -1,8 +1,8 @@
-# obilayeredmap — layered kmer index crate
+# obikindex::layer — the Layer tier
 
 ## Purpose
 
-`obilayeredmap` implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a `ptr_hash` MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.
+`obikindex::layer` (the `layer/` submodule of the `obikindex` crate — a standalone `obilayeredmap` crate until 2026-08-21, folded back in alongside `obikpartition` as part of a broader `Index { Partition { Layer } }` submodule regrouping) implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a `ptr_hash` MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.
 
 ---
 
@@ -250,6 +250,59 @@ Mode 3 (`PersistentBitMatrix`) has no `push_layer` on `LayeredMap`; callers buil
 
 ---
 
+## Layer\ — raw mapping, iteration, and batch access
+
+Beyond `query`/`find` (membership-checked), `Layer` exposes lower-level access used by consumers that already know a kmer is in the layer (e.g. cross-partition sibling resolution) or that need to sweep every kmer/slot without paying for a membership check each time.
+
+### Raw kmer → slot mapping
+
+```rust
+pub fn index(&self, kmer: CanonicalKmer) -> usize
+pub fn index_batch(&self, kmers: &[CanonicalKmer]) -> Vec
+```
+
+Pure MPHF mapping, no evidence/fingerprint check — equivalent to `MphfOnly::index`. Only meaningful when the caller already knows `kmer` belongs to the layer; on an absent kmer the MPHF still returns *some* slot (undefined, not `None`).
+
+### Kmer iteration
+
+Four iterators, all built from `unitigs.bin` (physical layout order, **not** correlated with MPHF slot numbers):
+
+```rust
+pub fn iter_kmers(&self) -> KmerIter<'_>
+pub fn enumerate_kmers(&self) -> Enumerate>              // (order_index, kmer)
+pub fn iter_kmers_batch(&self, n: usize) -> KmerBatchIter<'_>          // Vec of size ≤ n
+pub fn enumerate_kmers_batch(&self, n: usize) -> impl Iterator)> + Send + 'static
+```
+
+`KmerIter`/`KmerBatchIter` own a clone of the underlying `Arc` rather than borrowing `self` — `Send + 'static`, streamed from disk one kmer at a time, never materialised as a whole. Multiple instances can coexist concurrently, each with its own cursor. `enumerate_kmers_batch`'s index is the batch's starting offset in iteration order (a multiple of `n` except for the final, possibly shorter, batch).
+
+### Batch lookup on payload vectors/views
+
+`PersistentCompactIntVec`, `PersistentBitVec`, `IntSliceView`, `BitSliceView` all expose:
+
+```rust
+fn get_batch(&self, slots: &[usize]) -> Vec
+fn fill_batch(&self, slots: &[usize], out: &mut [T])
+```
+
+Both sort `slots` internally for sequential mmap access, then reorder results back to the caller's original order. `fill_batch` fills a caller-provided buffer, avoiding the `Vec` allocation.
+
+### sub_matrix / fill_sub_matrix
+
+```rust
+// Layer
+pub fn sub_matrix(&self, slots: &[usize]) -> Vec>            // column-first
+pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec])
+
+// Layer (and any D: BinaryMatrix, e.g. PersistentSparseBitMatrix)
+pub fn sub_matrix(&self, slots: &[usize]) -> Vec>
+pub fn fill_sub_matrix(&self, slots: &[usize], out: &mut [Vec])
+```
+
+Column-first to match the on-disk column-major layout. `fill_sub_matrix` sorts `slots` once, then calls each column's `fill_batch` in turn — no redundant per-column sort. On `PersistentSparseBitMatrix` (k-mer-major, no column method) this degrades to a row-by-row decode; see [siblings.md](../architecture/siblings.md).
+
+---
+
 ## LayeredStore\ and aggregation traits
 
 `LayeredStore` is a generic aggregation wrapper over `Vec`. It propagates three traits from `obicompactvec::traits` up the hierarchy via blanket impls:
diff --git a/DevDocMD/implementation/obilayeredmap.refs.md b/DevDocMD/implementation/layer_tier.refs.md
similarity index 54%
rename from DevDocMD/implementation/obilayeredmap.refs.md
rename to DevDocMD/implementation/layer_tier.refs.md
index 5340ab3e..77197823 100644
--- a/DevDocMD/implementation/obilayeredmap.refs.md
+++ b/DevDocMD/implementation/layer_tier.refs.md
@@ -1,14 +1,14 @@
 
-# Coverage: implementation/obilayeredmap.md
+# Coverage: implementation/layer_tier.md
 
 ## Code couvert
 
-- `obilayeredmap/src/mphf_layer.rs` — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
-- `obilayeredmap/src/layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
-- `obilayeredmap/src/map.rs` — LayeredMap, push_layer(), query()
-- `obilayeredmap/src/evidence.rs` — Evidence, EvidenceWriter, encodage chunk_id:rank
-- `obilayeredmap/src/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, matches()
-- `obilayeredmap/src/meta.rs` — LayerMeta, EvidenceKind (Exact / Approx { b, z })
+- `obikindex/src/layer/mphf_layer.rs` — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
+- `obikindex/src/layer/typed_layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
+- `obikindex/src/layer/map.rs` — LayeredMap, push_layer(), query()
+- `obikindex/src/layer/evidence.rs` — Evidence, EvidenceWriter, encodage chunk_id:rank
+- `obikindex/src/layer/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, matches()
+- `obikindex/src/layer/meta.rs` — LayerMeta, EvidenceKind (Exact / Approx { b, z })
 
 ## Notes
 
diff --git a/DevDocMD/implementation/merge.md b/DevDocMD/implementation/merge.md
index cd273495..b93332d6 100644
--- a/DevDocMD/implementation/merge.md
+++ b/DevDocMD/implementation/merge.md
@@ -194,3 +194,14 @@ spectrums/
          | --in-place }
+        --output 
         [--group    :  ...]
         [--group-op :    ...]
         [--aggregate-by           ]
         [--aggregate-op            ]
         [--select       ]
         [--presence-threshold       ]
+        [--force-copy                  ]
 ```
 
 ---
 
 ## Output destination
 
-Exactly one of `--output` or `--in-place` must be specified.
+`--output ` is required — `select` always writes a new index; there is no
+`--in-place` mode (2026-08-28: never implemented, removed from the design). The
+source index is unchanged.
 
-**`--output `** — writes a new index to ``. The source index is
-unchanged. The MPHF and unitig files are copied; only the data matrices are
-rewritten with the new column layout.
+Each layer's kmer-identity files (`mphf.bin`/`unitigs.bin`/`evidence.bin`/
+`unitigs.bin.idx`/`fingerprint.bin`/`layer_meta.json`) are never rewritten by a
+column projection/aggregation, so they are hard-linked into the output rather
+than copied — no extra disk for them even on a large index. Falls back to a
+real copy automatically if linking fails (different filesystems); `--force-copy`
+forces a real copy always, for an output that must survive independently of the
+source on disk (a hard link shares the same inode — rewriting one path outside
+`select` itself would affect the other). Only the `presence`/`counts`
+subdirectory is ever a genuinely new, independent file.
 
-**`--in-place`** — rewrites the data matrices of the source index directly.
-Removed or replaced columns are lost. The operation writes to temporary files
-first, then renames atomically, so an interrupted run leaves the index intact.
+To replace an index with a selected version of itself, select to a temporary
+directory and swap it in (`rm -rf INDEX && mv INDEX.tmp INDEX`) — the case
+`--in-place` used to cover.
 
 ---
 
@@ -186,12 +195,6 @@ obikmer select myindex --output out \
   --select "Betula_nana--TROM-V-149986,Betula_nana--AG-P04-25-01"
 ```
 
-### In-place: keep only group A
-
-```sh
-obikmer select myindex --in-place --group "A:group=A" --select "A"
-```
-
 ### Compose with filter
 
 ```sh
@@ -207,28 +210,45 @@ obikmer select filtered --output final --aggregate-by site
 
 ## Implementation notes
 
-`select` does not rebuild the MPHF. The 256 partitions are processed in parallel
-(rayon), each writing its output independently; results require no synchronisation
-because every partition owns a distinct set of files.
+`select` does not rebuild the MPHF. Every partition is processed independently
+(`PartitionRunner`), each writing its own output layers; no cross-partition
+synchronisation is needed.
 
-For each layer in each partition:
+For each layer in each partition (`obikselect::select_layer::select_partition`):
 
-1. The slot count `n` is read by opening the source data matrix.
-2. A new data matrix is built with M columns (M = number of output columns).
-3. For each slot `s` in `0..n`:
-   - `old_row = matrix.fill_row(s)` — reads the original `N`-column row without allocating.
-   - For each output column `j`:
-     - `new_row[j] = aggregate(op, old_row[group_indices])`.
-     - Pass-through columns are represented as single-element groups with the
-       default operator (`any` for presence, `sum` for count) — same code path.
-   - The new row is written slot by slot into each column builder.
-4. All plain files in the source layer directory (`mphf.bin`, `unitigs.bin`,
-   evidence files, `layer_meta.json`) are copied verbatim; only the `presence/`
-   or `counts/` subdirectory is rewritten.
+1. `copy_layer_files` hard-links the source layer's kmer-identity files
+   (`mphf.bin`/`unitigs.bin`/`evidence.bin`/`unitigs.bin.idx`/
+   `fingerprint.bin`/`layer_meta.json`) into the destination — never a real
+   copy unless linking fails or `--force-copy` is given.
+2. A new data matrix is built with M columns (M = number of output columns),
+   under a fresh `presence/`/`counts/` subdirectory (never touching the
+   source's own).
+3. **Presence source (2026-08-28: `batch_presence_counts`)**: one shared pass
+   over the source bit matrix computes every output group's presence count at
+   once — row-major native for a `Sparse` source (`for_each_genome_in_row`,
+   which has no column representation to read a `col_view` from at all — the
+   reason this replaced the old per-group loop, not just an optimisation of
+   it), deduplicated column-major (one `col_view` per *distinct referenced
+   column*, not per group) for `Columnar`/`Packed`. Every `AggOp` for a bit
+   matrix is then a cheap derivation of that one count vector (`sum` = the
+   count itself, `any`/`max` = `count ≥ 1`, `all`/`min` = `count == group
+   size`, `none` = `count == 0`) — see
+   `obikselect::select_layer::agg_result_from_count`.
+4. **Count source**: unchanged, one `col_view`-driven pass per output column
+   via `MatrixGroupOps` — `sum`/`min`/`max` are genuine per-value reductions
+   for a count matrix, not derivable from a single presence count the way
+   they are for a bit matrix.
 5. `index.meta` is rewritten with the new genome list and updated `with_counts`.
 
-**`--in-place` write strategy:** new data is written to a temporary sibling
-directory (`presence_new/` or `counts_new/`); on success the old directory is
-removed and the temporary one is renamed into place. An interrupted run leaves
-at most one stale `*_new/` directory; the original data is intact until the
-rename step.
+### Known gap (not yet fixed, 2026-08-28)
+
+Step 4 above still panics (`col_view() not available on Sparse
+PersistentCompactIntMatrix`) if the source is a **count** index packed
+sparse — `batch_presence_counts`' row-major treatment was only ported to the
+bit-matrix (`Presence`) case, since that was the one actually blocking a real
+benchmark run. `select`/`filter` on a sparse-packed count index still hits
+this; the fix would follow the same shape (a `PersistentSparseCompactIntMatrix`
+row-major decode, analogous to `for_each_genome_in_row`), just not done. Since
+`obisys::numa::runner::PartitionRunner`'s panic-propagation fix (see
+`architecture/numa_partition_runner.md`), this at least fails fast (process
+panic, exit 101) instead of hanging.
diff --git a/DevDocMD/implementation/storage.refs.md b/DevDocMD/implementation/storage.refs.md
index 69874018..dabd4c06 100644
--- a/DevDocMD/implementation/storage.refs.md
+++ b/DevDocMD/implementation/storage.refs.md
@@ -5,7 +5,7 @@
 
 - `obikindex/src/meta.rs` — IndexMeta, IndexConfig (version, config, genomes)
 - `obikindex/src/index.rs` — layout sur disque : partitions/, index.meta
-- `obilayeredmap/src/meta.rs` — LayerMeta (evidence kind), PartitionMeta (n_layers)
+- `obikindex/src/layer/meta.rs` — LayerMeta (evidence kind), PartitionMeta (n_layers)
 - `obiskio/src/unitig_index.rs` — fichiers unitigs.bin + unitigs.bin.idx
 
 ## Notes
diff --git a/DevDocMD/implementation/unitig_evidence.refs.md b/DevDocMD/implementation/unitig_evidence.refs.md
index d25b2009..185b6cee 100644
--- a/DevDocMD/implementation/unitig_evidence.refs.md
+++ b/DevDocMD/implementation/unitig_evidence.refs.md
@@ -4,7 +4,7 @@
 ## Code couvert
 
 - `obiskio/src/unitig_index.rs` — format unitigs.bin + unitigs.bin.idx, UnitigFileWriter, UnitigFileReader, build_unitig_idx(), DEFAULT_BLOCK_BITS=0, chemin chaud block_bits=0 dans chunk_start()
-- `obilayeredmap/src/evidence.rs` — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
+- `obikindex/src/layer/evidence.rs` — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
 - `obidebruinj/src/debruijn.rs` — extraction unitigs, chunking à MAX_KMERS_PER_CHUNK
 
 ## Notes
diff --git a/DevDocMD/theory/evolutionary_distances.md b/DevDocMD/theory/evolutionary_distances.md
index d1cf4927..7b1a3506 100644
--- a/DevDocMD/theory/evolutionary_distances.md
+++ b/DevDocMD/theory/evolutionary_distances.md
@@ -1489,7 +1489,7 @@ instead of a per-partition `partial_*`, run the sequential source sweep:
 
 ```text
 for p in 0..n_partitions:                       # OUTER — sequential
-    open source partition p's layers (QueryLayer-style, obikpartitionner)
+    open source partition p's layers (QueryLayer-style, obikindex::partition)
     enumerate distinct canonical k-mers of p (one per MPHF slot) with their
         presence/count vectors                   # column-major, as query stage 2
     par_iter over these source k-mers:           # INNER — rayon, thread-local tally
@@ -1513,7 +1513,7 @@ for p in 0..n_partitions:                       # OUTER — sequential
 ```
 
 The inner lookup is precisely `QueryLayer::find_slot` +
-`col_value(g, slot)` (`obikpartitionner/src/query_layer.rs`) — reuse or factor
+`col_value(g, slot)` (`obikindex/src/partition/query_layer.rs`) — reuse or factor
 out that path rather than reimplementing MPHF access. Enumerating "all distinct
 k-mers of a partition with their vectors" is the `dump`/`query` stage-2
 column-major scan already implemented in `dump_layer.rs` /
@@ -2182,6 +2182,230 @@ Covered by `iqtree::tests::iqtree_min_freq_folds_rare_states_into_missing`
 from the written `_iqtree_states.csv` and `A`/`C` still present). Full
 workspace `cargo test` green.
 
+## `--distance` unification: SNP corrections as first-class metrics (implemented, 2026-08-28)
+
+**Implemented.** `--metric` (renamed `--distance` — several of
+its existing values, e.g. Bray-Curtis, aren't metrics in the strict sense,
+`--metric` was a misnomer) gains a family of `snp-*` values computed from the
+central-position SNP pipeline, routed internally to the sibling-annex
+machinery (`PairwiseTally`, `obikphylo::siblings::algorithms::pairwise`)
+instead of `cache.distance(...)`'s existing per-layer traversal — a different
+code path behind the same CLI surface, not just another branch of one
+formula function.
+
+**Why unify at the CLI level despite the implementation split**: phylogenetically
+a SNP-corrected distance is a distance like any other — NJ/UPGMA are agnostic
+to how the matrix was produced, so exposing it as a special-cased subcommand
+instead of a `--distance` value would misrepresent its role. The
+implementation divergence (sibling-annex-based vs. plain index scan) is real
+but belongs at the routing layer, invisible to the CLI's own vocabulary.
+
+**`--subsample` becomes optional for `snp-*` distances** (it stays mandatory
+for `--sankoff`/`--pseudo-alignment`, unrelated commands): absent means
+exhaustive, achieved for free by reusing `sample_index`'s existing
+proportional-per-layer-quota mechanism with `n` set to the index-wide total
+non-monomorphic-minorant count (already available from the sibling-annex
+stats) — every layer's quota then equals its own full count, giving Bernoulli
+`p = 1` everywhere, i.e. every eligible family is drawn. No second,
+exhaustive-only driver needed. Present means sampled, exactly as `--sankoff`
+already behaves.
+
+**One shared tally, many derived formulas.** `PairwiseTally`'s `subst[4][4]`
+per-pair substitution counts (plus marginal base frequencies derived from it)
+are the sufficient statistic for every closed-form correction below — each
+is a small pure function `PairwiseTally -> Array2`, at the same level as
+the already-implemented `raw_snp_distance`/`base_pair_tally`/
+`cardinality_tally`. No new full scan per formula, whether the tally itself
+was built exhaustively or from a subsample.
+
+**`--raw-snp-counts` stays a separate, unrelated flag** — same underlying
+tally, but a diagnostic (`n_snp`/`n_shared`/`n_eligible` per genome pair, one
+row per pair) rather than a distance value, and its long-table shape doesn't
+fold into a single N×N matrix the way a distance does. No change to its
+existing CSV format.
+
+### `snp-*` distance catalog
+
+All closed-form (method-of-moments / direct formula), none requiring
+per-pair or per-tree maximum-likelihood fitting — that excludes HKY85's
+*tree*-ML usage but not its *pairwise* estimator, which is closed-form like
+F84/TN93 and is included below. `snp-` prefix on every CLI value.
+
+| value | corrects for | inputs beyond raw counts |
+|---|---|---|
+| `snp-raw` | nothing (uncorrected p-distance) | — |
+| `snp-jc` (Jukes-Cantor, JC69) | multiple substitutions per site | — |
+| `snp-k2p` (Kimura 2-parameter, K80) | + transition/transversion rate bias | ts/tv split |
+| `snp-k81` (Kimura 3-parameter, K3ST) | + splits transversions into 2 categories | ts/tv split, by category |
+| `snp-f81` (Felsenstein 81) | + unequal base frequencies (no ts/tv split) | empirical base freqs |
+| `snp-tajima-nei` (Tajima-Nei 1984) | same goal as F81 (equal-input model), different formula, better small-sample behavior | empirical base freqs |
+| `snp-t92` (Tamura 3-parameter) | K2P + GC-content bias | ts/tv split, GC content |
+| `snp-f84` (Felsenstein 84) | full empirical base freqs + single ts/tv rate | empirical base freqs, ts/tv split |
+| `snp-hky85` (Hasegawa-Kishino-Yano, pairwise estimator) | same inputs as F84, different formula | empirical base freqs, ts/tv split |
+| `snp-tn93` (Tamura-Nei) | full empirical base freqs + separate purine/pyrimidine transition rates + transversion rate | empirical base freqs, purine-ts/pyrimidine-ts/tv split |
+| `snp-logdet` (LogDet / paralinear) | no shared-model or stationarity assumption at all — general divergence-matrix determinant | full empirical 4×4 divergence matrix (already `subst[4][4]`) |
+| `snp-tv` (transversions-only p-distance) | diagnostic/deep-divergence variant — drops transitions entirely (they saturate first) | tv-only counts |
+
+**`+Γ` rate-heterogeneity modifier, applicable to `snp-jc`, `snp-k2p`,
+`snp-k81`, `snp-t92`, `snp-f84`, `snp-hky85`, `snp-tn93`** (not `snp-raw`,
+nothing to correct; not `snp-logdet`, no standard gamma formulation) — same
+formula as the base correction, weighted by a shape parameter `α` supplied
+by the user (`--gamma-shape `), not estimated by ML. A modifier on
+existing values, not a separate enum arm per distance.
+
+**Implemented now: `snp-raw`, `snp-jc`, `snp-k2p`, `snp-k81`, `snp-f81`,
+`snp-t92`, `snp-tn93`, `snp-tv`, all with `+Γ` except `raw`/`tv`** — see
+"Exact formulas" below. `snp-tajima-nei`, `snp-f84`, `snp-hky85`,
+`snp-logdet` are catalogued above but **not implemented**: `snp-logdet`
+needs the true *directional* per-pair base co-occurrence matrix
+(`PairwiseTally` only keeps the symmetrised substitution counts
+`BasePairTally` itself wants — see `snp_distance.rs`'s own module docs for
+why that loses exactly the compositional-asymmetry information LogDet
+exists to detect), `snp-tajima-nei` needs each genome's *own* base
+composition (not the pair-pooled estimate the formulas below use), and
+`snp-f84`/`snp-hky85` had no formula independently verified against a
+primary source at implementation time (unlike every formula below, checked
+line-by-line against [ape](https://github.com/emmanuelparadis/ape)'s own
+`src/dist_dna.c`, not re-derived from memory). Adding any of these later is
+a new function in `obikphylo::siblings::algorithms::snp_distance`, plus for
+`snp-logdet`/`snp-tajima-nei` a new field on `PairStats`/a per-genome
+accumulator — not an architecture change.
+
+### Exact formulas (implemented, 2026-08-28)
+
+Sufficient statistic, per genome pair `(i, j)`, from
+`PairwiseTally::categories`/`PairwiseTally::base_freq` (base order always
+`0=A, 1=C, 2=G, 3=T`, matching `FamilyMask`/`STATE_SYMBOL`):
+
+- \(n_{ts1}\): A↔G substitutions (purine transitions), \(n_{ts2}\): C↔T
+  (pyrimidine transitions)
+- \(n_{tv1}\): A↔C and G↔T substitutions, \(n_{tv2}\): A↔T and C↔G
+  (Kimura's two transversion categories)
+- \(n_{shared}\): loci where both genomes agree
+- \(L = n_{ts1} + n_{ts2} + n_{tv1} + n_{tv2} + n_{shared}\) (total eligible
+  loci for the pair)
+- \(\pi_A, \pi_C, \pi_G, \pi_T\): pair-pooled base frequencies,
+  \(\pi_a = \dfrac{2 \cdot (\text{agreements on } a) + \sum_b n_{a \leftrightarrow b}}{2L}\)
+  (both genomes' calls at this pair's eligible loci, pooled — Nei & Kumar's
+  standard pairwise estimator, not a whole-index average)
+
+Derived proportions used below:
+
+\[
+p = \frac{n_{ts1}+n_{ts2}+n_{tv1}+n_{tv2}}{L}, \quad
+P = \frac{n_{ts1}+n_{ts2}}{L}, \quad
+Q = \frac{n_{tv1}+n_{tv2}}{L}, \quad
+Q_1 = \frac{n_{tv1}}{L}, \quad
+Q_2 = \frac{n_{tv2}}{L}, \quad
+P_1 = \frac{n_{ts1}}{L}, \quad
+P_2 = \frac{n_{ts2}}{L}
+\]
+
+Every formula below was checked term-by-term against `ape`'s own
+`src/dist_dna.c` (not re-derived from memory) before being ported to
+`obikphylo::siblings::algorithms::snp_distance`.
+
+**`snp-raw`** — uncorrected p-distance:
+
+\[
+d_{raw} = p
+\]
+
+**`snp-tv`** — transversions-only p-distance (deliberately uncorrected —
+dropping transitions, which saturate first, *is* the correction):
+
+\[
+d_{tv} = Q
+\]
+
+**`snp-jc`** (Jukes-Cantor, JC69):
+
+\[
+d_{JC} = -\frac{3}{4} \ln\!\left(1 - \frac{4p}{3}\right)
+\]
+
+**`snp-k2p`** (Kimura 2-parameter, K80), with \(a_1 = 1-2P-Q\), \(a_2 = 1-2Q\):
+
+\[
+d_{K2P} = -\frac{1}{2}\ln a_1 - \frac{1}{4}\ln a_2
+\]
+
+**`snp-k81`** (Kimura 3-parameter, K3ST), with \(a_1 = 1-2P-2Q_1\),
+\(a_2 = 1-2P-2Q_2\), \(a_3 = 1-2Q_1-2Q_2\):
+
+\[
+d_{K81} = -\frac{1}{4}\left(\ln a_1 + \ln a_2 + \ln a_3\right)
+\]
+
+**`snp-f81`** (Felsenstein 81), with \(E = 1 - \left(\pi_A^2+\pi_C^2+\pi_G^2+\pi_T^2\right)\):
+
+\[
+d_{F81} = -E \ln\!\left(1 - \frac{p}{E}\right)
+\]
+
+**`snp-t92`** (Tamura 3-parameter), with GC content
+\(g = \pi_C+\pi_G\), \(w = 2g(1-g)\), \(a_1 = 1 - \dfrac{P}{w} - Q\),
+\(a_2 = 1-2Q\):
+
+\[
+d_{T92} = -w \ln a_1 - \frac{1}{2}(1-w)\ln a_2
+\]
+
+**`snp-tn93`** (Tamura-Nei), with purine/pyrimidine pooled frequencies
+\(g_R = \pi_A+\pi_G\), \(g_Y = \pi_C+\pi_T\), and
+
+\[
+k_1 = \frac{2\pi_A\pi_G}{g_R}, \quad
+k_2 = \frac{2\pi_C\pi_T}{g_Y}, \quad
+k_3 = 2\left(g_R g_Y - \frac{\pi_A\pi_G\, g_Y}{g_R} - \frac{\pi_C\pi_T\, g_R}{g_Y}\right)
+\]
+
+\[
+w_1 = 1 - \frac{P_1}{k_1} - \frac{Q}{2g_R}, \quad
+w_2 = 1 - \frac{P_2}{k_2} - \frac{Q}{2g_Y}, \quad
+w_3 = 1 - \frac{Q}{2g_R g_Y}
+\]
+
+\[
+d_{TN93} = -k_1 \ln w_1 - k_2 \ln w_2 - k_3 \ln w_3
+\]
+
+**`+Γ` gamma correction** (Jin & Nei 1990): every formula above is a
+weighted sum of \(-\ln(x)\) terms; the gamma-corrected version replaces
+each such term with the same weight applied to
+\(\alpha\left(x^{-1/\alpha} - 1\right)\) instead — the standard mechanical
+substitution (as \(\alpha \to \infty\), this expression → \(-\ln(x)\),
+recovering the uncorrected formula exactly). E.g. for JC:
+
+\[
+d_{JC,\Gamma} = \frac{3}{4}\,\alpha\left[\left(1-\frac{4p}{3}\right)^{-1/\alpha} - 1\right]
+\]
+
+Verified term-by-term against `ape`'s own gamma branches for JC69/K80/F81
+(including K80's two-term form — algebraically identical to the generic
+substitution applied to `snp-k2p`'s own \(a_1\)/\(a_2\) terms above, checked
+both symbolically and numerically before simplifying the implementation to
+share one `corrected_log` helper across every model rather than
+special-casing K80). K81/T92/TN93's gamma branches follow the same
+mechanical substitution but weren't independently checked against an
+`ape`-equivalent reference for those three specifically — flagged here, not
+silently assumed correct.
+
+### Output format: PHYLIP-relaxed by default for the distance matrix
+
+**Implemented.** The primary distance-matrix output
+(`_dist.csv` today) gains multiple formats: **PHYLIP-relaxed becomes the
+default** (widely read by external NJ tools — PHYLIP `neighbor`, FastME,
+T-REX, SplitsTree — relaxed rather than strict to avoid the 10-character
+label truncation, since genome labels here routinely exceed it), a `--csv`
+flag opts back into the current CSV format, PHYLIP-strict is a possible
+future addition (not now). This changes the *default* output of every
+existing `--distance` value (jaccard, hamming, bray-curtis, ...), not just
+the new `snp-*` ones — accepted explicitly (pre-release, single developer
+user, no external consumers to break). Scoped to the distance matrix only:
+`--shared-kmers` and `--raw-snp-counts` are counts, not distances, and keep
+their existing CSV-only format.
+
 ## References
 
 The Mash mutation-rate model this discussion contrasts with:
diff --git a/DevDocMD/theory/indexing.refs.md b/DevDocMD/theory/indexing.refs.md
index 444abffa..4eef9731 100644
--- a/DevDocMD/theory/indexing.refs.md
+++ b/DevDocMD/theory/indexing.refs.md
@@ -3,8 +3,8 @@
 
 ## Code couvert
 
-- `obikpartitionner/src/partition.rs` — routage par hash de minimiseur, choix des paramètres
-- `obikpartitionner/src/lib.rs` — structure KmerPartition, nombre de partitions
+- `obikindex/src/partition/partition.rs` — routage par hash de minimiseur, choix des paramètres
+- `obikindex/src/partition/mod.rs` — structure KmerPartition, nombre de partitions
 
 ## Notes
 
diff --git a/Le_bug_des_A.md b/Le_bug_des_A.md
deleted file mode 100644
index 3f25b9c2..00000000
--- a/Le_bug_des_A.md
+++ /dev/null
@@ -1,43 +0,0 @@
-Voici la version corrigée :
-
----
-
-**Bug** : dans `base_pair_tally`, toutes les transitions/comptes depuis/vers A valent 0 dans `_sankoff_params.yaml`, alors que C/G/T sont corrects.
-
-**Contexte** : obikmer, pipeline phylogénétique `--sankoff`. L’index est construit sur 20 génomes bactériens. Même symptôme sur un jeu de 100 génomes de plantes : A est toujours à 0.
-
-**Fichier clé** : `src/obikphylo/src/siblings/sankoff_bundle.rs` (Pass A + Pass B).
-
-**Ce qui a été vérifié** :
-- Le fichier de sortie `_sankoff_params.yaml` montre bien `composition_transitions` avec A à 0 partout.
-- L’index contient bien des familles avec A (`mask.has(0) == true`), et même des familles où A co-existe avec d’autres bases (`mask == 0b0011` par ex.).
-- Un k-mer propriétaire de famille avec `mask == 0b0001` (A seul) a été identifié : forward `GAACAAGAGATCTCGATCTTGTCTACAAGGA`, revcomp `TCCTTGTAGACAAGATCGAGATCTCTTGTTC`.
-- Le diagnostic CLI sur l’index réel donne :
-  - Pass A : `a_pairs=623342 a_snp=623342 a_shared=0 a_both_a=0`
-  - Pass B : `families_with_a=22965521 a_single_form_genomes=22913238 a_included_pairs=0 a_same_incremented=0 bp_same=[0, 96389, 222720, 277909] bp_counts[0]=[0, 0, 0, 0]`
-
-**Interprétation** : A est fréquemment en `single_form` (mask == 1) chez certains génomes, mais **jamais simultanément** chez deux génomes différents dans la même famille. Donc toutes les paires “avec A” sont 100% SNP → ratio = 1.0 > `ratio_ceiling=0.5` → toutes exclues par le filtre `included`. C’est pourquoi `bp_same[0]` et `bp_counts[0][*]` restent à 0.
-
-**Point crucial** : le bug n’apparaît **que sur l’index compacté sparse**. Sur le même index avant compaction (matrice dense `matrix.pbmx`), `--sankoff` produit des tallies corrects pour A. Dès qu’on compacte avec `pack --sparse`, A disparaît.
-
-**Vérifications supplémentaires (diagnostic sparse)** :
-- La compaction `pack --sparse` produit une matrice `PersistentSparseBitMatrix` dont le contenu est **strictement identique** à la matrice dense d'origine : vérification exhaustive coordonnée par coordonnée sur **1 804 774 880 cellules** (512 partitions × 2 layers), **zéro différence**.
-- `fill_row` et `fill_sub_matrix` (les deux chemins de lecture utilisés par le pipeline phylogénétique) restituent les mêmes bits sur dense et sparse.
-- **Conclusion** : le bug n'est **pas** dans la compaction sparse elle-même, ni dans les chemins de lecture individuels. La structure stocke correctement A, C, G, T.
-
-**Conséquence logique** :
-Si les matrices sont identiques mais que le résultat final diffère, le bug se situe dans l'**intersection** des informations — c'est-à-dire dans le code qui **combine** les lectures des deux matrices (ou qui transforme les résultats bruts en tallies). Deux endroits possibles :
-1. **Le scan `sankoff_bundle`** (`family_scan.rs` + `sankoff_bundle.rs`) : la boucle qui lit les matrices, construit `genome_mask`, et accumule `bp_counts` / `same`. C'est l'étape d'intersection proprement dite.
-2. **La conversion des tallies en YAML** (`obikmer/src/cmd/phylo/sankoff.rs`) : moins probable, mais possible si quelque chose sélectionne/filtre les transitions avant écriture.
-
-**Hypothèse la plus probable** : bug dans la résolution cross-partition lors de la construction de l'annex sibling (`build_sibling_annex`). A (bit 0) serait systématiquement manquant ou mal résolu quand on interroge les variants d'une famille depuis une partition différente. À vérifier dans `src/obikphylo/src/siblings/build.rs` et `src/obikphylo/src/siblings/cache.rs` (`PartitionCache::find` / `find_presence_batch`).
-
-**Prochaine étape logique** :
-1. Inspecter `build_sibling_annex` pour voir si les variants avec base A sont bien générés et bien recherchés dans `cache.find`.
-2. Vérifier `PartitionCache::find` et `resolve_layer_hits` pour un éventuel biais contre le bit 0.
-3. Si besoin, ajouter un diagnostic ciblé (compteurs par base) **uniquement** dans `cache.rs` ou `build.rs`, pas dans `sankoff_bundle.rs` qui est déjà propre.
-
-**Contraintes** :
-- Ne pas modifier `sankoff_bundle.rs` davantage.
-- Ne pas toucher à git.
-- Faire des diagnostics minimaux et ciblés.
diff --git a/UserDocMD/architecture.md b/UserDocMD/architecture.md
index 7c1da714..a3da8b95 100644
--- a/UserDocMD/architecture.md
+++ b/UserDocMD/architecture.md
@@ -20,7 +20,7 @@ An index directory is organized as `KmerIndex → partitions → layers`, with a
 
 ## Parallel execution and NUMA awareness
 
-Partition-level work (index construction, `merge`, `filter`, `reindex`, `select`, `phylo`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc:
+Partition-level work (index construction, `merge`, `filter`, `convert`, `select`, `phylo`'s sibling-annex/Sankoff computations) is dispatched by a partition runner that adapts to the machine's memory topology, detected automatically at startup via hwloc:
 
 - On a multi-socket / multi-NUMA-node machine, one thread pool is pinned per NUMA node, and each partition is processed entirely by threads pinned to one node — keeping the memory a partition touches local to that node's DRAM. This matters because touching kmer data across NUMA nodes without pinning can degrade throughput by an order of magnitude or more on large multi-socket machines.
 - On a single-socket machine, Apple Silicon, or if hwloc cannot report NUMA topology, all cores are treated as one node with no pinning and negligible overhead — this is the default behavior on macOS.
diff --git a/UserDocMD/formats/index_layout.md b/UserDocMD/formats/index_layout.md
index 09633b09..b590364f 100644
--- a/UserDocMD/formats/index_layout.md
+++ b/UserDocMD/formats/index_layout.md
@@ -19,7 +19,7 @@ Each partition's surviving kmers are mapped to a dense range of integer slots by
 
 ## Evidence: exact vs. approximate
 
-Two verification modes are available, selected at build time (`index --approx`) and convertible afterwards ([`reindex`](../usage/reindex.md)):
+Two verification modes are available, selected at build time (`index --approx`) and convertible afterwards ([`convert`](../usage/convert.md)):
 
 - **Exact** (default): the hashed slot stores a pointer back into the partition's unitig data. At query time the kmer is reconstructed from that location and compared directly to the query. Zero false positives, at the cost of one extra random read per lookup.
 - **Approximate** (`--approx`): the slot stores a short fingerprint (`--evidence-bits` bits) instead of a pointer; verification is a single fingerprint comparison. This trades a small, bounded false-positive rate ($1/2^b$ per kmer, reduced further to about $1/2^{b \cdot z}$ for a read requiring $z$ consecutive matching kmers via the `-z`/`--findere-z` parameter) for lower memory and disk usage, since no reconstruction index is needed. See [`estimate`](../usage/estimate.md) to explore this trade-off before building.
@@ -51,6 +51,6 @@ Two verification modes are available, selected at build time (`index --approx`)
 
 A **layer** corresponds to one increment of kmer content added to a partition — most commonly, one [`merge`](../usage/merge.md) operation that introduces kmers not already present in the index. Genomes already present in the index simply gain new columns in the existing layers' count/presence data; only genuinely new kmer content is assembled into a new layer. Because of this, merging cost scales with the novel kmer content being added, not with the accumulated size of the index. A query against an index with several layers checks each layer's MPHF in turn.
 
-Sources merged together must share the same kmer size, minimizer size, partition count, and evidence mode (including matching approximate-mode parameters); mismatches are rejected rather than silently reconciled — [`reindex`](../usage/reindex.md) one of the sources first if needed.
+Sources merged together must share the same kmer size, minimizer size, partition count, and evidence mode (including matching approximate-mode parameters); mismatches are rejected rather than silently reconciled — [`convert`](../usage/convert.md) one of the sources first if needed.
 
 `obikmer pack` consolidates a partition's per-column files (counts/presence) into a single file, reducing the number of file opens needed at query time.
diff --git a/UserDocMD/index.md b/UserDocMD/index.md
index 10aa2c2e..aa1fcc7b 100644
--- a/UserDocMD/index.md
+++ b/UserDocMD/index.md
@@ -24,11 +24,10 @@ All functionality is exposed through a single binary, `obikmer`, organized as su
 | [`query`](usage/query.md) | Query an index with sequences and annotate matches |
 | [`dump`](usage/dump.md) | Dump indexed kmers as CSV |
 | [`annotate`](usage/annotate.md) | Add, update, or dump genome metadata |
-| [`phylo`](usage/phylo.md) | Compute pairwise evolutionary-distance proxies, trees, and phylogenetic exports |
-| [`name-tree`](usage/name-tree.md) | Translate a TNT/PhyG numeric-label tree export back to real taxon names |
+| [`phylo`](usage/phylo.md) | Compute pairwise genome distances, trees, and phylogenetic exports |
 | [`unitig`](usage/unitig.md) | Dump the unitigs of an index as FASTA |
 | [`estimate`](usage/estimate.md) | Estimate approximate-index parameters before indexing |
-| [`reindex`](usage/reindex.md) | Convert an index's evidence representation (exact ↔ approximate) |
+| [`convert`](usage/convert.md) | Convert an index's evidence representation (exact/approximate/hybrid), in place |
 | [`utils`](usage/utils.md) | Miscellaneous index maintenance and inspection utilities |
 | [`pack`](usage/pack.md) | Pack per-column matrix files into a single-file format |
 
diff --git a/UserDocMD/usage/convert.md b/UserDocMD/usage/convert.md
new file mode 100644
index 00000000..f2a2b44a
--- /dev/null
+++ b/UserDocMD/usage/convert.md
@@ -0,0 +1,29 @@
+# convert
+
+Convert an existing index's evidence representation in place, between exact, approximate, and hybrid.
+
+```bash
+obikmer convert INDEX (--exact-evidence | --approx-evidence BITS | --hybrid-evidence) [OPTIONS]
+```
+
+## Arguments
+
+| Argument | Description |
+|---|---|
+| `INDEX` | Index directory to convert (modified in place) |
+
+## Options
+
+Exactly one of the first three is required:
+
+| Option | Description |
+|---|---|
+| `--exact-evidence` | Convert to exact evidence (zero false positives) |
+| `--approx-evidence BITS` | Convert to approximate (fingerprint-only) evidence; `BITS` = fingerprint bits per slot (b) |
+| `--hybrid-evidence` | Convert to hybrid evidence (both exact and approximate bundles kept) |
+| `--evidence-bits BITS` | Fingerprint bits per slot (b) — required with `--hybrid-evidence` when the source index is currently exact; rejected otherwise (the source already fixes `b`) |
+| `-z, --findere-z Z` | Findere z parameter: number of consecutive stored kmers that must all match to confirm a hit. This does not shorten the indexed kmer length (fixed forever at `index` build time) — it extends the effective match window: on a k=31 index, `z=2` requires 32 consecutive matching bases, not 30 |
+| `--fp FP` | Target false-positive rate per z-window (e.g. `0.01`); derives `b` or `z` when one of them isn't given directly |
+| `--block-size N` | Block size for exact evidence's on-disk index (unitigs per block). Ignored when converting to pure approximate evidence. Default `1` |
+
+See [`index`](index_command.md#exact-vs-approximate-evidence) for the exact/approximate trade-off and the underlying false-positive model, and [`estimate`](estimate.md) to explore parameters beforehand. The index directory is locked for exclusive access during conversion.
diff --git a/UserDocMD/usage/estimate.md b/UserDocMD/usage/estimate.md
index 64ee85a1..a8f326fb 100644
--- a/UserDocMD/usage/estimate.md
+++ b/UserDocMD/usage/estimate.md
@@ -15,4 +15,4 @@ obikmer estimate [OPTIONS]
 | `--evidence-bits` | none | Fingerprint bits per slot (b) |
 | `--fp` | none | Target false-positive rate per z-window |
 
-Any two of `-z`, `--evidence-bits`, `--fp` may be given; the third is derived using the same model as `index --approx` and `reindex --approx` ($FP = 1 / 2^{b \cdot z}$). The report printed to stdout includes: query $k$, effective indexed $k$ ($k-z+1$), $z$, evidence bits, per-kmer false-positive rate, and per-z-window false-positive rate.
+Any two of `-z`, `--evidence-bits`, `--fp` may be given; the third is derived using the same model as `index --approx` and `convert --approx-evidence` ($FP = 1 / 2^{b \cdot z}$). The report printed to stdout includes: query $k$, effective indexed $k$ ($k-z+1$), $z$, evidence bits, per-kmer false-positive rate, and per-z-window false-positive rate.
diff --git a/UserDocMD/usage/index_command.md b/UserDocMD/usage/index_command.md
index d887e1b6..5c1861a7 100644
--- a/UserDocMD/usage/index_command.md
+++ b/UserDocMD/usage/index_command.md
@@ -45,6 +45,6 @@ With `--approx`, evidence is stored as a compact **fingerprint** instead, tradin
 
 $$FP = \frac{1}{2^{b \cdot z}}$$
 
-where $b$ is `--evidence-bits` and $z$ is `--findere-z`. Any two of `-z`, `--evidence-bits`, `--fp` can be given and the third is derived; if none are given, defaults are $b=8$, $z=1$ ($FP \approx 1/256$). See [`estimate`](estimate.md) to explore this trade-off before building an index, and [`reindex`](reindex.md) to convert an existing index between the two representations.
+where $b$ is `--evidence-bits` and $z$ is `--findere-z`. Any two of `-z`, `--evidence-bits`, `--fp` can be given and the third is derived; if none are given, defaults are $b=8$, $z=1$ ($FP \approx 1/256$). See [`estimate`](estimate.md) to explore this trade-off before building an index, and [`convert`](convert.md) to change an existing index's representation afterwards.
 
 `z` must be strictly less than k: the effective indexed kmer length under approximate evidence is k−z+1.
diff --git a/UserDocMD/usage/name-tree.md b/UserDocMD/usage/name-tree.md
deleted file mode 100644
index 70f2943c..00000000
--- a/UserDocMD/usage/name-tree.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# name-tree
-
-Translate a numerically-labelled tree export (TNT, PhyG, or any plain Newick file with bare `1`, `2`, `3`, … leaf labels) back to real taxon names, reading the label order from the FASTA that produced it.
-
-```bash
-obikmer name-tree TREE --fasta FASTA -o OUTPUT
-```
-
-## Arguments
-
-| Argument | Description |
-|---|---|
-| `TREE` | Tree file to translate — a TNT-style NEXUS export (`tree NAME = [&U] ...;`) or a plain Newick file |
-| `--fasta` | FASTA file whose record order gives the numeric taxon labels (1-based) — typically the `_sankoff.fasta`/`_snp.fasta` used to produce `TREE` |
-| `-o, --output` | Output NEXUS file path |
-
-## Output
-
-A NEXUS file with a `taxa` block, a `translate` table (numeric label → taxon name, from `--fasta`'s header order), and every tree found in `TREE`, topology unchanged — readable directly in FigTree, PearTree, `ape` (R), etc.
-
-`--tnt`'s and `--phyg`'s exports (see [phylo](phylo.md)) both number taxa `1..N` in the same order as the pseudo-alignment FASTA they were built from (`_sankoff.fasta`), so pass that same file as `--fasta` here.
diff --git a/UserDocMD/usage/pack.md b/UserDocMD/usage/pack.md
index 285aea9c..cdffa707 100644
--- a/UserDocMD/usage/pack.md
+++ b/UserDocMD/usage/pack.md
@@ -16,14 +16,14 @@ obikmer pack INDEX [--sparse]
 
 | Option | Default | Description |
 |---|---|---|
-| `--sparse` | off | Pack presence/absence matrices into a sparse, deduplicated format instead of the dense one |
+| `--sparse` | off | Pack presence/absence and count matrices into a sparse, deduplicated format instead of the dense one |
 
 The index directory is locked for exclusive access while packing.
 
 ## `--sparse`
 
-Presence/absence data (which genomes carry each kmer) is often mostly empty — most kmers are present in only a handful of genomes out of the whole collection. The default (dense) packed format stores one bit per genome for every kmer regardless of how many genomes actually carry it; `--sparse` instead stores each kmer's genome list directly, and deduplicates identical lists shared by many kmers (common in real data, since kmers from the same conserved region tend to be carried by the same genomes).
+Matrix data (which genomes carry each kmer, or with what count) is often mostly empty — most kmers are present in only a handful of genomes out of the whole collection. The default (dense) packed format stores one entry per genome for every kmer regardless of how many genomes actually carry it; `--sparse` instead stores each kmer's genome list directly. For presence/absence matrices, identical genome lists shared by many kmers are also deduplicated (common in real data, since kmers from the same conserved region tend to be carried by the same genomes); for count matrices, the genome list is deduplicated the same way but each kmer's actual counts are kept per-kmer, since two kmers sharing the same genome list rarely carry the same counts.
 
-On real genome collections this has measured at roughly 7x smaller on disk than the dense format, and single-kmer lookups (the shape `phylo`'s sibling-annex/entropy/Sankoff computations use) are typically faster too, since the smaller files mean less data to read from disk. The trade-off: reading a whole genome column at once (used by `--metric` distance-matrix computations) is much slower on the sparse format than on the dense one, since there is no native column layout to read sequentially — prefer the dense format (the default, no `--sparse`) for indexes you mainly query with `phylo`'s plain `--metric` distance matrices.
+On real genome collections this has measured at roughly 7x smaller on disk than the dense format for presence/absence, and single-kmer lookups (the shape `phylo`'s sibling-annex/entropy/Sankoff computations use) are typically faster too, since the smaller files mean less data to read from disk. The trade-off: reading a whole genome column at once (used by `--distance` matrix computations) is much slower on the sparse format than on the dense one, since there is no native column layout to read sequentially — prefer the dense format (the default, no `--sparse`) for indexes you mainly query with `phylo`'s `--distance` matrices.
 
-Count matrices (`--metric` on a count index) are not affected by `--sparse` — only presence/absence matrices are.
+`--sparse` applies to both presence/absence and count matrices — a count index (`--distance` matrix computations included) is packed sparse the same as a presence index.
diff --git a/UserDocMD/usage/phylo.md b/UserDocMD/usage/phylo.md
index 7da90cc9..f933ddd4 100644
--- a/UserDocMD/usage/phylo.md
+++ b/UserDocMD/usage/phylo.md
@@ -1,6 +1,6 @@
 # phylo
 
-Compute pairwise evolutionary-distance proxies between the genomes stored in an index — a plain distance matrix, optionally trees (NJ/UPGMA), and optionally a central-position SNP model with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE).
+Compute pairwise distances between the genomes stored in an index, optionally build trees (NJ/UPGMA) from them, and optionally calibrate a 16-state parsimony model for a central-position SNP character with exports for external phylogenetic tools (TNT, PhyG, IQ-TREE).
 
 ```bash
 obikmer phylo INDEX [OPTIONS]
@@ -12,133 +12,217 @@ obikmer phylo INDEX [OPTIONS]
 |---|---|
 | `INDEX` | Index directory |
 
-## Distance matrix
+## Distance matrix (`--distance`)
 
 | Option | Default | Description |
 |---|---|---|
-| `--metric` | `jaccard` | One of `jaccard`, `mash`, `hamming`, `bray-curtis`, `relfreq-bray-curtis`, `euclidean`, `relfreq-euclidean`, `hellinger`, `hellinger-euclidean` |
-| `--presence-threshold` | `1` | Minimum count for a kmer to be considered present, for Jaccard/Mash on a count index |
-| `--shared-kmers` | off | Also write the shared-kmer count matrix |
+| `--distance` | `jaccard` | See the two tables below for the full list of accepted values |
+| `--gamma-shape ALPHA` | none | Rate-heterogeneity correction, for `snp-*` values that support it (see below). No effect on the other values; rejected if given together with a value that doesn't support it |
+| `--presence-threshold` | `1` | Minimum count for a kmer to be considered present, for `jaccard`/`mash` on a count index |
+| `--csv` | off | Write the matrix as plain CSV instead of the default relaxed-PHYLIP format |
+| `--shared-kmers` | off | Also write the shared-kmer count matrix. Only valid with a whole-index metric, not a `snp-*` value |
 | `--nj` | off | Compute and write a Neighbor-Joining tree (Newick) |
 | `--upgma` | off | Compute and write a UPGMA tree (Newick) |
-| `-o, --output` | none (stdout) | Output file prefix; without it, the distance matrix is printed to stdout as CSV |
+| `-o, --output` | none (stdout) | Output file prefix |
 
-`hamming` requires a presence/absence index. All other metrics work on either index type; on a presence index, `jaccard`/`mash`/`hamming` are the only ones available.
+Every value routes to one of two independent computations:
 
-### Metric definitions
+### Whole-index metrics
 
-- **jaccard**: $D = 1 - \dfrac{|A \cap B|}{|A \cup B|}$ over the sets of kmers present in each genome.
-- **mash**: derived from the Jaccard distance via $D = -\dfrac{1}{k} \ln\!\left(\dfrac{2J}{1+J}\right)$ where $J = 1 - D_{\text{jaccard}}$ and $k$ is the index's kmer size; clamped to 1.0 when $J \le 0$.
-- **hamming**: number of kmer positions where presence differs between the two genomes (presence index only, not normalized): $D = \sum_i \mathbb{1}[a_i \ne b_i]$.
-- **bray-curtis**: $D = 1 - \dfrac{2 \sum_i \min(c_i^A, c_i^B)}{\sum_i c_i^A + \sum_i c_i^B}$ on raw per-kmer counts.
-- **relfreq-bray-curtis**: the same formula computed on per-genome relative frequencies $p_i = c_i / \sum_j c_j$ instead of raw counts.
-- **euclidean**: $D = \sqrt{\sum_i (c_i^A - c_i^B)^2}$ on raw counts.
-- **relfreq-euclidean**: the same formula on relative frequencies.
-- **hellinger**: $D = \dfrac{1}{\sqrt{2}} \sqrt{\sum_i \left(\sqrt{p_i^A} - \sqrt{p_i^B}\right)^2}$ on relative frequencies, bounded in $[0, 1]$.
-- **hellinger-euclidean**: the unnormalized variant, $D = \sqrt{2} \times D_{\text{hellinger}}$.
+| Value | Definition |
+|---|---|
+| `jaccard` | $D = 1 - \dfrac{\lvert A \cap B \rvert}{\lvert A \cup B \rvert}$ over the sets of kmers present in each genome |
+| `mash` | derived from the Jaccard distance via $D = -\dfrac{1}{k} \ln\!\left(\dfrac{2J}{1+J}\right)$ where $J = 1 - D_{\text{jaccard}}$ and $k$ is the index's kmer size; clamped to 1.0 when $J \le 0$ |
+| `hamming` | number of kmer positions where presence differs between the two genomes (presence index only, not normalized): $D = \sum_i \mathbb{1}[a_i \ne b_i]$ |
+| `bray-curtis` | $D = 1 - \dfrac{2 \sum_i \min(c_i^A, c_i^B)}{\sum_i c_i^A + \sum_i c_i^B}$ on raw per-kmer counts |
+| `relfreq-bray-curtis` | the same formula computed on per-genome relative frequencies $p_i = c_i / \sum_j c_j$ instead of raw counts |
+| `euclidean` | $D = \sqrt{\sum_i (c_i^A - c_i^B)^2}$ on raw counts |
+| `relfreq-euclidean` | the same formula on relative frequencies |
+| `hellinger` | $D = \dfrac{1}{\sqrt{2}} \sqrt{\sum_i \left(\sqrt{p_i^A} - \sqrt{p_i^B}\right)^2}$ on relative frequencies, bounded in $[0, 1]$ |
+| `hellinger-euclidean` | the unnormalized variant, $D = \sqrt{2} \times D_{\text{hellinger}}$ |
+
+`hamming` requires a presence/absence index; the others work on either index type.
+
+### `snp-*` corrections
+
+Computed from the central-position SNP model (see "Central-position SNP model" below): a family is the set of up to 4 kmers sharing identical flanking sequence and differing only at the central base. These values require the sibling annex (`--sibling-annex`, below) and are, by default, computed exhaustively over every non-monomorphic family in the index; add `--subsample N` to bound the computation to approximately `N` families instead (see "Sampling at scale" below — the same flag `--pseudo-alignment`/`--sankoff` use, but optional here).
+
+For a genome pair, let $L$ be its total number of eligible loci (both genomes single-copy at that family), $p$ the raw proportion of substitutions among those loci, $P$/$Q$ the transition/transversion proportions, $Q_1$/$Q_2$ Kimura's two transversion categories (A↔C & G↔T vs. A↔T & C↔G), $P_1$/$P_2$ the purine (A↔G) / pyrimidine (C↔T) transition proportions, and $\pi_A,\pi_C,\pi_G,\pi_T$ the pair's pooled base frequencies.
+
+**`snp-raw`**
+
+$$d = p$$
+
+**`snp-jc`**
+
+$$d = -\frac{3}{4}\ln\!\left(1-\frac{4p}{3}\right)$$
+
+**`snp-k2p`**
+
+$$
+\begin{aligned}
+a_1 &= 1-2P-Q \\
+a_2 &= 1-2Q \\
+d   &= -\frac{1}{2}\ln a_1-\frac{1}{4}\ln a_2
+\end{aligned}
+$$
+
+**`snp-k81`**
+
+$$
+\begin{aligned}
+a_1 &= 1-2P-2Q_1 \\
+a_2 &= 1-2P-2Q_2 \\
+a_3 &= 1-2Q_1-2Q_2 \\
+d   &= -\frac{1}{4}\left(\ln a_1+\ln a_2+\ln a_3\right)
+\end{aligned}
+$$
+
+**`snp-f81`**
+
+$$
+\begin{aligned}
+E &= 1-\left(\pi_A^2+\pi_C^2+\pi_G^2+\pi_T^2\right) \\
+d &= -E\ln\!\left(1-\frac{p}{E}\right)
+\end{aligned}
+$$
+
+**`snp-t92`**
+
+$$
+\begin{aligned}
+g   &= \pi_C+\pi_G \\
+w   &= 2g(1-g) \\
+a_1 &= 1-\frac{P}{w}-Q \\
+a_2 &= 1-2Q \\
+d   &= -w\ln a_1-\frac{1}{2}(1-w)\ln a_2
+\end{aligned}
+$$
+
+**`snp-tn93`**
+
+$$
+\begin{aligned}
+g_R &= \pi_A+\pi_G \\
+g_Y &= \pi_C+\pi_T \\
+k_1 &= \frac{2\pi_A\pi_G}{g_R} \\
+k_2 &= \frac{2\pi_C\pi_T}{g_Y} \\
+k_3 &= 2\left(g_Rg_Y-\frac{\pi_A\pi_G\,g_Y}{g_R}-\frac{\pi_C\pi_T\,g_R}{g_Y}\right) \\
+w_1 &= 1-\frac{P_1}{k_1}-\frac{Q}{2g_R} \\
+w_2 &= 1-\frac{P_2}{k_2}-\frac{Q}{2g_Y} \\
+w_3 &= 1-\frac{Q}{2g_Rg_Y} \\
+d   &= -k_1\ln w_1-k_2\ln w_2-k_3\ln w_3
+\end{aligned}
+$$
+
+**`snp-tv`** — transversions only, deliberately uncorrected:
+
+$$d = Q$$
+
+`--gamma-shape ALPHA` applies to every value above except `snp-raw` and `snp-tv`: each $-\ln(x)$ term in the formulas above is replaced by $\alpha\left(x^{-1/\alpha}-1\right)$ (the same weight, same $x$).
+
+### Output
+
+Without `-o`, the matrix goes to stdout in relaxed-PHYLIP format (`n` on the first line, then one `labelvalue...` row per genome). With `--csv`, the format is instead a header row `genome,,,...` followed by one `

      unitigs.bin is the only file from which the indexed kmer content can be fully recovered; it is always retained. Every other file (MPHF, evidence, counts) is derived from it.

      A layer corresponds to one increment of kmer content added to a partition — most commonly, one merge operation that introduces kmers not already present in the index. Genomes already present in the index simply gain new columns in the existing layers' count/presence data; only genuinely new kmer content is assembled into a new layer. Because of this, merging cost scales with the novel kmer content being added, not with the accumulated size of the index. A query against an index with several layers checks each layer's MPHF in turn.

      -

      Sources merged together must share the same kmer size, minimizer size, partition count, and evidence mode (including matching approximate-mode parameters); mismatches are rejected rather than silently reconciled — reindex one of the sources first if needed.

      +

      Sources merged together must share the same kmer size, minimizer size, partition count, and evidence mode (including matching approximate-mode parameters); mismatches are rejected rather than silently reconciled — convert one of the sources first if needed.

      obikmer pack consolidates a partition's per-column files (counts/presence) into a single file, reducing the number of file opens needed at query time.

      diff --git a/doc/index.html b/doc/index.html index 37780eae..01928602 100644 --- a/doc/index.html +++ b/doc/index.html @@ -813,34 +813,6 @@ -
    98. - - - - - - - - name-tree - - - - - - - - -
      99. - - - - - - - - - -
    99. @@ -898,14 +870,14 @@
    100. - + - reindex + convert @@ -1282,11 +1254,7 @@ phylo -Compute pairwise evolutionary-distance proxies, trees, and phylogenetic exports - - -name-tree -Translate a TNT/PhyG numeric-label tree export back to real taxon names +Compute pairwise genome distances, trees, and phylogenetic exports unitig @@ -1297,8 +1265,8 @@ Estimate approximate-index parameters before indexing -reindex -Convert an index's evidence representation (exact ↔ approximate) +convert +Convert an index's evidence representation (exact/approximate/hybrid), in place utils diff --git a/doc/installation/index.html b/doc/installation/index.html index 23d10c38..f4f93c3b 100644 --- a/doc/installation/index.html +++ b/doc/installation/index.html @@ -888,34 +888,6 @@ -
    101. - - - - - - - - name-tree - - - - - - - - -
      102. - - - - - - - - - -
    102. @@ -973,14 +945,14 @@
    103. - + - reindex + convert diff --git a/doc/sitemap.xml.gz b/doc/sitemap.xml.gz index d240d53a..03a78e93 100644 Binary files a/doc/sitemap.xml.gz and b/doc/sitemap.xml.gz differ diff --git a/doc/theory/encoding/index.html b/doc/theory/encoding/index.html index ec54413d..533eb4dd 100644 --- a/doc/theory/encoding/index.html +++ b/doc/theory/encoding/index.html @@ -784,34 +784,6 @@ -
    104. - - - - - - - - name-tree - - - - - - - - -
      105. - - - - - - - - - -
    105. @@ -869,14 +841,14 @@
    106. - + - reindex + convert diff --git a/doc/theory/entropy_filter/index.html b/doc/theory/entropy_filter/index.html index 3cf2dc94..bc0f568a 100644 --- a/doc/theory/entropy_filter/index.html +++ b/doc/theory/entropy_filter/index.html @@ -806,34 +806,6 @@ -
    107. - - - - - - - - name-tree - - - - - - - - -
      108. - - - - - - - - - -
    108. @@ -891,14 +863,14 @@
    109. - + - reindex + convert diff --git a/doc/theory/indexing_architecture/index.html b/doc/theory/indexing_architecture/index.html index 6618904a..b7fbba06 100644 --- a/doc/theory/indexing_architecture/index.html +++ b/doc/theory/indexing_architecture/index.html @@ -784,34 +784,6 @@ -
    110. - - - - - - - - name-tree - - - - - - - - -
      111. - - - - - - - - - -
    111. @@ -869,14 +841,14 @@
    112. - + - reindex + convert diff --git a/doc/theory/kmers_and_superkmers/index.html b/doc/theory/kmers_and_superkmers/index.html index 012e16e9..3f9aa0a6 100644 --- a/doc/theory/kmers_and_superkmers/index.html +++ b/doc/theory/kmers_and_superkmers/index.html @@ -790,34 +790,6 @@ -
    113. - - - - - - - - name-tree - - - - - - - - -
      114. - - - - - - - - - -
    114. @@ -875,14 +847,14 @@
    115. - + - reindex + convert diff --git a/doc/theory/minimizer_selection/index.html b/doc/theory/minimizer_selection/index.html index 07d02554..947cd9f8 100644 --- a/doc/theory/minimizer_selection/index.html +++ b/doc/theory/minimizer_selection/index.html @@ -801,34 +801,6 @@ -
    116. - - - - - - - - name-tree - - - - - - - - -
      117. - - - - - - - - - -
    117. @@ -886,14 +858,14 @@
    118. - + - reindex + convert diff --git a/doc/usage/annotate/index.html b/doc/usage/annotate/index.html index 9ba50eb4..e8f31b7e 100644 --- a/doc/usage/annotate/index.html +++ b/doc/usage/annotate/index.html @@ -773,34 +773,6 @@ -
    119. - - - - - - - - name-tree - - - - - - - - -
      120. - - - - - - - - - -
    120. @@ -858,14 +830,14 @@
    121. - + - reindex + convert diff --git a/doc/usage/reindex/index.html b/doc/usage/convert/index.html similarity index 90% rename from doc/usage/reindex/index.html rename to doc/usage/convert/index.html index 37bb4a39..c83e1aa1 100644 --- a/doc/usage/reindex/index.html +++ b/doc/usage/convert/index.html @@ -23,7 +23,7 @@ - reindex - obikmer — User Guide + convert - obikmer — User Guide @@ -64,7 +64,7 @@
      - + Skip to content @@ -100,7 +100,7 @@
      - reindex + convert
      @@ -704,34 +704,6 @@ -
    122. - - - - - - - - name-tree - - - - - - - - -
      123. - - - - - - - - - -
    123. @@ -805,7 +777,7 @@ - reindex + convert @@ -823,7 +795,7 @@ - reindex + convert @@ -1144,9 +1116,9 @@ -

      reindex

      -

      Convert an existing index's evidence representation in place, between exact and approximate.

      -
      obikmer reindex INDEX [OPTIONS]
+
      convert
      
+
      Convert an existing index's evidence representation in place, between exact, approximate, and hybrid.
      
+
      obikmer convert INDEX (--exact-evidence | --approx-evidence BITS | --hybrid-evidence) [OPTIONS]
 
      
 
      Arguments
      
 
@@ -1164,39 +1136,42 @@
 
      
 
      
 
      Options
      
+
      Exactly one of the first three is required:
      
 
-
-
-
-
+
+
-
-
-
+
+
-
-
-
+
+
-
-
-
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
      
 
 
      
 OptionDefault
 Description
 
      
       
 
 
      --approxoffConvert to approximate evidence (default direction is approximate → exact); requires -z/--evidence-bits/--fp--exact-evidenceConvert to exact evidence (zero false positives)
 
      
 
      -z, --findere-znoneFindere z parameter (≥ 1)--approx-evidence BITSConvert to approximate (fingerprint-only) evidence; BITS = fingerprint bits per slot (b)
 
      
 
      --evidence-bitsnoneFingerprint bits per slot (b)--hybrid-evidenceConvert to hybrid evidence (both exact and approximate bundles kept)
 
      
 
      --fpnoneTarget false-positive rate per z-window--evidence-bits BITSFingerprint bits per slot (b) — required with --hybrid-evidence when the source index is currently exact; rejected otherwise (the source already fixes b)
 
      
 
      --block-size1Block size for the exact on-disk index (ignored when converting to approximate)-z, --findere-z ZFindere z parameter: number of consecutive stored kmers that must all match to confirm a hit. This does not shorten the indexed kmer length (fixed forever at index build time) — it extends the effective match window: on a k=31 index, z=2 requires 32 consecutive matching bases, not 30
      --fp FPTarget false-positive rate per z-window (e.g. 0.01); derives b or z when one of them isn't given directly
      --block-size NBlock size for exact evidence's on-disk index (unitigs per block). Ignored when converting to pure approximate evidence. Default 1
 
      
       
 
      
diff --git a/doc/usage/dump/index.html b/doc/usage/dump/index.html
index 90a84d06..631611aa 100644
--- a/doc/usage/dump/index.html
+++ b/doc/usage/dump/index.html
@@ -773,34 +773,6 @@
   
   
   
-    
    124. 
-      
-        
-  
-  
-  
-    
-  
-    name-tree
-  
-
-    
-  
-  
-  
-
-      
-    
      125. 
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
     
    125. 
       
         
@@ -858,14 +830,14 @@
   
   
     
    126. 
-      
+      
         
   
   
   
     
   
-    reindex
+    convert
   
 
     
diff --git a/doc/usage/estimate/index.html b/doc/usage/estimate/index.html
index 90637eec..849c5939 100644
--- a/doc/usage/estimate/index.html
+++ b/doc/usage/estimate/index.html
@@ -12,7 +12,7 @@
         
       
       
-        
+        
       
       
         
@@ -704,34 +704,6 @@
   
   
   
-    
    127. 
-      
-        
-  
-  
-  
-    
-  
-    name-tree
-  
-
-    
-  
-  
-  
-
-      
-    
      128. 
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
     
    128. 
       
         
@@ -847,14 +819,14 @@
   
   
     
    129. 
-      
+      
         
   
   
   
     
   
-    reindex
+    convert
   
 
     
@@ -1158,7 +1130,7 @@
 
 
 
-
      Any two of -z, --evidence-bits, --fp may be given; the third is derived using the same model as index --approx and reindex --approx (\(FP = 1 / 2^{b \cdot z}\)). The report printed to stdout includes: query \(k\), effective indexed \(k\) (\(k-z+1\)), \(z\), evidence bits, per-kmer false-positive rate, and per-z-window false-positive rate.
      
+
      Any two of -z, --evidence-bits, --fp may be given; the third is derived using the same model as index --approx and convert --approx-evidence (\(FP = 1 / 2^{b \cdot z}\)). The report printed to stdout includes: query \(k\), effective indexed \(k\) (\(k-z+1\)), \(z\), evidence bits, per-kmer false-positive rate, and per-z-window false-positive rate.
      
 
 
 
diff --git a/doc/usage/filter/index.html b/doc/usage/filter/index.html
index 7e278c2d..438b5ace 100644
--- a/doc/usage/filter/index.html
+++ b/doc/usage/filter/index.html
@@ -784,34 +784,6 @@
   
   
   
-    
    130. 
-      
-        
-  
-  
-  
-    
-  
-    name-tree
-  
-
-    
-  
-  
-  
-
-      
-    
      131. 
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
     
    131. 
       
         
@@ -869,14 +841,14 @@
   
   
     
    132. 
-      
+      
         
   
   
   
     
   
-    reindex
+    convert
   
 
     
diff --git a/doc/usage/index_command/index.html b/doc/usage/index_command/index.html
index e21cf24e..4d339687 100644
--- a/doc/usage/index_command/index.html
+++ b/doc/usage/index_command/index.html
@@ -784,34 +784,6 @@
   
   
   
-    
    133. 
-      
-        
-  
-  
-  
-    
-  
-    name-tree
-  
-
-    
-  
-  
-  
-
-      
-    
      134. 
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
     
    134. 
       
         
@@ -869,14 +841,14 @@
   
   
     
    135. 
-      
+      
         
   
   
   
     
   
-    reindex
+    convert
   
 
     
@@ -1301,7 +1273,7 @@
 
      By default, an index stores exact evidence: a kmer is either present or absent (or has an exact count with --with-counts), with no false positives.
      
 
      With --approx, evidence is stored as a compact fingerprint instead, trading a small, tunable false-positive rate for reduced memory/disk usage. The false-positive model is:
      
 
      \[FP = \frac{1}{2^{b \cdot z}}\]
      
-
      where \(b\) is --evidence-bits and \(z\) is --findere-z. Any two of -z, --evidence-bits, --fp can be given and the third is derived; if none are given, defaults are \(b=8\), \(z=1\) (\(FP \approx 1/256\)). See estimate to explore this trade-off before building an index, and reindex to convert an existing index between the two representations.
      
+
      where \(b\) is --evidence-bits and \(z\) is --findere-z. Any two of -z, --evidence-bits, --fp can be given and the third is derived; if none are given, defaults are \(b=8\), \(z=1\) (\(FP \approx 1/256\)). See estimate to explore this trade-off before building an index, and convert to change an existing index's representation afterwards.
      
 
      z must be strictly less than k: the effective indexed kmer length under approximate evidence is k−z+1.
      
 
 
diff --git a/doc/usage/merge/index.html b/doc/usage/merge/index.html
index cee175c1..61006d48 100644
--- a/doc/usage/merge/index.html
+++ b/doc/usage/merge/index.html
@@ -784,34 +784,6 @@
   
   
   
-    
    136. 
-      
-        
-  
-  
-  
-    
-  
-    name-tree
-  
-
-    
-  
-  
-  
-
-      
-    
      137. 
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
     
    137. 
       
         
@@ -869,14 +841,14 @@
   
   
     
    138. 
-      
+      
         
   
   
   
     
   
-    reindex
+    convert
   
 
     
diff --git a/doc/usage/pack/index.html b/doc/usage/pack/index.html
index e7dfbafe..0db80671 100644
--- a/doc/usage/pack/index.html
+++ b/doc/usage/pack/index.html
@@ -704,34 +704,6 @@
   
   
   
-    
    139. 
-      
-        
-  
-  
-  
-    
-  
-    name-tree
-  
-
-    
-  
-  
-  
-
-      
-    
      140. 
-  
-
-              
-            
-              
-                
-  
-  
-  
-  
     
    140. 
       
         
@@ -789,14 +761,14 @@
   
   
     
    141. 
-      
+      
         
   
   
   
     
   
-    reindex
+    convert
   
 
     
@@ -1198,15 +1170,15 @@
 
 --sparse
 off
-Pack presence/absence matrices into a sparse, deduplicated format instead of the dense one
+Pack presence/absence and count matrices into a sparse, deduplicated format instead of the dense one
 
 
 
 
      The index directory is locked for exclusive access while packing.
      
 
      --sparse
      
-
      Presence/absence data (which genomes carry each kmer) is often mostly empty — most kmers are present in only a handful of genomes out of the whole collection. The default (dense) packed format stores one bit per genome for every kmer regardless of how many genomes actually carry it; --sparse instead stores each kmer's genome list directly, and deduplicates identical lists shared by many kmers (common in real data, since kmers from the same conserved region tend to be carried by the same genomes).
      
-
      On real genome collections this has measured at roughly 7x smaller on disk than the dense format, and single-kmer lookups (the shape phylo's sibling-annex/entropy/Sankoff computations use) are typically faster too, since the smaller files mean less data to read from disk. The trade-off: reading a whole genome column at once (used by --metric distance-matrix computations) is much slower on the sparse format than on the dense one, since there is no native column layout to read sequentially — prefer the dense format (the default, no --sparse) for indexes you mainly query with phylo's plain --metric distance matrices.
      
-
      Count matrices (--metric on a count index) are not affected by --sparse — only presence/absence matrices are.
      
+
      Matrix data (which genomes carry each kmer, or with what count) is often mostly empty — most kmers are present in only a handful of genomes out of the whole collection. The default (dense) packed format stores one entry per genome for every kmer regardless of how many genomes actually carry it; --sparse instead stores each kmer's genome list directly. For presence/absence matrices, identical genome lists shared by many kmers are also deduplicated (common in real data, since kmers from the same conserved region tend to be carried by the same genomes); for count matrices, the genome list is deduplicated the same way but each kmer's actual counts are kept per-kmer, since two kmers sharing the same genome list rarely carry the same counts.
      
+
      On real genome collections this has measured at roughly 7x smaller on disk than the dense format for presence/absence, and single-kmer lookups (the shape phylo's sibling-annex/entropy/Sankoff computations use) are typically faster too, since the smaller files mean less data to read from disk. The trade-off: reading a whole genome column at once (used by --distance matrix computations) is much slower on the sparse format than on the dense one, since there is no native column layout to read sequentially — prefer the dense format (the default, no --sparse) for indexes you mainly query with phylo's --distance matrices.
      
+
      --sparse applies to both presence/absence and count matrices — a count index (--distance matrix computations included) is packed sparse the same as a presence index.
      
 
 
 
diff --git a/doc/usage/phylo/index.html b/doc/usage/phylo/index.html
index c5488ce8..9d9c30dd 100644
--- a/doc/usage/phylo/index.html
+++ b/doc/usage/phylo/index.html
@@ -12,7 +12,7 @@
         
       
       
-        
+        
       
       
         
@@ -748,22 +748,44 @@
 
      142. 
       
         
    142. 
-  
+  
     
       
-        Distance matrix
+        Distance matrix (--distance)
       
     
   
   
-    
      143. 
+      
+        
    143. 
+  
+    
+      
+        --exclude-genome, --min-shared-family
+      
+    
+  
+  
 
      144. 
       
         
    144. 
@@ -788,10 +821,10 @@