diff --git a/.serena/project.yml b/.serena/project.yml index 1a35e2f9..f40f4750 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,39 +1,6 @@ -# the name by which the project can be referenced within Serena +# the name by which the project can be referenced within Serena/when chatting with the LLM. project_name: "obikmer" - -# list of languages for which language servers are started; choose from: -# al angular ansible bash clojure -# cpp cpp_ccls crystal csharp csharp_omnisharp -# dart elixir elm erlang fortran -# fsharp go groovy haskell haxe -# hlsl html java json julia -# kotlin lean4 lua luau markdown -# matlab msl nix ocaml pascal -# perl php php_phpactor powershell python -# python_jedi python_ty r rego ruby -# ruby_solargraph rust scala scss solidity -# svelte swift systemverilog terraform toml -# typescript typescript_vts vue yaml zig -# (This list may be outdated. For the current list, see values of Language enum here: -# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py -# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) -# Note: -# - For C, use cpp -# - For JavaScript, use typescript -# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) -# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) -# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) -# - For Free Pascal/Lazarus, use pascal -# Special requirements: -# Some languages require additional setup/installations. -# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers -# When using multiple languages, the first language server that supports a given file will be used for that file. -# The first language is the default language and the respective language server will be used as a fallback. -# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. -languages: -- rust - # the encoding used by text files in the project # For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings encoding: "utf-8" @@ -55,23 +22,19 @@ ignore_all_files_in_gitignore: true # advanced configuration option allowing to configure language server-specific options. # Maps the language key to the options. -# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. -# No documentation on options means no options are available. +# The settings are considered only if the project is trusted (see global configuration to define trusted projects). +# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings ls_specific_settings: {} -# list of additional workspace folder paths for cross-package reference support (e.g. in monorepos). -# Paths can be absolute or relative to the project root. -# Each folder is registered as an LSP workspace folder, enabling language servers to discover -# symbols and references across package boundaries. -# Currently supported for: TypeScript. -# Example: -# additional_workspace_folders: -# - ../sibling-package -# - ../shared-lib -additional_workspace_folders: [] - # list of additional paths to ignore in this project. # Same syntax as gitignore, so you can use * and **. +# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases. +# Example: +# ignored_paths: +# - "examples/**" +# - ".worktrees/**" +# - "**/bin/**" +# - "**/obj/**" # Note: global ignored_paths from serena_config.yml are also applied additively. ignored_paths: [] @@ -131,3 +94,76 @@ read_only_memory_patterns: [] # Extends the list from the global configuration, merging the two lists. # Example: ["_archive/.*", "_episodes/.*"] ignored_memory_patterns: [] + +# list of additional workspace folder paths for cross-package reference support. +# Paths can be absolute or relative to the project root. +# Each folder is registered as an LSP workspace folder, enabling language servers to discover +# symbols and references across package boundaries, but these folders are not indexed by Serena, +# i.e. the respective symbols will not be found using Serena's symbol search tools. +# Example: +# additional_workspace_folders: +# - ../sibling-package +# - ../shared-lib +ls_additional_workspace_folders: [] + +# list of language servers to start when using the LSP backend; choose from: +# ada al angular ansible bash +# bsl clojure cpp cpp_ccls crystal +# csharp csharp_omnisharp cue dart deno +# elixir elm erlang fortran fsharp +# gdscript gleam go groovy haskell +# haxe hlsl html java json +# julia kotlin latex lean4 lua +# luau markdown matlab msl nextflow +# nix ocaml pascal perl php +# php_phpactor php_phpantom powershell python python_basedpyright +# python_jedi python_pyrefly python_ty qml r +# rego ruby ruby_solargraph rust scala +# scss solidity svelte swift systemverilog +# terraform toml typescript typescript_vts vue +# wolfram yaml zig +# (This list may be outdated; generated with scripts/print_language_list.py; +# For the current list, see values of the LanguageServerId enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py) +# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root) +# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm) +# - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH) +# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three) +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# Some language servers require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers +# When using multiple language servers, the first language server that supports a given file will be used for that file. +# The first language server is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +language_servers: +- rust + +# list of workspace folder paths (LSP backend only). +# These folders will be used to build up Serena's symbol index. +# Paths must be within the project root and should thus be relative to the project root. +# Furthermore, the paths should not be filtered by ignore settings. +# Default setting: The entire project root folder (".") is considered. +# In (large) monorepos, this can be used to index only subfolders of the project root, e.g. +# ls_workspace_folders: +# - "./subproject1" +# - "./subproject2" +ls_workspace_folders: +- . + +# optional shell command to run before the language backend (LSP or JetBrains) is initialised. +# the command runs in the project root directory and is only executed if the project is trusted +# (see trusted_project_path_patterns in the global configuration). +# serena waits for the command to exit: a non-zero exit code is logged as an error but does not +# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety +# backstop for non-terminating commands; on expiry the process is killed and activation continues. +# example: activation_command: "npx nx run-many -t build" +activation_command: + +# maximum time in seconds to wait for activation_command to complete before killing it (default 180s). +# must be a positive number. +activation_command_timeout: 180.0 diff --git a/CLAUDE.md b/CLAUDE.md index aa836553..82a228fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,7 +90,7 @@ Je continue à poser mes questions et à guider la discussion. - Diagnostics LSP (erreurs de compilation) : `mcp__serena__get_diagnostics_for_file` - Vue d'ensemble d'un fichier : `mcp__serena__get_symbols_overview` - Modifier le corps d'une fonction/impl : `mcp__serena__replace_symbol_body` -- Ne pas utiliser `cclsp` quand serena couvre le besoin +- Ne pas utiliser `cclsp` ni `rust-analyzer-mcp` quand serena couvre le besoin — `rust-analyzer-mcp` en repli ponctuel (hover, diagnostics workspace) si serena ne couvre pas le cas **Analyse architecturale → jcodemunch** - Hotspots, couplage, dead code, dépendances entre modules @@ -101,3 +101,8 @@ Je continue à poser mes questions et à guider la discussion. **Documentation de crates → context7** - Toujours consulter avant d'utiliser une API de bibliothèque externe + +**Recherche sémantique dans la doc → qmd** +- Les deux arbres de doc sont indexés comme collections `qmd` : `obikmer-devdoc` (`DevDocMD/`, 57 fichiers) et `obikmer-userdoc` (`UserDocMD/`, 25 fichiers) +- Interroger via les outils MCP `query`/`get`/`multi_get`/`status`, ou en CLI (`qmd search`/`qmd query -c obikmer-devdoc "..."`) +- Après ajout/modification de fichiers Markdown dans `DevDocMD/` ou `UserDocMD/`, relancer `qmd embed` pour que l'index reste à jour (les collections ne se rafraîchissent pas automatiquement) diff --git a/DevDoc/404.html b/DevDoc/404.html index 43ca7c0d..b04c63bb 100644 --- a/DevDoc/404.html +++ b/DevDoc/404.html @@ -722,14 +722,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -967,6 +967,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/architecture/index_architecture.refs/index.html b/DevDoc/architecture/index_architecture.refs/index.html index 5b17230a..78ae7534 100644 --- a/DevDoc/architecture/index_architecture.refs/index.html +++ b/DevDoc/architecture/index_architecture.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1242,10 +1270,10 @@

    Coverage: architecture/index_architecture.md

    Code couvert

    diff --git a/DevDoc/architecture/numa_partition_runner/index.html b/DevDoc/architecture/numa_partition_runner/index.html index 27b9cc37..c25fea11 100644 --- a/DevDoc/architecture/numa_partition_runner/index.html +++ b/DevDoc/architecture/numa_partition_runner/index.html @@ -729,14 +729,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -974,6 +974,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/architecture/numa_worker_pools/index.html b/DevDoc/architecture/numa_worker_pools/index.html index 043289da..97421f9a 100644 --- a/DevDoc/architecture/numa_worker_pools/index.html +++ b/DevDoc/architecture/numa_worker_pools/index.html @@ -731,14 +731,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -976,6 +976,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1532,7 +1560,7 @@ Replace available_parallelism() with per-NUMA core count for spawn criterion -obikpartitionner/src/merge_layer.rs +obikindex/src/partition/merge_layer.rs No change — merge_partition already works inside any Rayon context @@ -1540,7 +1568,7 @@ No change — par_iter and current_num_threads are pool-context-aware -obikpartitionner/src/partition.rs +obikindex/src/partition/partition.rs No change — same reason diff --git a/DevDoc/architecture/query.refs/index.html b/DevDoc/architecture/query.refs/index.html index b0fc23d0..ef859997 100644 --- a/DevDoc/architecture/query.refs/index.html +++ b/DevDoc/architecture/query.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1243,7 +1271,7 @@

    Code couvert

    Notes

    diff --git a/DevDoc/architecture/query/index.html b/DevDoc/architecture/query/index.html index 9cbb8a79..2ed91fb1 100644 --- a/DevDoc/architecture/query/index.html +++ b/DevDoc/architecture/query/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1527,7 +1555,7 @@

    Algorithm

    -

    The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside process_chunk (query.rs); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (obikpartitionner::query_partition_with), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.

    +

    The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside process_chunk (query.rs); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (obikindex::partition::query_partition_with), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk.

    for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk):
         build QueryBatch (QueryBatch::from_records):
             decompose all sequences into superkmers (SuperKmerIter) — construction only,
    @@ -1602,9 +1630,9 @@
     

    The -z CLI option overrides the index metadata value. A higher z increases stringency (lower FP, some true positives may be discarded at sequence ends); a lower z increases sensitivity.


    Layer lookup: MphfLayer::find

    -

    MphfLayer::open(dir, mode: &IndexMode) receives the mode from PartitionMeta — no per-layer file is read. The caller (QueryLayer) never chooses the dispatch path: it is fixed at open time by LayerEvidence. See obilayeredmap for the full find / find_strict API.

    +

    MphfLayer::open(dir, mode: &IndexMode) receives the mode from PartitionMeta — no per-layer file is read. The caller (QueryLayer) never chooses the dispatch path: it is fixed at open time by LayerEvidence. See obikindex::layer for the full find / find_strict API.

    QueryLayer variant selection

    -

    QueryLayer::open (obikpartitionner/src/query_layer.rs:28-45) only ever returns two variants — Presence or Count, checked in this order:

    +

    QueryLayer::open (obikindex/src/partition/query_layer.rs:28-45) only ever returns two variants — Presence or Count, checked in this order:

    @@ -1774,7 +1802,7 @@ Fix direction: restructure query's pipe with an initial Flat<

    3. Chunk-size memory formula ignores n_genomes. chunk_bytes = available_memory_bytes() / (n_workers * 16) (query.rs:407-414) assumes a fixed ~8–16× overhead per raw input byte. But KmerResults::new (query.rs:165-179) allocates data: Vec<u32> sized total_kmers_in_chunk × n_genomes — dense, for every k-mer position in the chunk, hit or not — plus win_min and (with --detail) cov, same scaling. Real per-chunk memory is O(n_genomes), not constant; the formula doesn't know n_genomes at all. This is the direct cause of the OOM kill on indexes with many reference genomes.

    4. MPHF lookup and matrix-row fetch are fused, not staged. -QueryLayer::find_into (obikpartitionner/src/query_layer.rs:48-67) does the MPHF find and the fill_row matrix read in one call per k-mer, inside a single-threaded loop (query_partition_with). There is no separation between "is this k-mer indexed" (cheap, O(1), independent of n_genomes) and "what are its per-genome values" (the expensive, n_genomes-scaling part).

    +QueryLayer::find_into (obikindex/src/partition/query_layer.rs:48-67) does the MPHF find and the fill_row matrix read in one call per k-mer, inside a single-threaded loop (query_partition_with). There is no separation between "is this k-mer indexed" (cheap, O(1), independent of n_genomes) and "what are its per-genome values" (the expensive, n_genomes-scaling part).

    5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup. QueryBatch::from_records currently dereplicates at the superkmer level (HashMap<RoutableSuperKmer, Vec<SKDesc>>, query.rs:112). This misses redundancy between k-mers shared by different superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer construction (SuperKmerIter) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on CanonicalKmer, in the same pass: HashMap<CanonicalKmer, Vec<(seq_idx, pos)>>. This also means the MPHF find itself runs once per distinct k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below.

    6. Stage 1 output: bucket confirmed hits by layer, keyed by MPHF slot. @@ -1806,12 +1834,12 @@ Stage 2's output should be HashMap<genome_idx, Vec<(seq_idx, positio

    None of this is implemented yet — parked here as a coherent roadmap while the design is discussed further. Suggested dependency order: (1) I/O parallelism → (3) genome-aware chunk sizing → (4)–(9) staged/k-mer-deduped/NUMA-aware-partition-and-column-major/sparse query engine (larger refactor, biggest structural payoff — reuses PartitionRunner rather than inventing a new parallelism mechanism) → (2) parallel gzip (separate, orthogonal, tracked in chunkreader.md) → secondary diagnostics patterns.


    Implementation plan

    -

    Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current query output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning obikmer, obikpartitionner, and obicompactvec — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense KmerResults) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if rapidgzip-rs is validated (see chunkreader.md).

    +

    Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current query output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning obikmer, obikindex::partition, and obicompactvec — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense KmerResults) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if rapidgzip-rs is validated (see chunkreader.md).

    Instrumentation is deliberately sequenced before the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith.

    Performance measurement on the reference 192-core/8-NUMA machine is done by the project owner, not from this development environment (macOS, 16 cores — PartitionRunner's NUMA pinning is Linux-only, so even phase 4's mechanism can't be functionally exercised for its actual purpose here). Each phase below is therefore written to be self-measuring: the debug-level logging it adds must be enough, on its own, to judge whether that phase's algorithmic choice paid off from a cluster run's logs, without needing to attach a profiler.

    Conventions applied to every phase below

    Debug logging. Every phase that changes an algorithmic choice (not phase 0, which is the logging) adds tracing::debug!/trace! at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF find calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured debug! per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable.

    -

    Unit tests. This project's convention (obiread, obikseq, obidebruinj, obicompactvec, obilayeredmap, obiskio, obifastwrite) is #[cfg(test)] #[path = "tests/<name>.rs"] mod tests; at the bottom of the source file, with the actual test code in a sibling src/tests/<name>.rs. Neither obikmer nor obikpartitionner (the two crates phases 3 and 5 touch most) currently have a src/tests/ directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.

    +

    Unit tests. This project's convention (obiread, obikseq, obidebruinj, obicompactvec, obikindex::layer, obiskio, obifastwrite) is #[cfg(test)] #[path = "tests/<name>.rs"] mod tests; at the bottom of the source file, with the actual test code in a sibling src/tests/<name>.rs. Neither obikmer nor obikindex::partition (the two crates phases 3 and 5 touch most) currently have a src/tests/ directory at all — this needs creating, following the existing pattern exactly, not inventing a new one.

    Workflow (jj). Work happens in a fresh jj commit, easy to abandon. jj new between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then query_layer.rs split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet.

    Phase 0 — Instrumentation (prerequisite for measuring every later phase)

    Goal: make core utilization, throughput, and per-stage timing visible on a real run, so phases 1–5 can be justified with numbers instead of assumption.

    @@ -1854,26 +1882,26 @@ Stage 2's output should be HashMap<genome_idx, Vec<(seq_idx, positio
  • obikmer/src/cmd/query.rs:
  • Replace QueryBatch::from_records's dedup map (HashMap<RoutableSuperKmer, Vec<SKDesc>>, current query.rs:112) with a per-partition HashMap<CanonicalKmer, Vec<(seq_idx: u32, pos: u32)>>, built in the same SuperKmerIter pass: superkmer construction and partition routing (part_idx from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each CanonicalKmer within a superkmer is inserted individually instead of the whole superkmer being the dedup key.
  • Verified: CanonicalKmer (obikseq/src/kmer.rs:390, pub type CanonicalKmer = CanonicalKmerOf<KLen>) — the underlying CanonicalKmerOf<L> derives Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash (kmer.rs:269). Usable as a HashMap/HashSet key as-is, no change needed.
  • -
  • obikpartitionner/src/query_layer.rs:
  • +
  • obikindex/src/partition/query_layer.rs:
  • Split QueryLayer::find_into (query_layer.rs:48-67) into two methods: find_slot(&self, kmer: CanonicalKmer) -> Option<usize> (MPHF only, no matrix touch) and keep fill_row as-is for phase 4 to call later.
  • Replace query_partition_with's inner loop (query_layer.rs:103-113) with a version that, for each unique CanonicalKmer, calls find_slot across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records (layer_idx, slot).
  • New return shape for the partition-level query, replacing today's on_hit(sk_idx, kmer_idx, row) callback: HashMap<layer_idx, HashMap<slot, Vec<(seq_idx, pos)>>> (roadmap point 6) — built directly from the k-mer dedup map's Vec<(seq_idx,pos)> values, keyed by the resolved slot instead of the k-mer.
  • This phase alone has no throughput benefit yet (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above.
  • Log, per chunk: total k-mer occurrences vs. unique CanonicalKmer count (the dedup ratio — the entire justification for this phase) and the resulting MPHF find call count. If the dedup ratio is close to 1.0 on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path.
  • -
  • Unit tests: create obikmer/src/cmd/tests/query.rs (new src/tests/ dir for this crate, following the project's #[cfg(test)] #[path = "tests/query.rs"] mod tests; convention) and obikpartitionner/src/tests/query_layer.rs (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the find_slot/bucket-by-layer-and-slot construction against a small hand-built QueryLayer fixture, asserting the (layer_idx, slot, seq_idx, pos) tuples match what the old per-occurrence loop would have produced.
  • +
  • Unit tests: create obikmer/src/cmd/tests/query.rs (new src/tests/ dir for this crate, following the project's #[cfg(test)] #[path = "tests/query.rs"] mod tests; convention) and obikindex/src/partition/tests/query_layer.rs (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the find_slot/bucket-by-layer-and-slot construction against a small hand-built QueryLayer fixture, asserting the (layer_idx, slot, seq_idx, pos) tuples match what the old per-occurrence loop would have produced.
  • Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred

    Goal (revised during implementation): replace fill_row-per-hit (row-major, worst-case mmap locality) with a column-major scan. PartitionRunner turned out to be the wrong mechanism for this at this call granularity — see below; the column-major fetch itself is implemented and validated, without it.

    What shipped: - obicompactvec: the per-column accessors this phase needed already existedPersistentCompactIntMatrix::col_view(c) and PersistentBitMatrix::col_view(c) are public, and IntSliceView::get(slot)/BitSliceView::get(slot) are public — the original plan underestimated how much of this plumbing the pairwise-distance code (dump/select/stats) had already required. The one real gap: PersistentBitMatrix::col_view() panics on the Implicit variant (the documented mono-genome fast path, bitmatrix.rs). Added PersistentBitMatrix::get(c, slot) -> u32 (bitmatrix.rs), a non-panicking column-major point lookup that returns 1 for Implicit regardless of c — the smallest surface needed, not a new col_get API from scratch. -- obikpartitionner/src/query_layer.rs: query_partition_with is now two explicit stages, matching roadmap points 6–8: stage 1 (MPHF-only, per unique k-mer, bucket hits by (layer_idx, slot), emits QueryHit::Found) then stage 2 (per layer with ≥1 hit, column-major: for each genome column g in 0..layer.n_cols().min(n_genomes), scan that layer's bucketed slots and call col_value(g, slot), emitting QueryHit::Value(descs, g, value) on nonzero). QueryHit is a single enum delivered through one FnMut(QueryHit) callback — an earlier two-closure design (on_found + on_value) didn't borrow-check, since the caller's single mutable accumulator (KmerResults) can't be captured by two separate FnMut closures passed to the same call. +- obikindex/src/partition/query_layer.rs: query_partition_with is now two explicit stages, matching roadmap points 6–8: stage 1 (MPHF-only, per unique k-mer, bucket hits by (layer_idx, slot), emits QueryHit::Found) then stage 2 (per layer with ≥1 hit, column-major: for each genome column g in 0..layer.n_cols().min(n_genomes), scan that layer's bucketed slots and call col_value(g, slot), emitting QueryHit::Value(descs, g, value) on nonzero). QueryHit is a single enum delivered through one FnMut(QueryHit) callback — an earlier two-closure design (on_found + on_value) didn't borrow-check, since the caller's single mutable accumulator (KmerResults) can't be captured by two separate FnMut closures passed to the same call. - obikmer/src/cmd/query.rs: KmerResults::set (row-major, whole-row-at-once) replaced by mark_found (stage 1: flag a position as indexed, independent of any genome's value) and set_one (stage 2: write one genome's value at one position). QueryStats extended with n_columns_scanned/n_col_get_calls, logged per chunk. - Total get()-equivalent calls are unchanged from the row-major version (n_hits × n_cols in the worst case, confirmed by n_col_get_calls in the debug log) — the win is locality (sequential access within one layer's column at a time, across mmap'd regions, instead of jumping across all columns per hit), exactly as predicted.

    What did not ship, and why — PartitionRunner is architecturally the wrong tool here: Reading obikindex/src/numa.rs's actual run() body (not just its doc comments) shows every call spawns a timer thread plus one OS thread per worker slot on every NUMA node (std::thread::scope + one s.spawn() per node per max_workers) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned per call. This is fine for its actual, established usage in this codebase (merge.rs, index.rs's build_layers): one PartitionRunner::new() + one run() call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for query's call pattern: query_partition_with runs once per (chunk, partition), potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact. The column-major loop in stage 2 is therefore a plain sequential loop for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) rayon's already-warm global pool (into_par_iter()) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern numa_worker_pools.md warns about for a different workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one mmap'd region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so PartitionRunner is invoked once per whole query run (or per large batch of chunks) rather than per (chunk, partition), amortising its spawn cost the way merge/build_layers do — a bigger structural change than this phase's scope. - Log (implemented): QueryStats::n_columns_scanned/n_col_get_calls, folded into the existing per-chunk debug!("k-mer dedup + column-major fetch", ...) line (query.rs) alongside phase 3's dedup counters. -- Unit tests: extended obikpartitionner/src/tests/query_layer.rs (phase 3's file) — query_partition_with's empty/missing-index paths updated for the new QueryStats fields and single-callback signature. +- Unit tests: extended obikindex/src/partition/tests/query_layer.rs (phase 3's file) — query_partition_with's empty/missing-index paths updated for the new QueryStats fields and single-callback signature. - Validation performed: full workspace build + cargo test --workspace, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same kmer_count/kmer_strict_matches on every record); (2) the existing 20-genome benchmark/global_index_presence index — runs correctly, n_hits=0 for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the Implicit/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) the critical correctness case: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from genomeA matched only genomeA (kmer_count identical to the pre-dedup occurrence count, zero leakage into genomeB's column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in n_cols, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly. - Not yet done: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-n_genomes layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements.

    Phase 5 — Sparse Findere rework (roadmap point 9)

    diff --git a/DevDoc/architecture/rebuild_filter/index.html b/DevDoc/architecture/rebuild_filter/index.html index 36c8d27b..a21d25de 100644 --- a/DevDoc/architecture/rebuild_filter/index.html +++ b/DevDoc/architecture/rebuild_filter/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1417,9 +1445,9 @@ IO pattern: sequential read of each source column → random write into new colu

    Files to modify

      -
    • src/obikpartitionner/src/rebuild_layer.rsrebuild_partition and iter_src_layers
    • +
    • src/obikindex/src/partition/rebuild_layer.rsrebuild_partition and iter_src_layers
    • Possibly src/obicompactvec/ — add column iterator API if not already present
    • -
    • src/obilayeredmap/ — check if per-column sequential access is exposed on SrcLayerData
    • +
    • src/obikindex/src/layer/ — check if per-column sequential access is exposed on SrcLayerData

    Open questions

    diff --git a/DevDoc/architecture/sequences/invariant/index.html b/DevDoc/architecture/sequences/invariant/index.html index cba1d01b..38f3fb60 100644 --- a/DevDoc/architecture/sequences/invariant/index.html +++ b/DevDoc/architecture/sequences/invariant/index.html @@ -9,7 +9,7 @@ - + @@ -731,14 +731,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -976,6 +976,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/architecture/siblings/index.html b/DevDoc/architecture/siblings/index.html index 02bcdb46..a010b3cc 100644 --- a/DevDoc/architecture/siblings/index.html +++ b/DevDoc/architecture/siblings/index.html @@ -731,14 +731,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -976,6 +976,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1328,6 +1356,17 @@ + + +
  • + + + + Implemented (2026-08-20) + + + +
  • @@ -1602,6 +1641,17 @@ + + +
  • + + + + Implemented (2026-08-20) + + + +
  • @@ -1637,7 +1687,7 @@ one-way function, not an invertible bijection with a stored inverse. Any method that reconstructs a kmer from a bare slot number is wrong by construction, regardless of the mechanism used (MPHF re-hash, or evidence decode + direct unitig read). See MphfLayer::kmer_at -(obilayeredmap/src/mphf_layer.rs) — flagged for removal, currently called +(obikindex/src/layer/mphf_layer.rs) — flagged for removal, currently called from obikphylo/siblings/build.rs and family_scan.rs (since removed — see "Pending work" status below).

    Two pipelines, never mixed

    @@ -1700,7 +1750,7 @@ partition is unknown) and must keep going through index().

    Pending work — done

    The plan above shipped: obikphylo (a new crate — phylo-domain extension -traits over obikindex::KmerIndex/obilayeredmap::Layer<D>, replacing the +traits over obikindex::KmerIndex/obikindex::layer::Layer<D>, replacing the old obikindex::siblings module) builds and reads the annex purely in iteration order (SiblingLayerExt::iter_siblings/iter_minorants, both with batch variants, mirroring Layer<D>'s own KmerIter/KmerBatchIter @@ -2268,13 +2318,13 @@ BRWT-style column-correlation exploitation.

    PersistentSparseBitMatrix went from a validated but unused type to a real, selectable on-disk format:

      -
    • Generic Layer<D>: obilayeredmap::Layer<D>'s presence-only methods +
    • Generic Layer<D>: obikindex::layer::Layer<D>'s presence-only methods (n_cols, sub_matrix, fill_sub_matrix) are generic over any D: LayerData<Item = Box<[bool]>> + BinaryMatrix, not hardcoded to PersistentBitMatrixPersistentSparseBitMatrix implements LayerData (open/read) the same way. find_slot/index_batch were already generic over any D: LayerData, so they needed no change. - Verified by obilayeredmap's + Verified by obikindex::layer's presence_layer_generic_over_sparse_matches_dense test: build a dense presence layer, convert it to sparse via build_from_dense, open both as Layer<PersistentBitMatrix>/Layer<PersistentSparseBitMatrix> on @@ -2284,7 +2334,7 @@ real, selectable on-disk format:

      AtomicUsizes in test builds, not thread-local, so a test using a different k races every other test in the same crate binary; a k=11 version of this test passed alone but failed under the full - obilayeredmap suite for exactly that reason before being fixed.)
    • + obikindex::layer suite for exactly that reason before being fixed.)
    • obikphylo::siblings::cache::Mat gained a third variant, SparsePresence(Layer<PersistentSparseBitMatrix>), alongside Count and Presence — every method (find_slot, index_batch, @@ -2314,7 +2364,7 @@ real, selectable on-disk format:

      dense-path test (sibling_annex_one_sibling_each) exactly — proves the sparse format round-trips through the real build pipeline (PartitionCache sparse-detection included), not just the - obicompactvec/obilayeredmap unit layers below it.
    • + obicompactvec/obikindex::layer unit layers below it.

    Full workspace cargo test (all crates, unit + doc tests) green after this change.

    @@ -2360,7 +2410,7 @@ this change.

    (activation: either given; defaults 1.0/0.5 for whichever is unset).

    Full workspace cargo test green after this change (167 unit tests in -obicompactvec+obilayeredmap+obikphylo alone, plus every other +obicompactvec+obikindex::layer+obikphylo alone, plus every other crate's suite, no regressions).

    Still open, not part of this change (per "Correction to the 'single pass' framing" above): --raw-snp-distance/--raw-snp-counts (the @@ -2380,7 +2430,7 @@ reproducible across two runs with warm disk cache), the opposite of pack --sparse's stated intent ("faster for single-row access... like query").

    Root cause, read from source, not measured in isolation: -KmerPartition::query_partition_with (obikpartitionner/src/query_layer.rs:155-220) +KmerPartition::query_partition_with (obikindex/src/partition/query_layer.rs:155-220) is architecturally column-major: stage 2 walks for g in 0..n_cols { for slot in hit_slots { layer.col_value(g, slot) } }, documented (correctly) as the right locality strategy for the packed/columnar formats, where @@ -2407,45 +2457,131 @@ consumes as QueryHit::Value — not a materialized sub-matrix.

    Proposed primitive (design only, not implemented — explicit ask: keep count matrices not excluded, even though effort right now is presence/absence only):

    -
    /// Visit every nonzero cell among `slots`. Order unspecified.
    -fn for_each_nonzero(&self, slots: &[usize], f: impl FnMut(usize /*idx into slots*/, usize /*col*/, u32 /*value*/));
    +

    Not a closure-driven for_each — a real Iterator, one concrete struct +per matrix format, so the traversal state (current position in the sorted +slot list, current column, permutation, sparse-row decode cursor…) lives +in named struct fields instead of being threaded implicitly through +recursion or a captured closure. RPITIT (stable since 1.75, and this +workspace is edition 2024) means the trait method can return it without +naming or boxing the concrete type:

    +
    /// Yields every nonzero cell among `slots`, in implementation-defined order.
    +fn nonzero_iter<'a>(&'a self, slots: &'a [usize]) -> impl Iterator<Item = (usize, usize, u32)> + 'a;
    +// item: (idx into `slots`, col, value)
     
    +

    This is the one foundational primitive per format — both fill_sub_matrix +and a for_each-style callback become trivial, free consumers of it +(.for_each(f) is already Iterator::for_each from std; fill_sub_matrix +becomes "drain the iterator, scatter into out[][]"), instead of two +independently-maintained traversals that can silently diverge (see the bug +below — this is exactly how it happened).

      -
    • On PersistentSparseBitMatrix: native override, one pass per slot via - the existing (currently private) for_each_genome_in_row — O(Σ row - nnz), zero n_cols-wide allocation. This is the whole point: expose code - that already exists rather than write anything new for the sparse side.
    • -
    • On PersistentBitMatrix::{Packed,Columnar,Implicit}: provided - default, derived from fill_sub_matrix (materialize, then filter to - true cells) — reuses the already-optimal column-major/mmap path for - those formats, no new code needed there either.
    • -
    • On PersistentCompactIntMatrix (counts): same provided-default - treatment, derived from its own existing fill_sub_matrix (u32-typed - already, intmatrix.rs:387) — not hand-optimized (no sparse count - format exists — "Explicitly deferred" per traits.rs:9-12), but not - excluded either: it gets a working, not-pathological implementation for - free today, on the same trait, ready for a native override the day a - sparse count format lands. This is why the signature carries u32 - rather than bool — presence is 0/1, counts are u32, one trait - covers both without a bool/u32 split forcing counts out of the design.
    • +
    • On PersistentSparseBitMatrix: the struct is nearly free to write — + it wraps the existing (currently private) for_each_genome_in_row + per-row decode, advancing to the next slots entry on exhaustion. O(Σ + row nnz), zero n_cols-wide allocation.
    • +
    • On PersistentBitMatrix::{Packed,Columnar}: revised — cheaper than + first thought, by reusing the same split already used for + fill_matrix's own implementation instead of hand-writing a resumable + state machine at the matrix level. The base-vector layer + (BitSliceView, views.rs) already separates the two concerns: + fill_batch_sorted (views.rs:55-60, sorted-slot batch lookup) and a + genuine per-bit Iterator (BitSliceIter, views.rs:94+) sit side by + side there, one level below the matrix. Adding a + "positions among sorted_slots where the bit is set" iterator at that + same vector level is a filter over the existing get() — no new + state machine, since std::iter::Filter already is one, generated by + the compiler: +
      fn nonzero_among_sorted<'s>(&'s self, sorted_slots: &'s [usize]) -> impl Iterator<Item = usize> + 's {
      +    sorted_slots.iter().copied().filter(move |&slot| self.get(slot))
      +}
      +
      + The matrix-level nonzero_iter then composes these per column with + flat_map over 0..n_cols (each column's hits, tagged with c, + slot mapped back through the sort permutation fill_batch/ + fill_batch_sorted already carry) — again a combinator chain, not a + hand-rolled struct. Same algorithm, same mmap/sort locality as today's + fill_sub_matrix; just assembled from std iterator adaptors instead + of a loop body writing into a buffer, mirroring the vector/matrix split + the codebase already uses for fill_batch_sorted rather than + introducing a new shape.
    • +
    • Implicit: trivial (slots.iter().map(|&i| (i, 0, 1)), one column, + always present).
    • +
    • On PersistentCompactIntMatrix (counts): same treatment as + Packed/Columnar — no sparse count format exists yet ("Explicitly + deferred" per traits.rs:9-12), so no native low-effort case the way + Sparse has one, but not excluded either: the iterator's Item is + already (usize, usize, u32), not bool, specifically so presence + (0/1) and counts (arbitrary u32) share one primitive instead of a + bool/u32 split forcing counts out of the design. Ready for a native + sparse-count struct later without a signature change.

    Would let query_partition_with's stage 2 collapse to one -layer.matrix().for_each_nonzero(&hit_slots, |i, g, v| on_event(...)) -call per layer, format-agnostic, with each backend's existing (or -default-derived) implementation deciding the actual access pattern.

    -

    Separately, an existing bug in the generic path (found while tracing -this, itself not yet fixed): PersistentBitMatrix::fill_sub_matrix -(persistent.rs:190-215, the enum wrapper backing BinaryMatrix's -default trait impl) does not delegate to +for (i, g, v) in layer.matrix().nonzero_iter(&hit_slots) { on_event(...) } +per layer, format-agnostic, each backend's struct deciding the actual +traversal.

    +

    This also closes the existing dispatch bug for free, by construction: +PersistentBitMatrix::fill_sub_matrix (persistent.rs:190-215, the enum +wrapper backing BinaryMatrix's trait impl) today does not delegate to PersistentSparseBitMatrix::fill_sub_matrix for Self::Sparse — it reimplements the same naive per-(column, slot) fill_row_bool loop instead, bypassing the efficient native method one file over (sparse.rs:249-258). obikphylo::siblings::cache::Mat (cache.rs:138-145) independently built its own parallel enum wrapper -that dispatches correctly — a sign this was worked around rather than -fixed at the source. Any future for_each_nonzero work should fix this -dispatch too (or route through it), rather than adding a third -independently-dispatching wrapper.

    +that dispatches correctly — evidence this was worked around rather than +fixed at the source: two hand-written traversals for the same format, +free to drift apart, and they did. If fill_sub_matrix itself is +rewritten as "drain nonzero_iter, scatter into out[][]", there is only +one traversal per format left to get right — the bug class doesn't just +get fixed once, it stops being possible to reintroduce.

    +

    Implemented (2026-08-20)

    +

    Built as designed above, with one deviation from the original sketch: +nonzero_iter ended up Box<dyn Iterator<...>>, not a bare impl +Iterator, because Columnar/Packed/Sparse/Implicit are genuinely +different concrete types and this method isn't on a trait (kept off +BinaryMatrix deliberately — that trait is used as dyn BinaryMatrix in +tests/sparse.rs, and RPITIT methods aren't dyn-compatible). One Box +per nonzero_iter call, not per cell — negligible next to what it +replaces.

    +
      +
    • BitSliceView::nonzero_among_sorted / IntSliceView::nonzero_among_sorted + (obicompactvec/src/views.rs): the vector-level filter/filter_map + primitive, exactly as sketched — no new state machine, std's own.
    • +
    • PersistentSparseBitMatrix::nonzero_iter (bitmatrix/sparse.rs): native, + std::iter::from_fn over one buffered row at a time via the existing + for_each_genome_in_row — no n_cols-wide allocation, ever.
    • +
    • PersistentBitMatrix::nonzero_iter (bitmatrix/persistent.rs): dispatches + to the above for Sparse; for Columnar/Packed, loops columns, + collects each column's nonzero_among_sorted hits via .extend() (not + flat_map — a flat_map closure can't lazily return something + borrowing its own captured sort permutation across separate calls + without either boxing per-column or fighting the borrow checker; eager + collection into one Vec sidesteps it, at zero cost since + fill_sub_matrix already fully materialized anyway). Implicit trivial.
    • +
    • PersistentBitMatrix::fill_sub_matrix and sub_matrix rewritten to + drain nonzero_iter — the dispatch bug is gone because there is now + only one traversal per format, not because the old one was patched. + PersistentCompactIntMatrix::nonzero_iter added the same way (counts + not excluded, per the earlier ask) — no native low-effort case, since no + sparse count format exists, but on the same primitive, ready for one.
    • +
    • KmerPartition::query_partition_with (obikindex/src/partition/query_layer.rs): + stage 2's column-major for g { for slot { col_value } } replaced by one + layer.nonzero_iter(&slot_list) call per layer, format-agnostic.
    • +
    • Tests: nonzero_iter_matches_dense, nonzero_iter_matches_row, and — + the one that actually targets the dispatch bug rather than each type's + own correctness — enum_wrapper_dispatches_to_native_sparse (builds + PersistentBitMatrix::Sparse(...) directly, not through open, since + open only auto-detects Sparse from a presence/ dir layout). + cargo test --workspace: green, no regressions.
    • +
    +

    Measured: re-ran the benchmark/ query branch (100k reads × 2 +specimens, same setup as the original finding). Correctness still 0 +mismatches. The dense/sparse performance gap is gone — previously sparse +~30-50% slower than dense, reproducibly; now within ~1-3% either way +(7.42s dense vs 7.60s sparse for Escherichia_coli--K-12_MG1655; 5.25s vs +5.30s for Saccharolobus_islandicus--M.16.4) — noise-level, not a +systematic gap. pack --sparse's claimed query win isn't confirmed +outright by this (sparse should arguably now beat dense on truly sparse +real data, not just tie), but the pathological regression is fixed.

    diff --git a/DevDoc/implementation/benchmark_query_testing/index.html b/DevDoc/implementation/benchmark_query_testing/index.html index 1ac8d0ae..c8e7c979 100644 --- a/DevDoc/implementation/benchmark_query_testing/index.html +++ b/DevDoc/implementation/benchmark_query_testing/index.html @@ -12,7 +12,7 @@ - + @@ -733,14 +733,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1113,6 +1113,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/chunkreader.refs/index.html b/DevDoc/implementation/chunkreader.refs/index.html index 3be4d1b2..d19f2081 100644 --- a/DevDoc/implementation/chunkreader.refs/index.html +++ b/DevDoc/implementation/chunkreader.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/chunkreader/index.html b/DevDoc/implementation/chunkreader/index.html index 0fea7d09..b1ed11a9 100644 --- a/DevDoc/implementation/chunkreader/index.html +++ b/DevDoc/implementation/chunkreader/index.html @@ -405,11 +405,11 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -509,6 +509,18 @@ + + +
  • +
  • + + + + + Partition and layer caching (discussion) + + +
  • diff --git a/DevDoc/implementation/evidence_elimination.refs/index.html b/DevDoc/implementation/evidence_elimination.refs/index.html index aaa4b067..1fa176b3 100644 --- a/DevDoc/implementation/evidence_elimination.refs/index.html +++ b/DevDoc/implementation/evidence_elimination.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1242,9 +1270,9 @@

    Coverage: implementation/evidence_elimination.md

    Code couvert

      -
    • obilayeredmap/src/fingerprint.rs — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
    • -
    • obilayeredmap/src/mphf_layer.rs — build_approx_evidence(dir, b, z), find_approx()
    • -
    • obilayeredmap/src/meta.rs — EvidenceKind::Approx { b, z }, LayerMeta
    • +
    • obikindex/src/layer/fingerprint.rs — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches()
    • +
    • obikindex/src/layer/mphf_layer.rs — build_approx_evidence(dir, b, z), find_approx()
    • +
    • obikindex/src/layer/meta.rs — EvidenceKind::Approx { b, z }, LayerMeta
    • obikindex/src/reindex.rs — KmerIndex::reindex(), conversion exact↔approx en place
    • obikmer/src/cmd/reindex.rs — CLI reindex, options --approx, -z, --evidence-bits, --fp, --block-size
    • obikmer/src/cmd/index.rs — resolve_approx_params(), options --approx, -z, --evidence-bits, --fp
    • diff --git a/DevDoc/implementation/evidence_elimination/index.html b/DevDoc/implementation/evidence_elimination/index.html index 7fa83734..70621d90 100644 --- a/DevDoc/implementation/evidence_elimination/index.html +++ b/DevDoc/implementation/evidence_elimination/index.html @@ -12,7 +12,7 @@ - + @@ -868,14 +868,14 @@
    • - + - obilayeredmap crate + obikindex layer tier @@ -1113,6 +1113,34 @@ + + + + + + +
    • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    • + + + +
    diff --git a/DevDoc/implementation/filtering/index.html b/DevDoc/implementation/filtering/index.html index 8ca925a4..6fdab235 100644 --- a/DevDoc/implementation/filtering/index.html +++ b/DevDoc/implementation/filtering/index.html @@ -733,14 +733,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1236,6 +1236,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -2158,7 +2186,7 @@ obikmer phylo myindexImplementation diff --git a/DevDoc/implementation/kmer/index.html b/DevDoc/implementation/kmer/index.html index af80cf0b..f367ae72 100644 --- a/DevDoc/implementation/kmer/index.html +++ b/DevDoc/implementation/kmer/index.html @@ -868,14 +868,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1113,6 +1113,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/obilayeredmap.refs/index.html b/DevDoc/implementation/layer_tier.refs/index.html similarity index 93% rename from DevDoc/implementation/obilayeredmap.refs/index.html rename to DevDoc/implementation/layer_tier.refs/index.html index 49fa3c13..92d01fcf 100644 --- a/DevDoc/implementation/obilayeredmap.refs/index.html +++ b/DevDoc/implementation/layer_tier.refs/index.html @@ -19,7 +19,7 @@ - Obilayeredmap.refs - obikmer + Layer tier.refs - obikmer @@ -60,7 +60,7 @@
    - + Skip to content @@ -96,7 +96,7 @@
    - Obilayeredmap.refs + Layer tier.refs
    @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1239,15 +1267,15 @@ -

    Coverage: implementation/obilayeredmap.md

    +

    Coverage: implementation/layer_tier.md

    Code couvert

      -
    • obilayeredmap/src/mphf_layer.rs — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
    • -
    • obilayeredmap/src/layer.rs — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
    • -
    • obilayeredmap/src/map.rs — LayeredMap, push_layer(), query()
    • -
    • obilayeredmap/src/evidence.rs — Evidence, EvidenceWriter, encodage chunk_id:rank
    • -
    • obilayeredmap/src/fingerprint.rs — FingerprintVec, FingerprintVecWriter, matches()
    • -
    • obilayeredmap/src/meta.rs — LayerMeta, EvidenceKind (Exact / Approx { b, z })
    • +
    • obikindex/src/layer/mphf_layer.rs — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx()
    • +
    • obikindex/src/layer/typed_layer.rs — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column()
    • +
    • obikindex/src/layer/map.rs — LayeredMap, push_layer(), query()
    • +
    • obikindex/src/layer/evidence.rs — Evidence, EvidenceWriter, encodage chunk_id:rank
    • +
    • obikindex/src/layer/fingerprint.rs — FingerprintVec, FingerprintVecWriter, matches()
    • +
    • obikindex/src/layer/meta.rs — LayerMeta, EvidenceKind (Exact / Approx { b, z })

    Notes

    FORT RISQUE DE DÉRIVE. C'est le fichier le plus affecté par les changements récents : diff --git a/DevDoc/implementation/obilayeredmap/index.html b/DevDoc/implementation/layer_tier/index.html similarity index 98% rename from DevDoc/implementation/obilayeredmap/index.html rename to DevDoc/implementation/layer_tier/index.html index 2439b8a6..2dc88b7a 100644 --- a/DevDoc/implementation/obilayeredmap/index.html +++ b/DevDoc/implementation/layer_tier/index.html @@ -23,7 +23,7 @@ - obilayeredmap crate - obikmer + obikindex layer tier - obikmer @@ -64,7 +64,7 @@

    - + Skip to content @@ -100,7 +100,7 @@
    - obilayeredmap crate + obikindex layer tier
    @@ -749,7 +749,7 @@ - obilayeredmap crate + obikindex layer tier @@ -767,7 +767,7 @@ - obilayeredmap crate + obikindex layer tier @@ -1352,6 +1352,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1923,9 +1951,9 @@ -

    obilayeredmap — layered kmer index crate

    +

    obikindex::layer — the Layer tier

    Purpose

    -

    obilayeredmap implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a ptr_hash MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.

    +

    obikindex::layer (the layer/ submodule of the obikindex crate — a standalone obilayeredmap crate until 2026-08-21, folded back in alongside obikpartition as part of a broader Index { Partition { Layer } } submodule regrouping) implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a ptr_hash MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers.


    Three usage modes

    The MPHF + evidence infrastructure is the same for all modes. The payload varies.

    diff --git a/DevDoc/implementation/merge.refs/index.html b/DevDoc/implementation/merge.refs/index.html index ea0f4334..8f4e288c 100644 --- a/DevDoc/implementation/merge.refs/index.html +++ b/DevDoc/implementation/merge.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1243,9 +1271,9 @@

    Code couvert

    • obikindex/src/merge.rsKmerIndex::merge(), validation de compatibilité d'évidence, validate_evidence_compat()
    • -
    • obikpartitionner/src/merge_layer.rsmerge_partition(), construction de la nouvelle layer, paramètre block_bits
    • -
    • obikpartitionner/src/rebuild_layer.rsrebuild_partition(), paramètre block_bits
    • -
    • obilayeredmap/src/layer.rsLayer::append_genome_column() (PersistentCompactIntMatrix et PersistentBitMatrix)
    • +
    • obikindex/src/partition/merge_layer.rsmerge_partition(), construction de la nouvelle layer, paramètre block_bits
    • +
    • obikindex/src/partition/rebuild_layer.rsrebuild_partition(), paramètre block_bits
    • +
    • obikindex/src/layer/typed_layer.rsLayer::append_genome_column() (PersistentCompactIntMatrix et PersistentBitMatrix)
    • obicompactvec/src/intmatrix.rsappend_column pour PersistentCompactIntMatrix
    • obicompactvec/src/bitmatrix.rsappend_column pour PersistentBitMatrix
    diff --git a/DevDoc/implementation/merge/index.html b/DevDoc/implementation/merge/index.html index 0a65b288..7cfe4472 100644 --- a/DevDoc/implementation/merge/index.html +++ b/DevDoc/implementation/merge/index.html @@ -733,14 +733,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1185,6 +1185,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/merge_parallelism/index.html b/DevDoc/implementation/merge_parallelism/index.html index 41b5bd0f..b7395e98 100644 --- a/DevDoc/implementation/merge_parallelism/index.html +++ b/DevDoc/implementation/merge_parallelism/index.html @@ -733,14 +733,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1180,6 +1180,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/mphf.refs/index.html b/DevDoc/implementation/mphf.refs/index.html index 8c39c997..331707e7 100644 --- a/DevDoc/implementation/mphf.refs/index.html +++ b/DevDoc/implementation/mphf.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1242,8 +1270,8 @@

    Coverage: implementation/mphf.md

    Code couvert

      -
    • obilayeredmap/src/mphf_layer.rs — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, build(), build_exact_evidence(), build_approx_evidence(), build_evidence()
    • -
    • obikpartitionner/src/index_layer.rsbuild_index_layer() avec passage de block_bits
    • +
    • obikindex/src/layer/mphf_layer.rs — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, build(), build_exact_evidence(), build_approx_evidence(), build_evidence()
    • +
    • obikindex/src/partition/index_layer.rsbuild_index_layer() avec passage de block_bits

    Notes

    FORT RISQUE DE DÉRIVE. Changements récents : diff --git a/DevDoc/implementation/mphf/index.html b/DevDoc/implementation/mphf/index.html index 726d9ba0..010a5215 100644 --- a/DevDoc/implementation/mphf/index.html +++ b/DevDoc/implementation/mphf/index.html @@ -946,14 +946,14 @@

  • - + - obilayeredmap crate + obikindex layer tier @@ -1191,6 +1191,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1605,7 +1633,7 @@

    Why two phases are needed

    Kmer indexing per partition proceeds in two phases. The separation is necessary because the exact number of surviving unique kmers is not known until after counting and filtering low-abundance kmers.

    Phase 1 — provisional MPHF + kmer spectrum

    -

    Implemented in obikpartitionner::KmerPartition::count_kmer()count_partition().

    +

    Implemented in obikindex::partition::KmerPartition::count_kmer()count_partition().

    1. External sort: read the dereplicated superkmer file; extract the raw u64 canonical kmer value for every kmer of every superkmer. Sort in RAM-bounded chunks (adaptive budget: 40% of available RAM ÷ n_threads, minimum 1 M kmers per chunk), then k-way merge with inline dedup. Result: sorted_unique.bin — a flat array of f0 distinct sorted u64 values. Exact kmer count f0 is known at this point.
    2. Build provisional MPHF (ptr_hash, same configuration as phase 2) over sorted_unique.bin using new_from_par_iter. Delete sorted_unique.bin immediately after. Persist to mphf1.bin.
    3. @@ -1695,7 +1723,7 @@

      Multilayer index architecture

      Layer structure

      -

      Each layer is a self-contained unit. See obilayeredmap for the full on-disk layout. The MPHF-relevant files are:

      +

      Each layer is a self-contained unit. See obikindex::layer for the full on-disk layout. The MPHF-relevant files are:

      layer_i/
         unitigs.bin      — packed 2-bit nucleotide sequences (kmer evidence source)
         unitigs.bin.idx  — random-access block index (block_bits controls granularity)
      @@ -1730,7 +1758,7 @@ MphfLayer::build_approx_evidence(dir, b, z)
           Uses open_sequential(); never writes .idx
       

      There is no build_evidence dispatch wrapper. Callers choose the appropriate post-hoc build directly.

      -

      In obikpartitionner, build_index_layer receives block_bits: u8 from IndexConfig::block_bits and forwards it directly to Layer::build and Layer::build_approx_evidence.

      +

      In obikindex::partition, build_index_layer receives block_bits: u8 from IndexConfig::block_bits and forwards it directly to Layer::build and Layer::build_approx_evidence.

      Membership verification

      ptr_hash maps any input to a valid slot — it does not natively detect absent keys. Membership is verified using the evidence entry:

      @@ -1247,7 +1275,7 @@

      Notes

      Document stable (librairie générique, peu de risque de dérive). -Vérifier si obipipeline est toujours utilisé dans la phase scatter de obikpartitionner +Vérifier si obipipeline est toujours utilisé dans la phase scatter de obikindex::partition ou s'il a été remplacé par Rayon dans certains chemins.

      diff --git a/DevDoc/implementation/obipipeline/index.html b/DevDoc/implementation/obipipeline/index.html index 08a6fdf8..edebe6b6 100644 --- a/DevDoc/implementation/obipipeline/index.html +++ b/DevDoc/implementation/obipipeline/index.html @@ -896,14 +896,14 @@
    4. - + - obilayeredmap crate + obikindex layer tier @@ -1141,6 +1141,34 @@ + + + + + + +
    5. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    6. + + + + diff --git a/DevDoc/implementation/obitaxonomy/index.html b/DevDoc/implementation/obitaxonomy/index.html index ce0218b7..b1886a24 100644 --- a/DevDoc/implementation/obitaxonomy/index.html +++ b/DevDoc/implementation/obitaxonomy/index.html @@ -733,14 +733,14 @@
    7. - + - obilayeredmap crate + obikindex layer tier @@ -1153,6 +1153,34 @@ + + + + + + +
    8. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    9. + + + + diff --git a/DevDoc/implementation/partition_layer_cache/index.html b/DevDoc/implementation/partition_layer_cache/index.html new file mode 100644 index 00000000..f3f36012 --- /dev/null +++ b/DevDoc/implementation/partition_layer_cache/index.html @@ -0,0 +1,2843 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Partition and layer caching (discussion) - obikmer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      + +
      + + + + + +

      Partition and layer caching (discussion)

      +

      Superseded (2026-08-21): obikpartition and obilayeredmap are no +longer separate workspace crates — both were folded back into obikindex +as submodules (obikindex::partition, obikindex::layer), alongside the +crate's original content as obikindex::index, purely to reduce the +crate count (no behavior change). Every mention of obikpartition/ +obilayeredmap as a crate below, and every dependency-direction +argument phrased in terms of "which crate depends on which" (e.g. "this +crate depends only on obilayeredmap and below, never on obikindex"), +describes that now-superseded split-crate architecture and is kept as-is +for historical context — read obikpartition::X as obikindex:: +partition::X and obilayeredmap::X as obikindex::layer::X throughout. +The underlying module boundary and its rationale (Layer tier / Partition +tier / Index tier, each depending only downward) are unchanged; only the +crate-vs-module packaging changed. See obikindex::layer +for the current module doc.

      +

      Status (2026-08-20, latest pass): (1) done — obilayeredmap::Layer +exists, Mat is gone. (1b) done — Layer::Empty, the first non-ready +state, added (panics on every read method). (2a) done — the +obikpartition crate and KmerPartition itself exist (open/n_layers/ +layer/layers/find). (2b) — migrating PartitionCache/QueryLayer +onto it — not started, deliberately deferred. (3) done — the +obikindex ↔ obikpartitionner dependency inverted: PartitionRouter now +takes &mut KmerIndex and produces Layer::Empty shells directly, closing +the gap Layer::Empty was built for in (1b) — see "(3) done" below. (4) +done — dereplication split out into its own crate, obikderep, first step +of an incremental "one algorithm at a time" split of obikpartitionner's +remaining bundle (count_kmer/build_layers not yet moved) — see "(4) +done" below. (5) — full design agreed, not yet implemented — +KmerPartition was found to be wired into nothing (KmerIndex never +calls it; every path is still computed via free functions), and the fix +turned out to be bigger than KmerPartition alone: Layer's own +constructors don't self-name either. Full redesign of both, agreed in +detail, session ended (budget) before implementation — see "(5) design +agreed" below; read it before touching KmerPartition/Layer +signatures, the shape is fully specified. Earlier mix-up, for +context: an earlier +version of this doc used the name KmerPartition (singular) for what was +actually the collection type (later renamed KmerPartitions, later +merged into KmerIndex — see "Major restructuring" below), and never +retracted that usage before this section was rewritten. An agent working +from that stale wording built the wrong thing. If you are about to +implement (2), read "Definitions: obikpartition and KmerPartition" +below — it is the current, authoritative naming — before touching any +other section of this file, some of which still describe superseded +states of the code and are kept only as dated history.

      +

      Definitions: obikpartition and KmerPartition (not yet created)

      +

      obikpartition — a new workspace crate, not created yet. Holds the +Partition tier of the Index { Partition { Layer } } model, the same +way obilayeredmap already holds the Layer tier as its own crate +rather than living inside obikindex. Depends only on obilayeredmap +(for Layer) and lower (obikseq, obiskio). Does not depend on +obikindex, obikpartitionner, or obikphylo. Dependency direction: +obikindex → obikpartition → obilayeredmap; obikphylo → obikindex +(and/or obikpartition directly if it ends up needing it without going +through KmerIndex).

      +

      KmerPartition (singular) — the one type this crate exists for. +Represents one partition's already-open layers — a read cache, built +once per partition and held for the run, not rebuilt per lookup. Shape:

      +
      pub struct KmerPartition {
      +    layers: Vec<obilayeredmap::Layer>,
      +}
      +
      +

      Nothing else. In particular: +- No path computation. KmerPartition::open takes an already-resolved + index_dir: &Path (plus mode: &IndexMode, n_layers: usize, + with_counts: bool — whatever it needs, as plain arguments), the same + discipline obikpartitionner::PartitionRouter::open already follows. + Computing index_dir/layer_dir from a partition number is + KmerIndex's job (obikindex, which owns that already — see "Major + restructuring" below); KmerPartition never reaches back into + KmerIndex to get it (would require obikpartition → obikindex, the + wrong direction). +- No routing/write state. Writing raw superkmers, dereplicate, + count_kmer stay in obikpartitionner::PartitionRouter — a completely + different crate, a completely different phase (pre-layer, whereas + KmerPartition only makes sense once layers exist). +- No multi-partition collection baked in. KmerPartition is one + partition. Whatever ends up caching several of them (replacing + obikphylo::siblings::cache::PartitionCache's Vec<Vec<Layer>> and + obikindex::query_layer's per-call reopen) holds Vec<KmerPartition> — + that collection can live in obikpartition too, or in obikindex + alongside KmerIndex; not yet decided, secondary to getting + KmerPartition itself right first.

      +

      Do not confuse with KmerPartitions (plural — note the s): that +type is gone. It used to be obikpartitionner's (nee obikpartition, +briefly — see the crate-rename history below, itself a separate rename +from this one) do-everything struct — routing, dereplication, and path +lookups all in one. It was deleted on 2026-08-20; its read-side (paths, +n_layers, partition_meta) was absorbed into KmerIndex, its +write-side became PartitionRouter. KmerPartition (this section, +singular, no final s) is a brand-new type with a different job, in a +crate that doesn't exist yet — not a revival, not a renaming, of +KmerPartitions.

      +

      Type-to-concept mapping: Index / Partition / Layer

      +

      The conceptual nesting Index { Partition { Layer { MPHF, Evidence, Matrix +} } } }, current state:

      +
        +
      • Index = obikindex::KmerIndex{ root_path, meta: IndexMeta }. + Also directly exposes the partition-path/metadata accessors + (partition_dir(i), index_dir(i), layer_dir(i, l), + partition_meta(i), n_layers(i), partition_mode(i), + n_partitions()) since KmerPartitions merged into it (see "Major + restructuring" below) — KmerIndex today is "index + collection of + partitions' paths & metadata," just without a Vec of open layers.
      • +
      • Partition, the collection = no dedicated type today; the closest + thing is KmerIndex itself (previous bullet). Once KmerPartition + (singular, see Definitions above) exists, a Vec<KmerPartition> + somewhere would be this — still open, see "Direction agreed" below.
      • +
      • Partition, one of them = obikpartition::KmerPartitionto be + built, see Definitions above. Nothing plays this role today; + obikphylo::siblings::cache::PartitionCache and + obikindex::query_layer::QueryLayer each independently reinvent a + fragment of it.
      • +
      • Layer = obilayeredmap::Layer (format-erased: Count/Presence, + each wrapping a TypedLayer<D>) — see "(1) done" below for how this + came to be; TypedLayer<D> ({ mphf: MphfLayer, data: D }, monomorphic) + is the lower-level, D-fixed building block Layer is built on, not + what other crates should reach for directly.
      • +
      • MPHF = MphfLayer.mphf: MemCase<MphfEps> — kmer → slot.
      • +
      • Evidence = MphfLayer.ev: LayerEvidence (Exact/Approx/ + Hybridevidence.bin/fingerprint.bin; see EvidenceKind).
      • +
      • Matrix = TypedLayer<D>.data: DPersistentBitMatrix / + PersistentCompactIntMatrix.
      • +
      +

      Target nesting once KmerPartition exists:

      +
      KmerIndex                                    (obikindex)
      + └─ (opened on demand, per i)  KmerPartition  (obikpartition — not yet built)
      +     └─ layers: Vec<Layer>                    (obilayeredmap)
      +         └─ Layer::Count/Presence(TypedLayer<D>)
      +             └─ TypedLayer<D> { mphf: MphfLayer, data: D }
      +                 ├─ mphf.mphf   → MPHF
      +                 ├─ mphf.ev     → Evidence
      +                 └─ data        → Matrix
      +
      +

      Major restructuring (2026-08-20): KmerPartitions merged into KmerIndex

      +

      Prompted by a direct question: why keep KmerIndex/KmerPartitions split +when, one level down, KmerPartitions is going to directly hold +Vec<KmerPartition> rather than being split again into +"collection-holder" + "collection"? Investigating the actual justification +("KmerPartitions has an independent lifecycle, used before an index +exists") turned out to be falseKmerPartitions::create was called +in exactly one place, inside KmerIndex::create, and every +open_with_config reopen outside KmerIndex's own constructors was a +redundant re-derivation of a KmerPartitions already reachable via +index.partition() (the exact kind of duplication this whole doc has been +tracking). Once that was gone, so was the reason to keep them separate.

      +

      Second correction, from the same conversation: obikpartitionner had +accumulated query/merge/select/rebuild/dump/distance logic that has +nothing to do with partitioning super-kmers — it operates on layers, +which don't exist yet at the phase obikpartitionner is actually +responsible for (scatter → dereplicate → count, all pre-layer). That +logic moved to obikindex, which already depends on obilayeredmap and +never needed obikpartitionner for it. No crate-dependency inversion was +needed — obikindex → obikpartitionner stays the same direction as before.

      +

      Result: +- obikpartitionner (renamed back from obikpartition) now contains only + PartitionRouter (superkmer routing: write/write_batch/flush/ + close, dereplicate, count_kmer, KmerSpectrum) and the + partition_dir(root, i) naming primitive both PartitionRouter and + KmerIndex build on. KmerPartitions no longer exists as a type. +- KmerIndex (obikindex) absorbed KmerPartitions's read-side entirely: + partition_dir/index_dir/layer_dir/partition_meta/n_layers/ + partition_mode/n_partitions (the last now derived from + 2^config.n_bits, no longer a stored, independently-set duplicate field + — kmer_size/minimizer_size used to be double-stored, in both + KmerPartitions and IndexMeta.config, a latent-drift risk flagged + earlier in this doc; now single-sourced from IndexMeta.config). Seven + whole files moved from obikpartitionner into obikindex verbatim as + impl KmerIndex blocks, kept as separate files (not merged into + existing same-topic files): index_layer.rs, query_layer.rs, + merge_layer/, select_layer.rs, rebuild_layer.rs, dump_layer.rs, + plus distance.rs's count_store/presence_store (renamed + matrix_store.rs to avoid colliding with obikindex's own pre-existing + distance.rs), and their shared support (common.rs's load_meta/ + olm_to_sk, filter.rs, graph_pipeline.rs). +- obikphylo::siblings::cache::PartitionCache::build now takes &KmerIndex + directly instead of a separately-opened &KmerPartitions — this deleted + the redundant-reopen pattern at all 8 call sites + (alignment/build/cardinality/distance/entropy×2/ + sankoff_bundle/stats), the same bug flagged earlier in this + conversation as a side effect of investigating the false "independent + lifecycle" claim. +- KmerIndex::partition()/partition_mut() are gone; scatter() + (obikmer) and any write-side code get a transient PartitionRouter via + KmerIndex::partition_router(). +- A real bug caught by the test suite during this move: + PartitionRouter::open initially defaulted to closed: true (inherited + from KmerPartitions::open_with_config's old read-only-reopen + semantics), which broke every write through a router obtained via + partition_router(). Fixed — PartitionRouter is exclusively a + write/processing tool now, so open always starts open.

      +

      Full workspace test suite green (0 failed) after, including all 27 +obikphylo::siblings tests.

      +

      (1) done (2026-08-20): Layer is now the heterogeneous handle, Mat is gone

      +

      Resolved the naming question left open above. Layer<D> (the old +generic/monomorphic type) renamed to TypedLayer<D> throughout +(obilayeredmap, obikindex, obikphylo — 12 files, mechanical) to free +Layer for the type that's actually meant to be everyone's default +handle. obilayeredmap::content_layer::Layer (re-exported at the crate +root) is that type — Count(TypedLayer<PersistentCompactIntMatrix>)/ +Presence(TypedLayer<PersistentBitMatrix>), Layer::open doing the same +disk probe Mat::open used to, find_slot/index_batch/n_cols/ +fill_sub_matrix_carries dispatching per variant exactly as Mat did.

      +

      obikphylo::siblings::cache::Mat deleted outright — PartitionCache now +holds Vec<Vec<obilayeredmap::Layer>> directly. The one sibling-specific +method Mat carried (iter_minorants_batch) is not on obilayeredmap:: +Layer (phylo concepts don't belong in obilayeredmap) — it's an +impl SiblingLayerExt for obilayeredmap::Layer in iter.rs, dispatching +to each variant's existing impl<D: LayerData> SiblingLayerExt for +TypedLayer<D>.

      +

      Full workspace suite green (0 failed) after, including all 27 +obikphylo::siblings tests.

      +

      Still not built: (2) — KmerPartition (singular, one partition's open +Vec<Layer>) and a multi-partition cache in obikpartitionner to replace +obikphylo::siblings::cache::PartitionCache and obikindex::query_layer's +still-separate QueryLayer (which still independently bundles MPHF+matrix, +2-way not using Layer at all). Both remaining consumers now sit one +Layer::open call away from unifying onto (2) once it exists.

      +

      (1b) done (2026-08-20): Layer::Empty — the first non-ready-to-read state

      +

      First step toward Layer representing a layer's whole life, not just the +open-for-reading end of it (see "Definitions" above: KmerPartition will +hold Vec<Layer> regardless of each layer's state, states in between +included). Added one variant:

      +
      pub enum Layer {
      +    Empty { dir: PathBuf },
      +    Count(TypedLayer<PersistentCompactIntMatrix>),
      +    Presence(TypedLayer<PersistentBitMatrix>),
      +}
      +
      +

      Layer::create(dir) makes the directory and returns Empty { dir } — +nothing else; no MPHF/unitigs/evidence construction yet (that's the +deferred next step: build_mphf()/build_unitigs()/build_evidence() +methods to progress Empty → eventually Count/Presence). Empty +carries path accessors so builder code has one place to get +mphf_path()/unitigs_path()/evidence_path()/fingerprint_path()/ +counts_dir()/presence_dir() from, instead of redeclaring the +mphf.bin/unitigs.bin/… filenames at each write site — reusing the +constants layer.rs/mphf_layer.rs already own (COUNTS_DIR/ +PRESENCE_DIR widened from private to pub(crate), file-name constants +already were).

      +

      Every read method (content/evidence_kind/n/find_slot/ +index_batch/n_cols/fill_sub_matrix_carries) panics on Empty with a +one-line message naming the method — confirmed as the right behaviour: +calling any of them on an Empty layer means the caller assumed a layer +was ready when it wasn't, an implementation error to surface loudly, not +a case to design around (Option/Result would let it silently +propagate instead of failing at the actual mistake). Same panic added to +obikphylo::siblings::iter.rs's impl SiblingLayerExt for +obilayeredmap::Layer (4 methods), the one other place that exhaustively +matched Layer's variants.

      +

      Full workspace suite green (cargo check --workspace --all-targets then +cargo test --workspace, exit code 0) after.

      +

      Still deferred, per explicit instruction: build_mphf()/ +build_unitigs()/build_evidence() to progress Empty further, and (2) +— KmerPartition itself — unchanged from above (see "(2a) done" below, +added next).

      +

      (2a) done (2026-08-20): obikpartition crate + KmerPartition

      +

      Built exactly the shape "Definitions" (top of file) specifies, nothing +more — deliberately scoped down from the full "Direction agreed" plan +below: only steps 1–2 (open/n_layers/layer/layers/find), not 3–4 +(migrating PartitionCache/QueryLayer onto it), per explicit +instruction to implement KmerPartition first and decide the wiring +("comment on branche tout ça dans la construction") separately, later.

      +
      pub struct KmerPartition {
      +    layers: Vec<obilayeredmap::Layer>,
      +}
      +
      +impl KmerPartition {
      +    pub fn open(index_dir: &Path, mode: &IndexMode, n_layers: usize, with_counts: bool) -> OLMResult<Self>;
      +    pub fn n_layers(&self) -> usize;
      +    pub fn layer(&self, i: usize) -> &Layer;
      +    pub fn layers(&self) -> &[Layer];
      +    pub fn find(&self, kmer: CanonicalKmer) -> Option<usize>;
      +}
      +
      +

      open takes index_dir/mode/n_layers/with_counts as plain +arguments — no reach-back into KmerIndex (would need obikpartition → +obikindex, the wrong direction) — and builds each layer's path via +obilayeredmap::layer_dir(index_dir, l), the same shared naming +primitive KmerIndex::layer_dir itself delegates to, not a second copy of +the layer_N convention. find mirrors PartitionCache::find's +semantics (first layer that carries the kmer wins) but doesn't yet cover +find_presence_batch/find_presence_batch_fast — those exist only to +serve PartitionCache, so they're part of the (2b) migration, not this +step; building them now against the current sibling-specific tuple shape +(CanonicalKmer, usize, u8, u8) would either bake phylo vocabulary +(family_idx, base) into obikpartition or require deciding a generic +payload shape — a real design fork, deferred to when (2b) is actually +tackled rather than guessed at here.

      +

      Crate deps: obikseq, obilayeredmap only (dev-deps add obiskio, +obicompactvec, tempfile for tests) — matches the "Definitions" +constraint (obikpartition depends on obilayeredmap and below, never +obikindex/obikpartitionner/obikphylo). Registered as a new workspace +member (src/Cargo.toml). 3 new tests (open_reads_every_layer_in_order, +find_reports_the_first_layer_that_carries_the_kmer, +find_returns_none_for_an_absent_kmer). Full workspace suite green +(cargo check --workspace --all-targets then cargo test --workspace) +after.

      +

      Still not done: (2b) — migrating obikphylo::siblings::cache:: +PartitionCache (currently Vec<Vec<Layer>>) and +obikindex::query_layer::QueryLayer (currently uncached, bypasses Layer +entirely) onto KmerPartition/Vec<KmerPartition>; deciding whether that +collection lives in obikpartition or obikindex; deciding the +batch-lookup surface's exact shape (generic payload vs. as-is sibling +tuple moved in wholesale); scan_layer_families's still-independent +PartitionMeta::load (see "Remaining instance…" below) — all explicitly +deferred to whenever wiring is tackled next.

      +

      (3) done (2026-08-20): obikindex ↔ obikpartitionner dependency inverted, PartitionRouter now fills Layer::Empty shells

      +

      Resolved a question left implicit since "Major restructuring": that pass +set the direction obikindex → obikpartitionner (so KmerIndex could +delegate partition_dir to it) without questioning whether that was the +right direction at all. Challenged directly: obikpartitionner is an +algorithm (superkmer routing/dereplication/counting) operating on an +index (KmerIndex, the data structure) — algorithms depend on the data +types they need, not the other way around. [[feedback_no_precedent_defense]] +applied here: "that's the direction we already picked" was not treated as +a justification for keeping it.

      +

      New direction: obikpartitionner → obikindex (+ obilayeredmap, +obipipeline, obiread directly, for what run's pipeline itself needs). +obikindex → obikpartitionner is gone entirely — KmerIndex no longer +imports PartitionRouter/KmerSpectrum in any form. Two path-naming +primitives that used to make this edge necessary moved down a tier instead +of staying put: +- partition_dir/PARTITIONS_SUBDIR moved from obikpartitionner into + obikpartition (the Partition-tier crate KmerPartition already lives + in), alongside a new index_dir(root, i) — both free functions, + mirroring obilayeredmap::layer_dir one tier down. KmerIndex:: + partition_dir/index_dir now delegate here instead of to + obikpartitionner/an inline .join("index"). +- KmerIndex::create/create_skeleton no longer call + PartitionRouter::create to lay out an empty partitions/ skeleton + upfront — turned out to be dead weight once traced: select_layer.rs/ + rebuild_layer.rs already create_dir_all their own partition/layer + directories on demand, and Layer::create's directory-creation covers + the scatter path the same way. Partitions and their layer-0 shells now + come into existence lazily, on first write, with nothing to pre-create. + KmerIndex::create's now-unused force: bool parameter was dropped + (4 call sites updated) rather than left as a dead parameter.

      +

      PartitionRouter reshaped (obikpartitionner/src/partition/router.rs) +around the "création, paramétrage, run()" shape agreed on: new(index: +&mut KmerIndex) -> Self (no disk access), chainable setters +(level_max/theta/workers/max_open, defaults matching the CLI's old +hardcoded values), then run(path_source, on_progress). write/ +write_batch/flush/close/dereplicate/count_kmer stay public, +unconsumed (&self/&mut self, not self) — callers needing fine-grained +control (tests, obikphylo's test harness) still get it, run is a +convenience layered on top, not the only way in.

      +

      run absorbs the entire body of what used to be the free function +obikmer::steps::scatter (now deleted, along with the steps module +entirely) — the obipipeline::make_pipe! two-stage pipeline +(file→pages→superkmers), throttling, per-file logging. What changed: +- Every ensure_writer(partition) call now does Layer::create(&layer0_dir) + (layer0_dir = obilayeredmap::layer_dir(&index.index_dir(i), 0)) before + opening raw.{ext} inside it — raw/dereplicated superkmer files and the + provisional mphf1.bin/counts1.bin/kmer_spectrum_raw.json now live + under <partition>/index/layer_0/, not flat under <partition>/ as + before. This is Layer::Empty actually being used as the "builder code + holding an Empty layer" its own (1b) docs anticipated, not just a shell + with no consumer. + - Caught by an end-to-end smoke test, not by cargo test: this path + move broke obikindex::index_layer::build_index_layer and + remove_build_artifacts, both of which still read/deleted + dereplicated.skmer.zst/mphf1.bin/counts1.bin from + self.partition_dir(i) (the old flat location) — no test in the + workspace suite exercises the real CLI's file-reading scatter path + end-to-end (obikphylo's test harness and obikpartitionner's own + tests both call write_batch directly, bypassing run/file discovery + entirely), so the whole suite stayed green while obikmer index on + real FASTA silently indexed 0 kmers. Found by running the actual CLI + against a small FASTA and noticing count.json's f0 (870, correct) + didn't match "0 total kmers indexed" at the final stage. Fixed by + retargeting both functions to self.layer_dir(i, 0). Lesson, + consistent with the retracted-claim lesson above: a green test suite + is not proof a refactor is correct when no test in it exercises the + specific path that changed — for anything touching the CLI's own + file-driven entry point, running the CLI for real is not optional + verification. +- The internal obisys::spinner("scatter") + hand-rolled EMA-rate display + is gone from the library entirely, replaced by an Option<impl + FnMut(obisys::Progress)> parameter — a new, deliberately generic + progress-reporting type (obisys::Progress { position: u64, total: + Option<u64> }, alongside the existing TracedBar/spinner/ + progress_bar) added specifically so every future algo crate's run() + reports progress the same shape, once, rather than each inventing its + own. total: None here (bases processed isn't knowable without + pre-scanning every input file) — deliberately simpler than the old + in-library rate/file-count/thread-count message; the caller can + recompute a Mbp/s rate from consecutive position values + + wall-clock time itself, which is exactly what cmd/index/mod.rs now + does to reproduce the old spinner message. This is a real, intentional + restriction of the library's job: it reports raw ticks, the CLI decides + what a human sees — same "generic vs. domain-specific" split applied + again, this time to progress reporting rather than to Layer content. + Explicitly not the same mechanism as Stage/Reporter (per + [[feedback_stage_reporter_in_cmd_layer]]): Stage/Reporter measures a + whole call's wall time from outside it; a progress callback has to fire + from inside a loop mid-call, which wrapping from outside cannot + express — two different needs, not the same rule reapplied under a new + name. Stage::start("scatter")/rep.push(...) stayed in + cmd/index/mod.rs, wrapping the whole run() call, unchanged in kind. +- dereplicate/count_kmer keep their existing internal + obisys::progress_bar(...) calls as-is (unconverted to the callback) — + explicitly out of scope for this pass, by agreement.

      +

      Forced, not optional, consequence of the dependency inversion: +KmerIndex::dereplicate_and_count/partition_router/write_spectrum(& +KmerSpectrum) could not stay on KmerIndex at all once obikindex can no +longer name obikpartitionner::{PartitionRouter, KmerSpectrum} in any +position — not a design choice, a mechanical requirement of severing the +edge. Replaced by: KmerIndex::write_spectrum(f0: u64, f1: u64, counts: +&BTreeMap<u32, u64>) (plain values, no KmerSpectrum dependency) and a +new KmerIndex::mark_counted() (symmetric to the already-existing +mark_scattered), with the orchestration itself (router.dereplicate() → +router.count_kmer()write_spectrummark_counted()) now living in +cmd/index/mod.rs, not obikindex.

      +

      Every PartitionRouter::new(&mut index) call in this codebase runs into +the same NLL trap once: PartitionRouter has a Drop impl (auto-close +on scope exit), which extends its &mut KmerIndex borrow to the end of +the enclosing scope even after its last real use — idx.mark_scattered() +right after router.run(...) (or idx.write_spectrum(...) right after +router.count_kmer(...)) fails to borrow-check unless the router is +drop()-ed explicitly first. Hit and fixed identically at all three call +sites that needed it (cmd/index/mod.rs ×2, obikphylo's test harness, +obikpartitionner's own tests).

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0) both before and after the +index_layer.rs fix above — the smoke test is what actually caught the +regression the suite missed.

      +

      (4) done (2026-08-20): obikderep — dereplication split out of obikpartitionner, one algorithm at a time

      +

      Follow-on question after (3): the indexing pipeline has 4 stages (scatter, +dereplicate, count_kmer, index-build — see the CLI's own Reporter output, +one line per stage), but obikpartitionner — a name that says +partitioning — owned three of them (routing, dereplication, counting). +Challenged directly, same as (3)'s dependency-direction question: a crate +should hold what its name says, not accumulate unrelated stages just +because they happened to land there first. Two ways to fix it — one crate +renamed to hold all remaining stages, or one crate per stage — decided in +favour of the latter, explicitly incremental: build the second algo +crate first (obikderep, dereplication only), only then look at what it +and PartitionRouter actually have in common, and factor a shared +Algorithm trait (future obikalgorithm crate) from that real overlap — +not guessed at from a single example. count_kmer and build_layers +(currently KmerIndex inherent methods — itself flagged as inconsistent +with "KmerIndex is a data structure, not a compute structure") are left +alone this round, on purpose — one stage moves at a time.

      +

      obikderep (new crate): Dereplicator<'a> { index: &'a KmerIndex, n_partitions, level } +— new(index: &KmerIndex) (shared borrow, not &mut: dereplication never +writes index metadata), no setters yet (nothing to configure), run(on_progress) +does the two-phase split+merge dereplication in parallel across partitions, +ported unchanged from PartitionRouter::dereplicate (moved wholesale: +optimal_buckets/dereplicate_partition/load_bucket/flush_map/ +remove_skmer_file, now private to this crate in dereplicate.rs). +obikpartitionner::PartitionRouter::dereplicate is gone; count_kmer +stays.

      +

      A real signature difference from PartitionRouter::run, not an +inconsistency: Dereplicator::run takes Option<impl Fn(Progress) + +Sync>, not FnMut. PartitionRouter::run's callback is invoked from one +sequential loop (FnMut is fine); Dereplicator::run's work is +rayon::par_iter, so the callback can be invoked concurrently from +multiple worker threads — same reason obisys::TracedBar's own methods +take &self, not &mut self. Progress position is tracked with an +AtomicU64, incremented from inside the parallel closure so each +completed partition reports immediately — collecting all results first and +reporting after (the first draft of this) would have delivered every tick +in one burst at the very end, defeating the point of a live progress bar. +total: Some(n_partitions) (known up front, unlike scatter's bases count) +— cmd/index/mod.rs renders a real progress_bar, not a spinner, driven +by the callback exactly like scatter's spinner is.

      +

      A second, pre-existing instance of the exact bug (3) fixed, caught +before it shipped: dereplicated.skmer.zst was hand-built as a string +literal independently in three places — obikpartitionner's +dereplicate.rs/count.rs and obikindex's index_layer.rs (a literal +that already predated this session, never caught until now). Splitting +dereplication into its own crate turns this from "two places, still +matching by luck" into "three independent crates that must agree on a +filename with no shared dependency forcing them to" — no longer +deferrable. Fixed by adding obilayeredmap::{raw_superkmers_path, +dereplicated_superkmers_path} (free functions, layer_dir: &Path -> +PathBuf, mirroring layer_dir itself) — the filename lives in one place, +in the Layer-tier crate every consumer here already depends on +(obikpartitionner, obikderep, obikindex all reach it without a new +edge), and no external crate ever sees the literal "skmer.zst" again. +This reverses (3)'s own earlier call to keep SK_EXT private to +obikpartitionner — that call assumed a single owner; a second owner +appearing (obikderep) removed the assumption it rested on, so the +decision changed with it, not out of inconsistency.

      +

      Every count_kmer call site that used to run after router.dereplicate() +on the same PartitionRouter now runs after a separate +Dereplicator::new(&idx).run(...) call, on a freshly-constructed +PartitionRouterPartitionRouter no longer offers a combined +"dereplicate then count" path. Updated at all three call sites that had +one: cmd/index/mod.rs, obikphylo's test harness, obikpartitionner's +own tests.

      +

      Full workspace suite green (cargo check --workspace --all-targets + +cargo test --workspace, exit code 0), plus an end-to-end CLI smoke test +against real FASTA data (scatter → dereplicate → count → index-build → +query, same numbers as (3)'s smoke test: 870 kmers) — required this time +too, per (3)'s own lesson: no test in the suite exercises obikmer index's +real file-driven path.

      +

      Still not done: count_kmer/build_layers staying where they are, the +obikalgorithm shared-trait extraction (deliberately deferred until a +third data point exists), and everything already listed under (2b).

      +

      (5) design agreed, not yet implemented (2026-08-20): KmerPartition rewritten, Layer gains self-naming, a future cache crate over KmerIndex

      +

      Session ended (out of budget) before any of this was coded. Everything +below is a fully specified plan, agreed sentence by sentence with the +user — not a sketch to re-derive, not a proposal to re-litigate. Implement +it as written; if something here turns out to be wrong once coded, fix it +and update this section, don't restart the design conversation.

      +

      How this was found

      +

      Direct question from the user: "tu as bien créé une structure +KmerPartition ?" — yes (2a), but investigating exposed that it is +wired into nothing. KmerIndex has no partition(i) method at all; +partition_dir/index_dir/layer_dir still call obikpartition:: +partition_dir/index_dir and obilayeredmap::layer_dir as bare free +functions directly, never touching a KmerPartition/Layer object to get +there. The end result on disk is identical (same paths), which is exactly +why no test caught it — but the responsibility is in the wrong place: +one function (on KmerIndex) knows the whole three-tier naming +convention, instead of each tier asking the one below it for its own +path. User's framing, verbatim, now saved as [[feedback_no_spaghetti_petits_pois]]: +"spaghetti" (logic untraceable, split across too many unrelated crates) +and "petits pois" (small bits of naming logic dispersed with no owning +object) are strictly forbidden — this was a live example of both.

      +

      Pushed further, twice: +1. First correction: Layer::create(&obilayeredmap::layer_dir(&dir, 0)) — + still a free-function call from outside Layer to compute where it + should live. "Le layer n'est pas con, c'est lui qui dit où est-ce qu'il + doit être sauvé" (the layer isn't stupid, it says itself where it + should be saved). +2. Second correction, the general principle: "une partition est juste + identifiée par un numéro, tout se calcule à partir du numéro, et un + layer est identifié à partir d'un numéro et tout se calcule à partir de + ce numéro." Concretely: each object stores its own local identifying + number plus its immediate parent's path (captured once, at + construction) — never a path handed in again later by a caller, and + never a free function outside the object that can compute that path + independently. Explicitly rejected along the way: making users pass + "the partition's path that contains the layer" to open a layer — the + parent path is captured once, at the child's construction, not + re-supplied at every call.

      +

      The agreed shape

      +

      Layer (obilayeredmap) — identified by l + its parent partition's +directory, both captured at construction, never received again:

      +
      pub enum Layer {
      +    Empty { partition_dir: PathBuf, l: usize },   // pure identification, no disk I/O
      +    Count(TypedLayer<PersistentCompactIntMatrix>),
      +    Presence(TypedLayer<PersistentBitMatrix>),
      +}
      +
      +impl Layer {
      +    pub fn at(partition_dir: &Path, l: usize) -> Self;                 // identify only
      +    fn dir(&self) -> PathBuf;                                          // private — layer_dir() no longer a public free function, folded in here
      +    pub fn create(self) -> io::Result<Self>;                           // creates the directory if needed; no path parameter anymore
      +    pub fn open(self, mode: &IndexMode, with_counts: bool) -> OLMResult<Self>;  // no path parameter anymore
      +    // mphf_path()/unitigs_path()/evidence_path()/fingerprint_path()/counts_dir()/presence_dir()
      +    // unchanged in spirit, implemented via self.dir() instead of a stored `dir` field read directly
      +}
      +
      +

      Note this replaces Layer::Empty { dir: PathBuf } from (1b) — dir +becomes a computed value (partition_dir.join(format!("layer_{l}"))), not +a stored field. obilayeredmap::layer_dir/raw_superkmers_path/ +dereplicated_superkmers_path (currently public free functions, +introduced in (3)/(4)) stop being called from outside obilayeredmap +entirely once this lands — they were the right fix for their moment (a +second crate, obikderep, needed to agree on a filename with no owner), +but the real fix, now visible with a third data point, is that Layer +itself should be the only thing anyone asks.

      +

      KmerPartition (obikpartition) — same principle, one tier up:

      +
      pub struct KmerPartition {
      +    index_root: PathBuf,   // the parent KmerIndex's root, captured once
      +    i: usize,
      +}
      +
      +impl KmerPartition {
      +    pub fn new(index_root: PathBuf, i: usize) -> Self;                 // identify only, no disk I/O
      +    pub fn create(index_root: PathBuf, i: usize) -> io::Result<Self>;  // creates this partition's directory + an empty layer 0 (a partition is never born without one — that knowledge lives here, not in whoever calls create)
      +    pub fn partition_dir(&self) -> PathBuf;                            // part_{i:05}
      +    pub fn index_dir(&self) -> PathBuf;                                // part_{i:05}/index
      +    pub fn layer(&self, l: usize) -> Layer;                            // Layer::at(&self.index_dir(), l) — caller never touches a path
      +    pub fn meta(&self) -> SKResult<PartitionMeta>;                     // n_layers + mode; must absorb the recovery-on-missing-file logic
      +                                                                         // currently private in obikindex::common::load_meta (obikpartition
      +                                                                         // can't depend on obikindex to reuse it — this logic moves down)
      +    pub fn n_layers(&self) -> SKResult<usize>;                         // meta()?.n_layers
      +    pub fn mode(&self) -> SKResult<IndexMode>;                         // meta()?.mode — "exact/approximatif"
      +    pub fn is_filled(&self) -> bool;                                   // does this partition's directory exist at all
      +    pub fn n_kmers(&self) -> io::Result<usize>;                        // LayerMeta::load(&self.layer(0).dir()).n — reads layer 0's count as
      +                                                                         // a representative figure, same "read the first one" trick
      +                                                                         // n_layers_per_partition() already uses at the KmerIndex level
      +}
      +
      +

      This replaces (2a)'s KmerPartition { layers: Vec<Layer> } entirely +— no eagerly-opened Vec<Layer>, no find() (both belong to the future +cache, see below, which is the thing that actually holds opened layers +alive across many lookups). (2a)'s version is safe to delete outright: it +was never wired into anything (confirmed above), so nothing depends on +its current shape. New dependencies needed: obikpartition gains +obiskio (for SKResult) and obicompactvec (for LayerMeta).

      +

      Deliberately not built this round: cross-level consistency checks +("verify everything below me is in the same state") — a real idea, raised +by the user, but nothing concrete needs it yet; building it speculatively +would be exactly the premature-abstraction pattern this project avoids.

      +

      KmerIndex (obikindex) — becomes the sole entry point:

      +
      pub fn partition(&self, i: usize) -> KmerPartition;   // KmerPartition::new(self.root_path.clone(), i)
      +
      +

      partition_dir(i)/index_dir(i)/layer_dir(i, l) stay as public +methods (≈30 existing call sites across obikindex/obikphylo — see (3)'s +option A, applied identically here) but become pure delegations: +self.partition(i).partition_dir(), self.partition(i).index_dir(), +self.partition(i).layer(l).dir() (needs Layer::dir() to be visible +enough for this — likely pub(crate) in obilayeredmap plus a thin +public wrapper, or a public accessor on Layer itself; not fully nailed +down, decide while implementing). No caller outside obikindex changes.

      +

      Known blast radius (why this wasn't done in the same session)

      +
        +
      • 14 files call Layer::open/Layer::create directly today + (obikphylo/siblings/{cache,build,family_scan,tests}.rs, + obikpartitionner/partition/router.rs, obikpartition/src/lib.rs, + obikindex/{rebuild_layer,dump_layer,index,query_layer}.rs, + obilayeredmap/{mphf_layer,layer,map,content_layer}.rs) — every one + loses its path parameter and gains a (partition_dir, l) or an + already-identified Layer to call .create()/.open() on instead.
      • +
      • ≈30 files call KmerIndex::partition_dir/index_dir/layer_dir — + unaffected in their own code (same public signatures), but worth + re-checking once (5) lands that none of them were relying on the old + free-function-based implementation in a way the new delegation breaks.
      • +
      • obikpartitionner::PartitionRouter::ensure_writer and obikderep's + run both currently call obilayeredmap::{layer_dir, raw_superkmers_path, + dereplicated_superkmers_path} directly (from (3)/(4)) — both need to + switch to going through index.partition(i).layer(0) instead.
      • +
      +

      Also agreed, separately: PartitionRouter::new never needed &mut KmerIndex

      +

      Verified by reading the code: every call PartitionRouter makes on +index is &self (index.kmer_size(), index.index_dir(i)). The &mut +in its current signature (from (3)) was inherited from the original +"the router writes to the partitions" reasoning, never actually required +by any method call. This is exactly what caused every drop(router) +workaround needed throughout (3)/(4) (cmd/index/mod.rs ×2, obikphylo's +test harness, obikpartitionner's own tests) — PartitionRouter holds a +Drop impl, which extends a &mut borrow to the end of its scope even +past its last real use. Fix alongside (5): change +PartitionRouter::new(index: &'a mut KmerIndex) to &'a KmerIndex, and +remove the now-unnecessary drop(router) calls at all four sites.

      +

      Also discussed: a future cache crate, not part of (5), not obikalgorithm either

      +

      Separate idea, explicitly not part of this design and not started: +a new crate whose only job is to cache open KmerPartitions (and their +opened Layers) across one run — replacing both obikphylo::siblings:: +cache::PartitionCache (today, sibling-specific, holds Vec<Vec<Layer>>) +and obikindex::query_layer::QueryLayer (today, uncached, bypasses +Layer entirely) — the two consumers (2b) already identified as each +reinventing a fragment of the same thing.

      +

      User's framing: this is not a third obikalgorithm data point — an +algorithm has a new → run → done shape; a cache has a fundamentally +different one (open, stay alive for a whole run, serve lookups, maybe +evict) — "on crée un cache sur un index, ça consomme un index." Two +distinct crate roles in this ecosystem (data crates: obikpartition/ +obilayeredmap; algorithm crates: obikpartitionner/obikderep/future +obikalgorithm implementors; and now a cache/service crate), not one +unified shape to force everything into.

      +

      Depends on (5) being done first: the cache crate's whole job is holding +Vec<KmerPartition>/opened Layers alive, built via KmerIndex:: +partition(i) as its factory — nothing to build it on top of until (5) +lands. Still open once (5) is done: eviction policy vs. holding everything +open for the process lifetime (the never-measured mmap/VM-mapping-count +question from earlier in this doc), and whether it lives in obikpartition +itself or a new crate.

      +

      Order of remaining work, as currently understood

      +
        +
      1. (5)Layer/KmerPartition/KmerIndex rewrite described above, + plus the PartitionRouter &mut& fix (same root cause, same + session, do together).
      2. +
      3. The future cache crate (name not chosen), consuming KmerIndex:: + partition(i) — unblocks migrating PartitionCache/QueryLayer (2b).
      4. +
      5. obikalgorithm — still deliberately waiting for a third run()-shaped + data point (count_kmer or build_layers migrating out of + KmerIndex/PartitionRouter) before extracting a shared trait; two + examples were judged not enough to be sure of the shape (Fn+Sync vs + FnMut callback bound already diverged between the two that exist).
      6. +
      +

      The problem

      +

      Reading a layer's data (MPHF + matrix) is not free: MphfLayer::open mmaps +mphf.bin plus (evidence.bin/fingerprint.bin + unitigs.bin), and the +matrix side mmaps matrix.pbmx/matrix.pcmx (or one file per genome column +if not yet packed). Any code path that reopens a layer per lookup instead of +once per run pays this cost repeatedly.

      +

      obikphylo::siblings::cache::PartitionCache was built to avoid exactly this +for build_sibling_annex/sibling_annex_stats: those commands probe many +partitions, once per source layer, over the whole run. Profiling a real run +showed wall-clock time dominated by repeated open()/mmap syscalls, not +computation — parallelising the naive per-lookup opens spread the cost +across cores without reducing it. PartitionCache::build opens every +partition's every layer once, up front, in parallel, and keeps the handles +alive for the run.

      +

      Three independent implementations of the same bundle (historical — (1) fixed this)

      +

      As of 2026-08-20 this table describes the pre-(1) state. Mat no +longer exists (deleted when obilayeredmap::Layer replaced it — see "(1) +done" above); Layer<D> in the table below is what's now called +TypedLayer<D>. QueryLayer is unaffected and still stands as described — +still uncached, still not using Layer at all — which is exactly what (2) +needs to fix. Kept for the original motivation, not as current fact:

      +

      Searching the codebase for "who bundles MPHF + matrix, with per-layer format +auto-detection" turned up three unrelated implementations:

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      lives inscopecached?
      Layer<D> (now TypedLayer<D>)obilayeredmapone layer, D fixed at compile timeheld alive by whoever owns the Layer, no policy of its own
      Mat (now deleted; superseded by obilayeredmap::Layer)obikphylo::siblings::cacheone layer, format resolved per instance from an enum of 3 Layer<D> variantsyes, via PartitionCache
      QueryLayer (unchanged, still current)obikindex::query_layer (moved crates since this was written — see "Major restructuring")one layer, (MphfLayer, PersistentBitMatrix\|PersistentCompactIntMatrix) pair, bypasses TypedLayer<D>/Layer entirelyno — opened fresh inside query_partition_with on every call
      +

      query_partition_with is obikmer query's normal query path — the one +most exposed to repeated cross-partition lookups — and it is still the one +with no cache at all. obikphylo built a cache first only because +sibling-annex construction hits the cost hardest, not because the need is +sibling-specific.

      +

      The gap in obilayeredmap's existing cache (historical — (1) fixed this)

      +

      obilayeredmap::LayeredMap<D> already caches correctly at the granularity +of one partition: open(root) opens every layer once, keeps +Vec<TypedLayer<D>> alive for the LayeredMap's lifetime. But it is +monomorphic — every layer in the Vec must share the same concrete D. +In practice this was false: layers in the same partition are packed +independently over time (pack --sparse converts one layer's presence +matrix at a time). This motivated (1) — obilayeredmap::Layer, done — but +note the specific PersistentSparseBitMatrix-mixing scenario described +here turned out to be moot: PersistentBitMatrix itself absorbed sparse +storage as a 4th internal variant before (1) was built (see "One bug found +… one earlier claim retracted" below), so the only heterogeneity Layer +actually needs to represent is Count vs. Presence, not dense-vs-sparse +presence. LayeredMap<D> itself is unaffected by any of this — it's still +monomorphic, still not used by Layer/KmerPartition (which bypass it +entirely, opening each TypedLayer<D> directly, the same way Mat did).

      +

      Resource cost: mmap does not hold a file descriptor

      +

      Before deciding how many layers/partitions a cache may hold open +simultaneously, the binding constraint needs to be identified correctly.

      +

      Confirmed against upstream documentation, not inferred from behaviour:

      +
      +

      "After the mmap() call has returned, the file descriptor, fd, can be +closed immediately without invalidating the mapping." +— mmap(2), man7.org

      +

      "The close(2) function does not unmap pages" +— mmap(2), Apple Developer

      +

      "A file backed Mmap ... will remain valid even after the File is dropped. +... the Mmap handle is completely independent of the File used to create +it." +— memmap2::Mmap, docs.rs

      +
      +

      Every read-only mmap in this codebase already follows this: Mmap::map(&File::open(path)?)? +— the File is a temporary, dropped (fd closed) immediately after the +mapping is established; every persistent struct (PersistentBitVec, +PersistentCompactIntVec, PackedBitMatrix, Evidence, FingerprintVec, +...) stores only the Mmap, never the File. So a cache built on these +types does not consume the process's open-file-descriptor budget +(ulimit -n, notoriously low by default on macOS) proportionally to how +many mmapped files it holds.

      +

      It does consume a different resource — the process's virtual-memory mapping +table (one entry per active mmap() region). Linux exposes this as +vm.max_map_count (default 65530). No documented macOS equivalent (fixed +numeric ceiling) was found; the constraint there appears to be virtual +address space rather than an explicit mapping counter, but this is not +sourced and should not be assumed. This is the resource actually worth +measuring before deciding on cache size, not fd count — and it is why +packing (matrix.pbmx/matrix.pcmx, one mmap for all columns) matters +independently of any caching decision: an unpacked Columnar matrix opens +one mmap per genome column, multiplying the mapping count a cache would +have to hold by n_genomes.

      +

      Layering: who owns what (superseded — see Definitions above)

      +

      This section used to argue nobody owned "the collection of partitions." +That's resolved: KmerIndex (obikindex) owns it now, directly (see +"Major restructuring" below). What's still genuinely unowned is one +partition's open layersKmerPartition, in the not-yet-created +obikpartition — see "Definitions" at the top of this file for the +current, authoritative answer. Left here only so old links/references to +this heading don't 404; don't read this section for current facts.

      +

      Direction agreed, not yet implemented

      +

      Only (2) remains — (1) shipped as obilayeredmap::Layer (see "(1) done" +above). Concretely, in order:

      +
        +
      1. Create the obikpartition crate (obikindex → obikpartition → + obilayeredmap, no other edges — see "Definitions" above for the exact + constraint and why).
      2. +
      3. KmerPartition { layers: Vec<obilayeredmap::Layer> }open, + n_layers, layer(i), find, plus whatever batch-lookup surface + obikphylo::siblings::cache::PartitionCache currently needs + (find_presence_batch/find_presence_batch_fast; fast_mode is + sibling-specific bookkeeping and should probably stay in obikphylo, + wrapping a KmerPartition/Vec<KmerPartition> rather than living + inside it — same "generic vs. domain-specific" split iter_minorants_batch + already went through for Layer in (1)).
      4. +
      5. Migrate obikphylo::siblings::cache::PartitionCache to hold + Vec<KmerPartition> instead of Vec<Vec<Layer>>.
      6. +
      7. Migrate obikindex::query_layer::QueryLayer/query_partition_with to + use KmerPartition too, closing the "no cache at all" gap on + obikmer's normal query path (see "Three independent implementations," + historical, above).
      8. +
      +

      Open before implementing: exact API shape of KmerPartition (propose, +confirm before coding — non-trivial), and whether the multi-partition +Vec<KmerPartition> needs an eviction policy or can simply hold every +partition open for the process lifetime (revisit once the +VM-mapping-count question above has a real number behind it for this +codebase's scale — still not measured).

      +

      Preparatory work done (2026-08-20)

      +

      Groundwork for (1)/(2), landed ahead of the design itself. Note: at +the time this was written, the collection type these bullets describe was +named KmerPartition (singular) in this doc; it was renamed +KmerPartitions (plural) shortly after, then deleted entirely and merged +into KmerIndex (see "Major restructuring" above). The bullets below are +edited to say KmerPartitions throughout, to not collide with the +unrelated, brand-new singular KmerPartition defined at the top of this +file — the accessors described here live on KmerIndex today, not on +any type called KmerPartition.

      +
        +
      • KmerPartitions (obikpartitionner, at the time) gained + partition_dir/index_dir/layer_dir as the single source of truth + for a partition's on-disk layout, replacing per-module duplicated + const INDEX_SUBDIR: &str = "index" (7 copies) and ad hoc path joins — + including one found duplicated inside the struct itself + (ensure_writer rebuilt part_dir's own logic by hand).
      • +
      • Same struct gained partition_meta/n_layers/index_mode, wrapping + obilayeredmap::meta::PartitionMeta::load (via the existing + common::load_meta, which also recovers indexes built before + meta.json existed). Before this, obikphylo and obikindex imported + obilayeredmap::meta::PartitionMeta directly and called ::load() + themselves at 21 call sites, each redoing its own error-mapping — + every one of those crates knew the on-disk metadata format instead of + going through an interface. Fixed everywhere except one remaining spot + (below). Caught as a side effect: dump_layer.rs/query_layer.rs had + been calling PartitionMeta::load directly, bypassing load_meta + entirely — they never got the missing-meta.json recovery the other + callers did.
      • +
      • Layer introspection API discussed but not yet implemented — three + axes, deliberately kept separate after an initial draft conflated them:
      • +
      • LayerContent { Count, Presence } — what the layer stores; a const + on LayerData (compile-time, zero-cost), not a runtime field.
      • +
      • StorageKind { Implicit, Columnar, Packed, Sparse } — how it's + stored; only meaningful for D that actually carry data (Layer<()> + has neither this nor LayerContent — it's a write-time-only state, + never a queryable content: once a layer is closed, "no matrix file" + reads back as Presence/Implicit via PersistentBitMatrix::open's + own fallback, not as some third "empty" content).
      • +
      • EvidenceKind { Exact, Approx, Hybrid } — from MphfLayer's own + already-in-memory LayerEvidence discriminant.
      • +
      • Not all (LayerContent, StorageKind) pairs are legal: Count never + has Implicit or Sparse.
      • +
      +

      Implemented (2026-08-20). LayerContent/StorageKind/EvidenceKind +now exist, each with two forms: +- A runtime accessor on an already-open value (Layer<D>::content()/ + storage_kind()/evidence_kind(), PersistentBitMatrix::storage_kind(), + PersistentCompactIntMatrix::storage_kind(), MphfLayer::evidence_kind()) + — reads a discriminant already in memory, zero disk access. +- A lightweight detect()/detect_storage() disk probe that mirrors the + corresponding open()'s own priority order by hand (file-existence + checks only, no mmap) — usable before committing to a D, unlike the + runtime accessors. Exposed per-layer on LayeredMap<D> as + detect_layer_content/detect_layer_storage/detect_layer_evidence + (work regardless of D, since they only use self.root + the layer + index).

      +

      StorageKind lives in obicompactvec (owner of PersistentBitMatrix/ +PersistentCompactIntMatrix); LayerContent/EvidenceKind live in +obilayeredmap. HasLayerContent/HasStorageKind gate Layer<()> out of +content()/storage_kind() (no matrix, nothing to report), matching the +"empty is transitional" conclusion above. 42 new tests across +obilayeredmap's tests/layer.rs and tests/map.rs; full workspace +suite green (0 failed) after.

      +

      Not done: these detect() probes don't yet replace Mat::open's or +QueryLayer::open's own hand-rolled equivalents (still duplicated content/ +storage decisions, now a third copy of the same logic to keep in sync) +— that consolidation is (1)/(2)'s job, not this prep step's.

      +

      One bug found while reading around this (signalled, not fixed); one earlier claim retracted

      +
        +
      • obicompactvec::bitmatrix::sparse.rs's module doc says + "Not used by any production code path yet" — false since obikmer pack + --sparse (cmd/pack/mod.rs) is wired to pack_sparse_bit_matrix and + Mat::open already reads the result back in the sibling-annex path. + Stale comment, not corrected.
      • +
      • Retracted (2026-08-20): an earlier pass through this doc claimed + obikpartition::query_layer::QueryLayer::open had no sparse-format + detection and would silently corrupt reads on a pack --sparsed layer. + False — PersistentBitMatrix (obicompactvec::bitmatrix::persistent) + is a 4-way enum (Columnar/Packed/Sparse/Implicit), not 3-way as + first read; its open() already detects Sparse via + presence/sparse_meta.json, and every method on the type (row, + fill_row, nonzero_iter, …) already dispatches all 4 arms. + QueryLayer::open's PersistentBitMatrix::open(layer_dir) call was + never the bug. Root cause of the false claim: a grep -n + "Implicit\|Columnar\|Packed" used to read the enum definition silently + skipped the Sparse(...) line because it matched none of those three + words — a self-inflicted blind spot from a filtered read, not a fact + about the code. Lesson: for a pub enum whose variant list matters, + read the definition unfiltered, don't grep for the variant names you + expect to find.
      • +
      • One real consequence of that same correction, fixed (2026-08-20): + obikphylo::siblings::cache::Mat::SparsePresence(Layer< + PersistentSparseBitMatrix>) was redundant — Mat::Presence(Layer< + PersistentBitMatrix>) alone already handles sparse layers + transparently, since PersistentBitMatrix absorbs Sparse internally. + Removed the variant, the presence/is_multi.prsb probe in Mat::open + (now just opens Layer::<PersistentBitMatrix> unconditionally for the + non-count case — sparse-vs-dense is PersistentBitMatrix::open's own + concern), and every now-single-armed match in find_slot/index_batch/ + iter_minorants_batch/n_cols/fill_sub_matrix_carries. Full + workspace test suite green after, including the 27 obikphylo::siblings + tests that exercise pack_matrices(true)/sparse through Mat.
      • +
      +

      Remaining instance of the PartitionMeta-encapsulation problem

      +

      obikphylo::siblings::family_scan::scan_layer_families still re-derives +index_dir from layer_dir.parent() and calls PartitionMeta::load +itself, purely to get .mode for obilayeredmap::Layer::open (was +Mat::open, same gap, survived the MatLayer swap in (1) unchanged). +Fixing it the way the 21 other call sites were fixed needs more than a 1:1 +swap: scan_layer_families only receives a bare layer_dir: &Path, not a +(partition, part, layer) triple, and its single upstream source of layer +paths, sibling_layer_dirs, returns a flat Vec<PathBuf> with the +partition/layer indices already discarded. Fixing it properly means either +having sibling_layer_dirs return (PathBuf, IndexMode) (or (part, +layer)) pairs, or threading &KmerIndex + indices through instead of +paths (not &KmerPartition — that type doesn't exist yet, and once it +does it still won't know IndexMode, which lives on KmerIndex/ +PartitionMeta) — and touching every one of scan_layer_families's 8 +callers (distance.rs, alignment.rs, cardinality.rs, entropy.rs ×2, +sankoff_bundle.rs ×2, stats.rs). Left alone this round; worth doing as +part of the same pass that builds KmerPartition, since those callers are +exactly the sibling-annex consumers it's meant to serve.

      + + + + + + + + + + + + + +
      +
      + + + +
      + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/DevDoc/implementation/persistent_bit_vec.refs/index.html b/DevDoc/implementation/persistent_bit_vec.refs/index.html index d4d77cc0..452c7db1 100644 --- a/DevDoc/implementation/persistent_bit_vec.refs/index.html +++ b/DevDoc/implementation/persistent_bit_vec.refs/index.html @@ -727,14 +727,14 @@
    10. - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
    11. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    12. + + + + diff --git a/DevDoc/implementation/persistent_bit_vec/index.html b/DevDoc/implementation/persistent_bit_vec/index.html index 17aaf5e9..3e40d825 100644 --- a/DevDoc/implementation/persistent_bit_vec/index.html +++ b/DevDoc/implementation/persistent_bit_vec/index.html @@ -733,14 +733,14 @@
    13. - + - obilayeredmap crate + obikindex layer tier @@ -1253,6 +1253,34 @@ + + + + + + +
    14. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    15. + + + + diff --git a/DevDoc/implementation/persistent_compact_int_vec.refs/index.html b/DevDoc/implementation/persistent_compact_int_vec.refs/index.html index 1eb1fdc4..9e306acb 100644 --- a/DevDoc/implementation/persistent_compact_int_vec.refs/index.html +++ b/DevDoc/implementation/persistent_compact_int_vec.refs/index.html @@ -727,14 +727,14 @@
    16. - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
    17. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    18. + + + + diff --git a/DevDoc/implementation/persistent_compact_int_vec/index.html b/DevDoc/implementation/persistent_compact_int_vec/index.html index 150a73c4..05ea4d9f 100644 --- a/DevDoc/implementation/persistent_compact_int_vec/index.html +++ b/DevDoc/implementation/persistent_compact_int_vec/index.html @@ -9,7 +9,7 @@ - + @@ -733,14 +733,14 @@
    19. - + - obilayeredmap crate + obikindex layer tier @@ -1236,6 +1236,34 @@ + + + + + + +
    20. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    21. + + + + diff --git a/DevDoc/implementation/pipeline.refs/index.html b/DevDoc/implementation/pipeline.refs/index.html index c2dc4f46..3ab74967 100644 --- a/DevDoc/implementation/pipeline.refs/index.html +++ b/DevDoc/implementation/pipeline.refs/index.html @@ -727,14 +727,14 @@
    22. - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
    23. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    24. + + + + @@ -1242,12 +1270,12 @@

      Coverage: implementation/pipeline.md

      Code couvert

        -
      • obikpartitionner/src/partition.rs — estimation des paramètres (phase 0)
      • +
      • obikindex/src/partition/partition.rs — estimation des paramètres (phase 0)
      • obiskbuilder/src/iter.rs — scatter : filtre entropie, extraction superkmers, routage partition (phase 1)
      • -
      • obikpartitionner/src/filter.rs — déduplication bucket-sort (phase 2)
      • -
      • obikpartitionner/src/kmer_sort.rs — tri externe + agrégation de comptages (phase 3)
      • +
      • obikindex/src/partition/filter.rs — déduplication bucket-sort (phase 2)
      • +
      • obikindex/src/partition/kmer_sort.rs — tri externe + agrégation de comptages (phase 3)
      • obidebruinj/src/debruijn.rs — graphe De Bruijn, extraction des unitigs (phase 5)
      • -
      • obikpartitionner/src/index_layer.rs — construction MPHF + évidence (phase 6), paramètre block_bits
      • +
      • obikindex/src/partition/index_layer.rs — construction MPHF + évidence (phase 6), paramètre block_bits
      • obikindex/src/index.rsbuild_layers(), dereplicate_and_count()

      Notes

      diff --git a/DevDoc/implementation/pipeline/index.html b/DevDoc/implementation/pipeline/index.html index c4d8a225..a697eee7 100644 --- a/DevDoc/implementation/pipeline/index.html +++ b/DevDoc/implementation/pipeline/index.html @@ -857,14 +857,14 @@
    25. - + - obilayeredmap crate + obikindex layer tier @@ -1102,6 +1102,34 @@ + + + + + + +
    26. + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
    27. + + + + @@ -1574,7 +1602,7 @@ branching / dead-end → unitig start or end counts/ ← PersistentCompactIntMatrix (with_counts only)

    Cleanup: unless --keep-intermediate is set, remove_build_artifacts deletes dereplicated.skmer.zst, mphf1.bin, and counts1.bin after all partitions are indexed.

    -

    See obilayeredmap and MPHF selection for data structure details.

    +

    See obikindex::layer and MPHF selection for data structure details.

    Query path (exact evidence):

    query kmer q
       → canonical_minimizer(q) → hash → PART → part_XXXXX/
    diff --git a/DevDoc/implementation/select/index.html b/DevDoc/implementation/select/index.html
    index 3199a4e0..97f4f7c1 100644
    --- a/DevDoc/implementation/select/index.html
    +++ b/DevDoc/implementation/select/index.html
    @@ -733,14 +733,14 @@
       
       
         
  • - + - obilayeredmap crate + obikindex layer tier @@ -1224,6 +1224,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/storage.refs/index.html b/DevDoc/implementation/storage.refs/index.html index d5387f03..03bfe3b3 100644 --- a/DevDoc/implementation/storage.refs/index.html +++ b/DevDoc/implementation/storage.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1244,7 +1272,7 @@
    • obikindex/src/meta.rs — IndexMeta, IndexConfig (version, config, genomes)
    • obikindex/src/index.rs — layout sur disque : partitions/, index.meta
    • -
    • obilayeredmap/src/meta.rs — LayerMeta (evidence kind), PartitionMeta (n_layers)
    • +
    • obikindex/src/layer/meta.rs — LayerMeta (evidence kind), PartitionMeta (n_layers)
    • obiskio/src/unitig_index.rs — fichiers unitigs.bin + unitigs.bin.idx

    Notes

    diff --git a/DevDoc/implementation/storage/index.html b/DevDoc/implementation/storage/index.html index e526bad4..7fd77097 100644 --- a/DevDoc/implementation/storage/index.html +++ b/DevDoc/implementation/storage/index.html @@ -929,14 +929,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1174,6 +1174,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/superkmer.refs/index.html b/DevDoc/implementation/superkmer.refs/index.html index 9ea4dbbc..946e4da1 100644 --- a/DevDoc/implementation/superkmer.refs/index.html +++ b/DevDoc/implementation/superkmer.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/superkmer/index.html b/DevDoc/implementation/superkmer/index.html index 4695129f..d5267a43 100644 --- a/DevDoc/implementation/superkmer/index.html +++ b/DevDoc/implementation/superkmer/index.html @@ -835,14 +835,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1080,6 +1080,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/implementation/unitig_evidence.refs/index.html b/DevDoc/implementation/unitig_evidence.refs/index.html index d1a1e6ff..1dfbaa9c 100644 --- a/DevDoc/implementation/unitig_evidence.refs/index.html +++ b/DevDoc/implementation/unitig_evidence.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1243,7 +1271,7 @@

    Code couvert

    • obiskio/src/unitig_index.rs — format unitigs.bin + unitigs.bin.idx, UnitigFileWriter, UnitigFileReader, build_unitig_idx(), DEFAULT_BLOCK_BITS=0, chemin chaud block_bits=0 dans chunk_start()
    • -
    • obilayeredmap/src/evidence.rs — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
    • +
    • obikindex/src/layer/evidence.rs — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter
    • obidebruinj/src/debruijn.rs — extraction unitigs, chunking à MAX_KMERS_PER_CHUNK

    Notes

    diff --git a/DevDoc/implementation/unitig_evidence/index.html b/DevDoc/implementation/unitig_evidence/index.html index a59911ec..a66adbd3 100644 --- a/DevDoc/implementation/unitig_evidence/index.html +++ b/DevDoc/implementation/unitig_evidence/index.html @@ -957,14 +957,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1202,6 +1202,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/index.html b/DevDoc/index.html index 2dd207f6..e6629e0a 100644 --- a/DevDoc/index.html +++ b/DevDoc/index.html @@ -831,14 +831,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1076,6 +1076,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/index.refs/index.html b/DevDoc/index.refs/index.html index 6a96f80c..5f0a7f61 100644 --- a/DevDoc/index.refs/index.html +++ b/DevDoc/index.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/installation/index.html b/DevDoc/installation/index.html index 4ce8a4a6..42e8a762 100644 --- a/DevDoc/installation/index.html +++ b/DevDoc/installation/index.html @@ -917,14 +917,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1162,6 +1162,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/kmers.refs/index.html b/DevDoc/kmers.refs/index.html index 847d8c23..b12faff2 100644 --- a/DevDoc/kmers.refs/index.html +++ b/DevDoc/kmers.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/kmers/index.html b/DevDoc/kmers/index.html index 5336e366..a707331c 100644 --- a/DevDoc/kmers/index.html +++ b/DevDoc/kmers/index.html @@ -830,14 +830,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1075,6 +1075,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/sitemap.xml.gz b/DevDoc/sitemap.xml.gz index 425040cd..f3981d08 100644 Binary files a/DevDoc/sitemap.xml.gz and b/DevDoc/sitemap.xml.gz differ diff --git a/DevDoc/theory/encoding.refs/index.html b/DevDoc/theory/encoding.refs/index.html index 2d5effdd..fe650220 100644 --- a/DevDoc/theory/encoding.refs/index.html +++ b/DevDoc/theory/encoding.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/theory/encoding/index.html b/DevDoc/theory/encoding/index.html index dd295cb4..e8a2dbb1 100644 --- a/DevDoc/theory/encoding/index.html +++ b/DevDoc/theory/encoding/index.html @@ -802,14 +802,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1047,6 +1047,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/theory/entropy.refs/index.html b/DevDoc/theory/entropy.refs/index.html index e0f18901..49c1b70d 100644 --- a/DevDoc/theory/entropy.refs/index.html +++ b/DevDoc/theory/entropy.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/theory/entropy/index.html b/DevDoc/theory/entropy/index.html index b0ae350f..5e201572 100644 --- a/DevDoc/theory/entropy/index.html +++ b/DevDoc/theory/entropy/index.html @@ -868,14 +868,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1113,6 +1113,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/theory/evolutionary_distances/index.html b/DevDoc/theory/evolutionary_distances/index.html index 87fadedf..542fbc37 100644 --- a/DevDoc/theory/evolutionary_distances/index.html +++ b/DevDoc/theory/evolutionary_distances/index.html @@ -1244,14 +1244,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1489,6 +1489,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -3582,7 +3610,7 @@ diagonal entry directly, at no extra lookup cost.

    Mirror distance.rs: open the presence or count store per partition. But instead of a per-partition partial_*, run the sequential source sweep:

    for p in 0..n_partitions:                       # OUTER — sequential
    -    open source partition p's layers (QueryLayer-style, obikpartitionner)
    +    open source partition p's layers (QueryLayer-style, obikindex::partition)
         enumerate distinct canonical k-mers of p (one per MPHF slot) with their
             presence/count vectors                   # column-major, as query stage 2
         par_iter over these source k-mers:           # INNER — rayon, thread-local tally
    @@ -3605,7 +3633,7 @@ instead of a per-partition partial_*, run the sequential source swe
         merge thread-local tallies into global SnpTally
     

    The inner lookup is precisely QueryLayer::find_slot + -col_value(g, slot) (obikpartitionner/src/query_layer.rs) — reuse or factor +col_value(g, slot) (obikindex/src/partition/query_layer.rs) — reuse or factor out that path rather than reimplementing MPHF access. Enumerating "all distinct k-mers of a partition with their vectors" is the dump/query stage-2 column-major scan already implemented in dump_layer.rs / diff --git a/DevDoc/theory/indexing.refs/index.html b/DevDoc/theory/indexing.refs/index.html index 17bc0289..890844ab 100644 --- a/DevDoc/theory/indexing.refs/index.html +++ b/DevDoc/theory/indexing.refs/index.html @@ -727,14 +727,14 @@

  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + @@ -1242,8 +1270,8 @@

    Coverage: theory/indexing.md

    Code couvert

      -
    • obikpartitionner/src/partition.rs — routage par hash de minimiseur, choix des paramètres
    • -
    • obikpartitionner/src/lib.rs — structure KmerPartition, nombre de partitions
    • +
    • obikindex/src/partition/partition.rs — routage par hash de minimiseur, choix des paramètres
    • +
    • obikindex/src/partition/mod.rs — structure KmerPartition, nombre de partitions

    Notes

    Vérifier que la doc mentionne bien que le nombre de partitions est une puissance de 2 diff --git a/DevDoc/theory/indexing/index.html b/DevDoc/theory/indexing/index.html index 87e40d5c..afbd21e6 100644 --- a/DevDoc/theory/indexing/index.html +++ b/DevDoc/theory/indexing/index.html @@ -802,14 +802,14 @@

  • - + - obilayeredmap crate + obikindex layer tier @@ -1047,6 +1047,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/theory/minimizer.refs/index.html b/DevDoc/theory/minimizer.refs/index.html index 4756a4c2..efa795a8 100644 --- a/DevDoc/theory/minimizer.refs/index.html +++ b/DevDoc/theory/minimizer.refs/index.html @@ -727,14 +727,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -972,6 +972,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDoc/theory/minimizer/index.html b/DevDoc/theory/minimizer/index.html index 0392b8be..f7d4cdc0 100644 --- a/DevDoc/theory/minimizer/index.html +++ b/DevDoc/theory/minimizer/index.html @@ -846,14 +846,14 @@
  • - + - obilayeredmap crate + obikindex layer tier @@ -1091,6 +1091,34 @@ + + + + + + +
  • + + + + + + + + Partition and layer caching (discussion) + + + + + + + + +
  • + + + + diff --git a/DevDocMD/architecture/index_architecture.refs.md b/DevDocMD/architecture/index_architecture.refs.md index ff9e3c00..4ca5528e 100644 --- a/DevDocMD/architecture/index_architecture.refs.md +++ b/DevDocMD/architecture/index_architecture.refs.md @@ -3,10 +3,10 @@ ## Code couvert -- `obilayeredmap/src/layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix -- `obilayeredmap/src/mphf_layer.rs` — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum -- `obilayeredmap/src/map.rs` — LayeredMap -- `obilayeredmap/src/meta.rs` — LayerMeta, PartitionMeta +- `obikindex/src/layer/typed_layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix +- `obikindex/src/layer/mphf_layer.rs` — MphfLayer, EvidenceKind (Exact / Approx), LayerEvidence enum +- `obikindex/src/layer/map.rs` — LayeredMap +- `obikindex/src/layer/meta.rs` — LayerMeta, PartitionMeta - `obikindex/src/meta.rs` — IndexConfig (kmer_size, n_bits, with_counts, evidence, block_bits), IndexMeta - `obikindex/src/index.rs` — KmerIndex, build_layers - `obicompactvec/src/` — PersistentCompactIntMatrix, PersistentBitMatrix (DataStore implementations) diff --git a/DevDocMD/architecture/numa_worker_pools.md b/DevDocMD/architecture/numa_worker_pools.md index 452430af..673a4ba9 100644 --- a/DevDocMD/architecture/numa_worker_pools.md +++ b/DevDocMD/architecture/numa_worker_pools.md @@ -72,9 +72,9 @@ Each NUMA group runs its own independent adaptive pool. Workers are distributed |------|--------| | `obikindex/src/merge.rs` | Detect NUMA topology; build N `ThreadPool`s with pinned threads; assign each pre-spawned worker to a pool; wrap `merge_partition` in `pool.install()` | | `obikindex/src/merge.rs` | Replace `available_parallelism()` with per-NUMA core count for spawn criterion | -| `obikpartition/src/merge_layer.rs` | No change — `merge_partition` already works inside any Rayon context | +| `obikindex/src/partition/merge_layer.rs` | No change — `merge_partition` already works inside any Rayon context | | `obidebruinj/src/debruijn.rs` | No change — `par_iter` and `current_num_threads` are pool-context-aware | -| `obikpartition/src/partition.rs` | No change — same reason | +| `obikindex/src/partition/partition.rs` | No change — same reason | ## Platform guard diff --git a/DevDocMD/architecture/query.md b/DevDocMD/architecture/query.md index 25742ece..98a1ebe1 100644 --- a/DevDocMD/architecture/query.md +++ b/DevDocMD/architecture/query.md @@ -16,7 +16,7 @@ Given a set of query sequences, determine for each sequence how many of its k-me ## Algorithm -The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikpartition::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk. +The query follows the same superkmer-based partitioning strategy used at indexing time. Everything below happens inside `process_chunk` (`query.rs`); there is no separate per-stage function, but the internal data flow is staged: k-mer-level dereplication, a two-part MPHF/column-major matrix lookup (`obikindex::partition::query_partition_with`), and a sparse Findere pass, each producing sparse intermediate structures rather than one dense allocation for the whole chunk. ``` for each chunk of sequences (parallel workers via obipipeline, one call to process_chunk): @@ -118,11 +118,11 @@ The `-z` CLI option overrides the index metadata value. A higher z increases str ## Layer lookup: `MphfLayer::find` -`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obilayeredmap](../implementation/obilayeredmap.md) for the full `find` / `find_strict` API. +`MphfLayer::open(dir, mode: &IndexMode)` receives the mode from `PartitionMeta` — no per-layer file is read. The caller (`QueryLayer`) never chooses the dispatch path: it is fixed at open time by `LayerEvidence`. See [obikindex::layer](../implementation/layer_tier.md) for the full `find` / `find_strict` API. ### `QueryLayer` variant selection -`QueryLayer::open` (`obikpartition/src/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order: +`QueryLayer::open` (`obikindex/src/partition/query_layer.rs:28-45`) only ever returns two variants — `Presence` or `Count`, checked in this order: | Order | Condition | Variant | Data returned per k-mer | |---|---|---|---| @@ -244,7 +244,7 @@ Fix direction: restructure `query`'s pipe with an initial `Flat` stage analogous `chunk_bytes = available_memory_bytes() / (n_workers * 16)` (`query.rs:407-414`) assumes a fixed ~8–16× overhead per raw input byte. But `KmerResults::new` (`query.rs:165-179`) allocates `data: Vec` sized `total_kmers_in_chunk × n_genomes` — dense, **for every k-mer position in the chunk, hit or not** — plus `win_min` and (with `--detail`) `cov`, same scaling. Real per-chunk memory is `O(n_genomes)`, not constant; the formula doesn't know `n_genomes` at all. This is the direct cause of the OOM kill on indexes with many reference genomes. **4. MPHF lookup and matrix-row fetch are fused, not staged.** -`QueryLayer::find_into` (`obikpartition/src/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part). +`QueryLayer::find_into` (`obikindex/src/partition/query_layer.rs:48-67`) does the MPHF `find` *and* the `fill_row` matrix read in one call per k-mer, inside a single-threaded loop (`query_partition_with`). There is no separation between "is this k-mer indexed" (cheap, `O(1)`, independent of `n_genomes`) and "what are its per-genome values" (the expensive, `n_genomes`-scaling part). **5. Dereplication should happen at k-mer granularity, directly — not via an intermediate superkmer-level dedup.** `QueryBatch::from_records` currently dereplicates at the *superkmer* level (`HashMap>`, `query.rs:112`). This misses redundancy between k-mers shared by *different* superkmers (read overlaps, repeats, a SNP splitting an otherwise-identical run). Superkmer *construction* (`SuperKmerIter`) stays mandatory — it is the mechanism that computes minimizers/partition routing, not an optional dedup layer — but the dedup structure built on top of it should key directly on `CanonicalKmer`, in the same pass: `HashMap>`. This also means the MPHF `find` itself runs once per **distinct** k-mer instead of once per occurrence — a win independent of the matrix-fetch cost below. @@ -285,7 +285,7 @@ None of this is implemented yet — parked here as a coherent roadmap while the ## Implementation plan -Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikpartition`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)). +Concrete, phased translation of the roadmap above. Phases 0–2 are small, independent, low-risk, and each individually testable against current `query` output — land them first, in order, and measure on the reference 192-core/8-NUMA machine before deciding whether phases 3–5 (the staged/sparse engine, the larger structural payoff) are still worth their cost. Phases 3–5 are one coordinated change spanning `obikmer`, `obikindex::partition`, and `obicompactvec` — they should not be split across releases mid-way, because the intermediate state (e.g. k-mer-level dedup feeding the old dense `KmerResults`) has no correctness or performance benefit on its own. Phase 6 is unrelated to phases 0–5 and can happen any time, independently, if `rapidgzip-rs` is validated (see [chunkreader.md](../implementation/chunkreader.md#future-work--parallel-gzip-decompression-in-xopen)). Instrumentation is deliberately sequenced *before* the I/O fix (reordering the roadmap's own listed order), because every later phase's justification rests on a measurement ("to be measured, not assumed" appears throughout the roadmap above) — without it, phases 3–5 would be undertaken on faith. @@ -295,7 +295,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the **Debug logging.** Every phase that changes an algorithmic choice (not phase 0, which *is* the logging) adds `tracing::debug!`/`trace!` at points that let a cluster run's logs answer "did this help": counts, ratios, and timings that quantify the specific claim that phase makes — e.g. phase 3 must log how many MPHF `find` calls were saved by k-mer-level dedup (the whole justification for that phase), phase 4 must log per-column scan timings, phase 5 must log actual retained-memory / sparsity ratios achieved. Prefer one structured `debug!` per chunk (fields, not prose) over free-text — the cluster logs will be the only evidence available for judging these choices, so they need to be grep/awk-able, not just readable. -**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obilayeredmap`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/.rs`. Neither `obikmer` nor `obikpartition` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one. +**Unit tests.** This project's convention (`obiread`, `obikseq`, `obidebruinj`, `obicompactvec`, `obikindex::layer`, `obiskio`, `obifastwrite`) is `#[cfg(test)] #[path = "tests/.rs"] mod tests;` at the bottom of the source file, with the actual test code in a sibling `src/tests/.rs`. Neither `obikmer` nor `obikindex::partition` (the two crates phases 3 and 5 touch most) currently have a `src/tests/` directory at all — this needs creating, following the existing pattern exactly, not inventing a new one. **Workflow (`jj`).** Work happens in a fresh `jj` commit, easy to abandon. `jj new` between phases is reasonable where it helps isolate a phase for review, but only when the working copy compiles at that point (project convention) — phase 3's internal sub-steps (batch dedup change, then `query_layer.rs` split, then the new return shape) will likely not each compile independently since they're one coupled change, so treat "commit boundary" and "plan phase boundary" as related but not forced to match 1:1; use judgement per phase rather than mechanically splitting on every bullet. @@ -345,13 +345,13 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the - `obikmer/src/cmd/query.rs`: - Replace `QueryBatch::from_records`'s dedup map (`HashMap>`, current `query.rs:112`) with a per-partition `HashMap>`, built in the same `SuperKmerIter` pass: superkmer construction and partition routing (`part_idx` from the superkmer's minimizer hash) are unchanged, only the granularity of what gets deduplicated changes — each `CanonicalKmer` within a superkmer is inserted individually instead of the whole superkmer being the dedup key. - **Verified**: `CanonicalKmer` (`obikseq/src/kmer.rs:390`, `pub type CanonicalKmer = CanonicalKmerOf`) — the underlying `CanonicalKmerOf` derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash` (`kmer.rs:269`). Usable as a `HashMap`/`HashSet` key as-is, no change needed. -- `obikpartition/src/query_layer.rs`: +- `obikindex/src/partition/query_layer.rs`: - Split `QueryLayer::find_into` (`query_layer.rs:48-67`) into two methods: `find_slot(&self, kmer: CanonicalKmer) -> Option` (MPHF only, no matrix touch) and keep `fill_row` as-is for phase 4 to call later. - Replace `query_partition_with`'s inner loop (`query_layer.rs:103-113`) with a version that, for each unique `CanonicalKmer`, calls `find_slot` across the partition's layers (stopping at first hit, same as today), and instead of immediately filling a row, records `(layer_idx, slot)`. - New return shape for the partition-level query, replacing today's `on_hit(sk_idx, kmer_idx, row)` callback: `HashMap>>` (roadmap point 6) — built directly from the k-mer dedup map's `Vec<(seq_idx,pos)>` values, keyed by the resolved slot instead of the k-mer. - **This phase alone has no throughput benefit yet** (matrix fetch still happens, just deferred) beyond the k-mer-level dedup itself (fewer MPHF calls when queries have overlapping/repeated k-mers) — its purpose is to produce the input phase 4 needs. Land phase 3+4 together, not phase 3 alone, per the "don't split 3–5 across releases" note above. - Log, per chunk: total k-mer occurrences vs. unique `CanonicalKmer` count (the dedup ratio — the entire justification for this phase) and the resulting MPHF `find` call count. If the dedup ratio is close to `1.0` on real query data (little redundancy), that's the cluster run telling us this phase wasn't worth it — the logging needs to be able to say that, not just confirm the happy path. -- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikpartition/src/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced. +- **Unit tests**: create `obikmer/src/cmd/tests/query.rs` (new `src/tests/` dir for this crate, following the project's `#[cfg(test)] #[path = "tests/query.rs"] mod tests;` convention) and `obikindex/src/partition/tests/query_layer.rs` (likewise new for this crate). Cover: the k-mer-level dedup map construction on synthetic sequences with known repeated/overlapping k-mers (assert unique-kmer count and occurrence lists); the `find_slot`/bucket-by-layer-and-slot construction against a small hand-built `QueryLayer` fixture, asserting the `(layer_idx, slot, seq_idx, pos)` tuples match what the old per-occurrence loop would have produced. ### Phase 4 — Column-major matrix fetch (roadmap points 7–8) — implemented, NUMA parallelism deferred @@ -359,7 +359,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the **What shipped:** - `obicompactvec`: the per-column accessors this phase needed **already existed** — `PersistentCompactIntMatrix::col_view(c)` and `PersistentBitMatrix::col_view(c)` are public, and `IntSliceView::get(slot)`/`BitSliceView::get(slot)` are public — the original plan underestimated how much of this plumbing the pairwise-distance code (`dump`/`select`/`stats`) had already required. The one real gap: `PersistentBitMatrix::col_view()` panics on the `Implicit` variant (the documented mono-genome fast path, `bitmatrix.rs`). Added `PersistentBitMatrix::get(c, slot) -> u32` (`bitmatrix.rs`), a non-panicking column-major point lookup that returns `1` for `Implicit` regardless of `c` — the smallest surface needed, not a new `col_get` API from scratch. -- `obikpartition/src/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call. +- `obikindex/src/partition/query_layer.rs`: `query_partition_with` is now two explicit stages, matching roadmap points 6–8: **stage 1** (MPHF-only, per unique k-mer, bucket hits by `(layer_idx, slot)`, emits `QueryHit::Found`) then **stage 2** (per layer with ≥1 hit, column-major: for each genome column `g` in `0..layer.n_cols().min(n_genomes)`, scan that layer's bucketed slots and call `col_value(g, slot)`, emitting `QueryHit::Value(descs, g, value)` on nonzero). `QueryHit` is a single enum delivered through one `FnMut(QueryHit)` callback — an earlier two-closure design (`on_found` + `on_value`) didn't borrow-check, since the caller's single mutable accumulator (`KmerResults`) can't be captured by two separate `FnMut` closures passed to the same call. - `obikmer/src/cmd/query.rs`: `KmerResults::set` (row-major, whole-row-at-once) replaced by `mark_found` (stage 1: flag a position as indexed, independent of any genome's value) and `set_one` (stage 2: write one genome's value at one position). `QueryStats` extended with `n_columns_scanned`/`n_col_get_calls`, logged per chunk. - Total `get()`-equivalent calls are unchanged from the row-major version (`n_hits × n_cols` in the worst case, confirmed by `n_col_get_calls` in the debug log) — the win is locality (sequential access within one layer's column at a time, across `mmap`'d regions, instead of jumping across all columns per hit), exactly as predicted. @@ -367,7 +367,7 @@ Performance measurement on the reference 192-core/8-NUMA machine is done by the Reading `obikindex/src/numa.rs`'s actual `run()` body (not just its doc comments) shows every call spawns a timer thread **plus one OS thread per worker slot on every NUMA node** (`std::thread::scope` + one `s.spawn()` per node per `max_workers`) — on the 192-core/8-NUMA reference machine, that's on the order of 190+ fresh OS threads spawned **per call**. This is fine for its actual, established usage in this codebase (`merge.rs`, `index.rs`'s `build_layers`): one `PartitionRunner::new()` + one `run()` call per command invocation, amortised over a batch of ~256 long-running partitions. It is not fine for `query`'s call pattern: `query_partition_with` runs once per `(chunk, partition)`, potentially thousands of times per second — spawning ~190 OS threads that often to scan a handful of genome columns would very likely cost far more than the row-major approach it's meant to replace. This is exactly the "resolve empirically, don't assume" composition risk the roadmap flagged, just resolved by reading the mechanism's actual cost before wiring it in, rather than by measuring a regression on the cluster after the fact. The column-major loop in stage 2 is therefore a **plain sequential loop** for now — it captures the whole, provable locality win (roadmap point 8's actual claim) without adding any parallelism mechanism. Genome-column-level parallelism (point 8's "bonus" axis) and partition-level parallelism (point 7) are both deferred — not abandoned. Candidates for a follow-up, once there's a concrete profiling need: (a) `rayon`'s already-warm global pool (`into_par_iter()`) for the column axis specifically — cheap to invoke repeatedly since it doesn't spawn threads per call, though it's the same "naive rayon" pattern `numa_worker_pools.md` warns about for a *different* workload (random pointer-chasing over large hash maps); a column scan's access pattern (sequential reads within one `mmap`'d region) has a different contention profile and hasn't been shown to have the same problem — needs its own measurement, not an assumption either way; (b) restructuring so `PartitionRunner` is invoked once per whole `query` run (or per large batch of chunks) rather than per `(chunk, partition)`, amortising its spawn cost the way `merge`/`build_layers` do — a bigger structural change than this phase's scope. - Log (implemented): `QueryStats::n_columns_scanned`/`n_col_get_calls`, folded into the existing per-chunk `debug!("k-mer dedup + column-major fetch", ...)` line (`query.rs`) alongside phase 3's dedup counters. -- **Unit tests**: extended `obikpartition/src/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature. +- **Unit tests**: extended `obikindex/src/partition/tests/query_layer.rs` (phase 3's file) — `query_partition_with`'s empty/missing-index paths updated for the new `QueryStats` fields and single-callback signature. - **Validation performed**: full workspace build + `cargo test --workspace`, zero failures. Functional validation against real indexes: (1) a single-genome index — output byte-identical to pre-phase-4 (same `kmer_count`/`kmer_strict_matches` on every record); (2) the existing 20-genome `benchmark/global_index_presence` index — runs correctly, `n_hits=0` for an unrelated query (expected: no shared k-mers between a plant read and a bacterial reference set), no panics, confirming the `Implicit`/multi-column bounds logic doesn't crash on a real multi-genome, mixed-format index; (3) **the critical correctness case**: built two single-sequence-pair test genomes, merged into one 2-genome index, queried with reads from both — reads from `genomeA` matched **only** `genomeA` (`kmer_count` identical to the pre-dedup occurrence count, zero leakage into `genomeB`'s column) and vice versa. This is the test that would have caught a column-index mixup, an off-by-one in `n_cols`, or cross-genome bleed from the stage-1/stage-2 split — it passed cleanly. - **Not yet done**: the microbenchmark comparing column-major vs. the old row-major access pattern's wall time / page-fault counters on a large-`n_genomes` layer — needs a realistically large multi-genome index and, for the page-fault counters specifically, Linux (not available from this development environment). Left for cluster validation alongside phases 1–3's own pending measurements. diff --git a/DevDocMD/architecture/query.refs.md b/DevDocMD/architecture/query.refs.md index c09e499d..d281c78c 100644 --- a/DevDocMD/architecture/query.refs.md +++ b/DevDocMD/architecture/query.refs.md @@ -4,7 +4,7 @@ ## Code couvert - `obikmer/src/cmd/query.rs` — commande query, format de sortie -- `obikpartition/src/query_layer.rs` — routage de la requête à travers les partitions +- `obikindex/src/partition/query_layer.rs` — routage de la requête à travers les partitions - `obiread/src/lib.rs` — lecture des séquences d'entrée pour la requête ## Notes diff --git a/DevDocMD/architecture/rebuild_filter.md b/DevDocMD/architecture/rebuild_filter.md index 00311032..6c0cd940 100644 --- a/DevDocMD/architecture/rebuild_filter.md +++ b/DevDocMD/architecture/rebuild_filter.md @@ -94,9 +94,9 @@ Option B avoids storing kmer values and works uniformly regardless of filter sel ## Files to modify -- `src/obikpartition/src/rebuild_layer.rs` — `rebuild_partition` and `iter_src_layers` +- `src/obikindex/src/partition/rebuild_layer.rs` — `rebuild_partition` and `iter_src_layers` - Possibly `src/obicompactvec/` — add column iterator API if not already present -- `src/obilayeredmap/` — check if per-column sequential access is exposed on `SrcLayerData` +- `src/obikindex/src/layer/` — check if per-column sequential access is exposed on `SrcLayerData` ## Open questions diff --git a/DevDocMD/architecture/siblings.md b/DevDocMD/architecture/siblings.md index 967a0b8e..6021a698 100644 --- a/DevDocMD/architecture/siblings.md +++ b/DevDocMD/architecture/siblings.md @@ -17,7 +17,7 @@ one-way function, not an invertible bijection with a stored inverse. Any method that reconstructs a kmer from a bare slot number is wrong by construction, regardless of the mechanism used (MPHF re-hash, or evidence decode + direct unitig read). See `MphfLayer::kmer_at` -(`obilayeredmap/src/mphf_layer.rs`) — flagged for removal, currently called +(`obikindex/src/layer/mphf_layer.rs`) — flagged for removal, currently called from `obikphylo/siblings/build.rs` and `family_scan.rs` (since removed — see "Pending work" status below). @@ -70,7 +70,7 @@ partition is unknown) and must keep going through ## Pending work — done The plan above shipped: `obikphylo` (a new crate — phylo-domain extension -traits over `obikindex::KmerIndex`/`obilayeredmap::Layer`, replacing the +traits over `obikindex::KmerIndex`/`obikindex::layer::Layer`, replacing the old `obikindex::siblings` module) builds and reads the annex purely in iteration order (`SiblingLayerExt::iter_siblings`/`iter_minorants`, both with batch variants, mirroring `Layer`'s own `KmerIter`/`KmerBatchIter` @@ -652,13 +652,13 @@ BRWT-style column-correlation exploitation. `PersistentSparseBitMatrix` went from a validated but unused type to a real, selectable on-disk format: -- **Generic `Layer`**: `obilayeredmap::Layer`'s presence-only methods +- **Generic `Layer`**: `obikindex::layer::Layer`'s presence-only methods (`n_cols`, `sub_matrix`, `fill_sub_matrix`) are generic over any `D: LayerData> + BinaryMatrix`, not hardcoded to `PersistentBitMatrix` — `PersistentSparseBitMatrix` implements `LayerData` (`open`/`read`) the same way. `find_slot`/`index_batch` were already generic over any `D: LayerData`, so they needed no change. - Verified by `obilayeredmap`'s + Verified by `obikindex::layer`'s `presence_layer_generic_over_sparse_matches_dense` test: build a dense presence layer, convert it to sparse via `build_from_dense`, open both as `Layer`/`Layer` on @@ -668,7 +668,7 @@ real, selectable on-disk format: `AtomicUsize`s in test builds, not thread-local, so a test using a different `k` races every other test in the same crate binary; a k=11 version of this test passed alone but failed under the full - `obilayeredmap` suite for exactly that reason before being fixed.) + `obikindex::layer` suite for exactly that reason before being fixed.) - **`obikphylo::siblings::cache::Mat`** gained a third variant, `SparsePresence(Layer)`, alongside `Count` and `Presence` — every method (`find_slot`, `index_batch`, @@ -698,7 +698,7 @@ real, selectable on-disk format: dense-path test (`sibling_annex_one_sibling_each`) exactly — proves the sparse format round-trips through the real build pipeline (`PartitionCache` sparse-detection included), not just the - `obicompactvec`/`obilayeredmap` unit layers below it. + `obicompactvec`/`obikindex::layer` unit layers below it. Full workspace `cargo test` (all crates, unit + doc tests) green after this change. @@ -746,7 +746,7 @@ Replaces the "four independent scans" problem above and implements (activation: either given; defaults `1.0`/`0.5` for whichever is unset). Full workspace `cargo test` green after this change (167 unit tests in -`obicompactvec`+`obilayeredmap`+`obikphylo` alone, plus every other +`obicompactvec`+`obikindex::layer`+`obikphylo` alone, plus every other crate's suite, no regressions). **Still open, not part of this change** (per "Correction to the 'single @@ -770,7 +770,7 @@ reproducible across two runs with warm disk cache), the opposite of query"). **Root cause, read from source, not measured in isolation:** -`KmerPartition::query_partition_with` (`obikpartition/src/query_layer.rs:155-220`) +`KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs:155-220`) is architecturally column-major: stage 2 walks `for g in 0..n_cols { for slot in hit_slots { layer.col_value(g, slot) } }`, documented (correctly) as the right locality strategy for the packed/columnar formats, where @@ -914,7 +914,7 @@ replaces. `PersistentCompactIntMatrix::nonzero_iter` added the same way (counts not excluded, per the earlier ask) — no native low-effort case, since no sparse count format exists, but on the same primitive, ready for one. -- `KmerPartition::query_partition_with` (`obikpartition/src/query_layer.rs`): +- `KmerPartition::query_partition_with` (`obikindex/src/partition/query_layer.rs`): stage 2's column-major `for g { for slot { col_value } }` replaced by one `layer.nonzero_iter(&slot_list)` call per layer, format-agnostic. - Tests: `nonzero_iter_matches_dense`, `nonzero_iter_matches_row`, and — diff --git a/DevDocMD/implementation/evidence_elimination.refs.md b/DevDocMD/implementation/evidence_elimination.refs.md index 0fedcb34..b5a17e28 100644 --- a/DevDocMD/implementation/evidence_elimination.refs.md +++ b/DevDocMD/implementation/evidence_elimination.refs.md @@ -3,9 +3,9 @@ ## Code couvert -- `obilayeredmap/src/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches() -- `obilayeredmap/src/mphf_layer.rs` — build_approx_evidence(dir, b, z), find_approx() -- `obilayeredmap/src/meta.rs` — EvidenceKind::Approx { b, z }, LayerMeta +- `obikindex/src/layer/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, stockage b bits/slot, matches() +- `obikindex/src/layer/mphf_layer.rs` — build_approx_evidence(dir, b, z), find_approx() +- `obikindex/src/layer/meta.rs` — EvidenceKind::Approx { b, z }, LayerMeta - `obikindex/src/reindex.rs` — KmerIndex::reindex(), conversion exact↔approx en place - `obikmer/src/cmd/reindex.rs` — CLI reindex, options --approx, -z, --evidence-bits, --fp, --block-size - `obikmer/src/cmd/index.rs` — resolve_approx_params(), options --approx, -z, --evidence-bits, --fp diff --git a/DevDocMD/implementation/filtering.md b/DevDocMD/implementation/filtering.md index f847e0a9..518d1e17 100644 --- a/DevDocMD/implementation/filtering.md +++ b/DevDocMD/implementation/filtering.md @@ -303,7 +303,7 @@ This parameter has no effect on presence/absence indexes (where values are alrea ## Implementation -- **`obikpartition::filter::GroupQuorumFilter`** — implements `KmerFilter` +- **`obikindex::partition::filter::GroupQuorumFilter`** — implements `KmerFilter` using pre-computed ingroup and outgroup index vectors. The heavy logic (predicate parsing, three-value evaluation, genome classification) happens once before any iteration; each k-mer row evaluation is a simple index @@ -314,7 +314,7 @@ This parameter has no effect on presence/absence indexes (where values are alrea `UnitigArgs`. `FilterArgs::build_filters()` returns a ready-to-use filter list. -- **`obikpartition::KmerPartition::iter_partition_kmers`** — accepts +- **`obikindex::partition::KmerPartition::iter_partition_kmers`** — accepts `filters: &[Box]` and applies them per-kmer before invoking the callback. `filter`, `dump`, and `unitig` all go through this single entry point. diff --git a/DevDocMD/implementation/obilayeredmap.md b/DevDocMD/implementation/layer_tier.md similarity index 97% rename from DevDocMD/implementation/obilayeredmap.md rename to DevDocMD/implementation/layer_tier.md index 1ce27b1d..57499b99 100644 --- a/DevDocMD/implementation/obilayeredmap.md +++ b/DevDocMD/implementation/layer_tier.md @@ -1,8 +1,8 @@ -# obilayeredmap — layered kmer index crate +# obikindex::layer — the Layer tier ## Purpose -`obilayeredmap` implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a `ptr_hash` MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers. +`obikindex::layer` (the `layer/` submodule of the `obikindex` crate — a standalone `obilayeredmap` crate until 2026-08-21, folded back in alongside `obikpartition` as part of a broader `Index { Partition { Layer } }` submodule regrouping) implements a persistent, incrementally extensible kmer index. Each layer covers a disjoint kmer set and wraps a `ptr_hash` MPHF with associated per-slot data. Adding a new dataset never rebuilds existing layers. --- diff --git a/DevDocMD/implementation/obilayeredmap.refs.md b/DevDocMD/implementation/layer_tier.refs.md similarity index 54% rename from DevDocMD/implementation/obilayeredmap.refs.md rename to DevDocMD/implementation/layer_tier.refs.md index 5340ab3e..77197823 100644 --- a/DevDocMD/implementation/obilayeredmap.refs.md +++ b/DevDocMD/implementation/layer_tier.refs.md @@ -1,14 +1,14 @@ -# Coverage: implementation/obilayeredmap.md +# Coverage: implementation/layer_tier.md ## Code couvert -- `obilayeredmap/src/mphf_layer.rs` — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx() -- `obilayeredmap/src/layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column() -- `obilayeredmap/src/map.rs` — LayeredMap, push_layer(), query() -- `obilayeredmap/src/evidence.rs` — Evidence, EvidenceWriter, encodage chunk_id:rank -- `obilayeredmap/src/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, matches() -- `obilayeredmap/src/meta.rs` — LayerMeta, EvidenceKind (Exact / Approx { b, z }) +- `obikindex/src/layer/mphf_layer.rs` — MphfLayer, LayerEvidence enum (Exact/Approx), find(), find_exact(), find_approx() +- `obikindex/src/layer/typed_layer.rs` — Layer, trait LayerData, modes () / PersistentCompactIntMatrix / PersistentBitMatrix, build(), build_evidence(), append_genome_column() +- `obikindex/src/layer/map.rs` — LayeredMap, push_layer(), query() +- `obikindex/src/layer/evidence.rs` — Evidence, EvidenceWriter, encodage chunk_id:rank +- `obikindex/src/layer/fingerprint.rs` — FingerprintVec, FingerprintVecWriter, matches() +- `obikindex/src/layer/meta.rs` — LayerMeta, EvidenceKind (Exact / Approx { b, z }) ## Notes diff --git a/DevDocMD/implementation/merge.refs.md b/DevDocMD/implementation/merge.refs.md index ce5ce93b..40e1e8aa 100644 --- a/DevDocMD/implementation/merge.refs.md +++ b/DevDocMD/implementation/merge.refs.md @@ -4,9 +4,9 @@ ## Code couvert - `obikindex/src/merge.rs` — `KmerIndex::merge()`, validation de compatibilité d'évidence, `validate_evidence_compat()` -- `obikpartition/src/merge_layer.rs` — `merge_partition()`, construction de la nouvelle layer, paramètre `block_bits` -- `obikpartition/src/rebuild_layer.rs` — `rebuild_partition()`, paramètre `block_bits` -- `obilayeredmap/src/layer.rs` — `Layer::append_genome_column()` (PersistentCompactIntMatrix et PersistentBitMatrix) +- `obikindex/src/partition/merge_layer.rs` — `merge_partition()`, construction de la nouvelle layer, paramètre `block_bits` +- `obikindex/src/partition/rebuild_layer.rs` — `rebuild_partition()`, paramètre `block_bits` +- `obikindex/src/layer/typed_layer.rs` — `Layer::append_genome_column()` (PersistentCompactIntMatrix et PersistentBitMatrix) - `obicompactvec/src/intmatrix.rs` — `append_column` pour PersistentCompactIntMatrix - `obicompactvec/src/bitmatrix.rs` — `append_column` pour PersistentBitMatrix diff --git a/DevDocMD/implementation/mphf.md b/DevDocMD/implementation/mphf.md index db2453d7..ad32e977 100644 --- a/DevDocMD/implementation/mphf.md +++ b/DevDocMD/implementation/mphf.md @@ -6,7 +6,7 @@ Kmer indexing per partition proceeds in two phases. The separation is necessary ### Phase 1 — provisional MPHF + kmer spectrum -Implemented in `obikpartition::KmerPartition::count_kmer()` → `count_partition()`. +Implemented in `obikindex::partition::KmerPartition::count_kmer()` → `count_partition()`. 1. **External sort**: read the dereplicated superkmer file; extract the raw `u64` canonical kmer value for every kmer of every superkmer. Sort in RAM-bounded chunks (adaptive budget: 40% of available RAM ÷ n_threads, minimum 1 M kmers per chunk), then k-way merge with inline dedup. Result: `sorted_unique.bin` — a flat array of f0 distinct sorted `u64` values. Exact kmer count f0 is known at this point. 2. **Build provisional MPHF** (ptr_hash, same configuration as phase 2) over `sorted_unique.bin` using `new_from_par_iter`. Delete `sorted_unique.bin` immediately after. Persist to `mphf1.bin`. @@ -101,7 +101,7 @@ type Mphf = PtrHash< ### Layer structure -Each layer is a self-contained unit. See [obilayeredmap](obilayeredmap.md) for the full on-disk layout. The MPHF-relevant files are: +Each layer is a self-contained unit. See [obikindex::layer](layer_tier.md) for the full on-disk layout. The MPHF-relevant files are: ``` layer_i/ @@ -148,7 +148,7 @@ MphfLayer::build_approx_evidence(dir, b, z) There is no `build_evidence` dispatch wrapper. Callers choose the appropriate post-hoc build directly. -In `obikpartition`, `build_index_layer` receives `block_bits: u8` from `IndexConfig::block_bits` and forwards it directly to `Layer::build` and `Layer::build_approx_evidence`. +In `obikindex::partition`, `build_index_layer` receives `block_bits: u8` from `IndexConfig::block_bits` and forwards it directly to `Layer::build` and `Layer::build_approx_evidence`. ### Membership verification diff --git a/DevDocMD/implementation/mphf.refs.md b/DevDocMD/implementation/mphf.refs.md index b8e9c3ee..fc02ceff 100644 --- a/DevDocMD/implementation/mphf.refs.md +++ b/DevDocMD/implementation/mphf.refs.md @@ -3,8 +3,8 @@ ## Code couvert -- `obilayeredmap/src/mphf_layer.rs` — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, `build()`, `build_exact_evidence()`, `build_approx_evidence()`, `build_evidence()` -- `obikpartition/src/index_layer.rs` — `build_index_layer()` avec passage de `block_bits` +- `obikindex/src/layer/mphf_layer.rs` — type Mphf (PtrHash + CubicEps + CachelineEfVec + Xx64), construction en 2 passes, `build()`, `build_exact_evidence()`, `build_approx_evidence()`, `build_evidence()` +- `obikindex/src/partition/index_layer.rs` — `build_index_layer()` avec passage de `block_bits` ## Notes diff --git a/DevDocMD/implementation/obipipeline.refs.md b/DevDocMD/implementation/obipipeline.refs.md index 1e9425a5..1499ac62 100644 --- a/DevDocMD/implementation/obipipeline.refs.md +++ b/DevDocMD/implementation/obipipeline.refs.md @@ -9,5 +9,5 @@ ## Notes Document stable (librairie générique, peu de risque de dérive). -Vérifier si `obipipeline` est toujours utilisé dans la phase scatter de `obikpartition` +Vérifier si `obipipeline` est toujours utilisé dans la phase scatter de `obikindex::partition` ou s'il a été remplacé par Rayon dans certains chemins. diff --git a/DevDocMD/implementation/partition_layer_cache.md b/DevDocMD/implementation/partition_layer_cache.md index 5ee7416a..9f669e03 100644 --- a/DevDocMD/implementation/partition_layer_cache.md +++ b/DevDocMD/implementation/partition_layer_cache.md @@ -1,5 +1,21 @@ # Partition and layer caching (discussion) +**Superseded (2026-08-21):** `obikpartition` and `obilayeredmap` are no +longer separate workspace crates — both were folded back into `obikindex` +as submodules (`obikindex::partition`, `obikindex::layer`), alongside the +crate's original content as `obikindex::index`, purely to reduce the +crate count (no behavior change). Every mention of `obikpartition`/ +`obilayeredmap` as a *crate* below, and every dependency-direction +argument phrased in terms of "which crate depends on which" (e.g. "this +crate depends only on `obilayeredmap` and below, never on `obikindex`"), +describes that now-superseded split-crate architecture and is kept as-is +for historical context — read `obikpartition::X` as `obikindex:: +partition::X` and `obilayeredmap::X` as `obikindex::layer::X` throughout. +The underlying module boundary and its rationale (Layer tier / Partition +tier / Index tier, each depending only downward) are unchanged; only the +crate-vs-module packaging changed. See [obikindex::layer](layer_tier.md) +for the current module doc. + Status (2026-08-20, latest pass): (1) done — `obilayeredmap::Layer` exists, `Mat` is gone. (1b) done — `Layer::Empty`, the first non-ready state, added (panics on every read method). (2a) done — the diff --git a/DevDocMD/implementation/pipeline.md b/DevDocMD/implementation/pipeline.md index b95a075f..65493121 100644 --- a/DevDocMD/implementation/pipeline.md +++ b/DevDocMD/implementation/pipeline.md @@ -203,7 +203,7 @@ part_XXXXX/ **Cleanup:** unless `--keep-intermediate` is set, `remove_build_artifacts` deletes `dereplicated.skmer.zst`, `mphf1.bin`, and `counts1.bin` after all partitions are indexed. -See [obilayeredmap](obilayeredmap.md) and [MPHF selection](mphf.md) for data structure details. +See [obikindex::layer](layer_tier.md) and [MPHF selection](mphf.md) for data structure details. **Query path (exact evidence):** diff --git a/DevDocMD/implementation/pipeline.refs.md b/DevDocMD/implementation/pipeline.refs.md index deaf8559..c7fe7cec 100644 --- a/DevDocMD/implementation/pipeline.refs.md +++ b/DevDocMD/implementation/pipeline.refs.md @@ -3,12 +3,12 @@ ## Code couvert -- `obikpartition/src/partition.rs` — estimation des paramètres (phase 0) +- `obikindex/src/partition/partition.rs` — estimation des paramètres (phase 0) - `obiskbuilder/src/iter.rs` — scatter : filtre entropie, extraction superkmers, routage partition (phase 1) -- `obikpartition/src/filter.rs` — déduplication bucket-sort (phase 2) -- `obikpartition/src/kmer_sort.rs` — tri externe + agrégation de comptages (phase 3) +- `obikindex/src/partition/filter.rs` — déduplication bucket-sort (phase 2) +- `obikindex/src/partition/kmer_sort.rs` — tri externe + agrégation de comptages (phase 3) - `obidebruinj/src/debruijn.rs` — graphe De Bruijn, extraction des unitigs (phase 5) -- `obikpartition/src/index_layer.rs` — construction MPHF + évidence (phase 6), paramètre `block_bits` +- `obikindex/src/partition/index_layer.rs` — construction MPHF + évidence (phase 6), paramètre `block_bits` - `obikindex/src/index.rs` — `build_layers()`, `dereplicate_and_count()` ## Notes diff --git a/DevDocMD/implementation/storage.refs.md b/DevDocMD/implementation/storage.refs.md index 69874018..dabd4c06 100644 --- a/DevDocMD/implementation/storage.refs.md +++ b/DevDocMD/implementation/storage.refs.md @@ -5,7 +5,7 @@ - `obikindex/src/meta.rs` — IndexMeta, IndexConfig (version, config, genomes) - `obikindex/src/index.rs` — layout sur disque : partitions/, index.meta -- `obilayeredmap/src/meta.rs` — LayerMeta (evidence kind), PartitionMeta (n_layers) +- `obikindex/src/layer/meta.rs` — LayerMeta (evidence kind), PartitionMeta (n_layers) - `obiskio/src/unitig_index.rs` — fichiers unitigs.bin + unitigs.bin.idx ## Notes diff --git a/DevDocMD/implementation/unitig_evidence.refs.md b/DevDocMD/implementation/unitig_evidence.refs.md index d25b2009..185b6cee 100644 --- a/DevDocMD/implementation/unitig_evidence.refs.md +++ b/DevDocMD/implementation/unitig_evidence.refs.md @@ -4,7 +4,7 @@ ## Code couvert - `obiskio/src/unitig_index.rs` — format unitigs.bin + unitigs.bin.idx, UnitigFileWriter, UnitigFileReader, build_unitig_idx(), DEFAULT_BLOCK_BITS=0, chemin chaud block_bits=0 dans chunk_start() -- `obilayeredmap/src/evidence.rs` — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter +- `obikindex/src/layer/evidence.rs` — encodage Evidence (chunk_id 25 bits | rank 7 bits), EvidenceWriter - `obidebruinj/src/debruijn.rs` — extraction unitigs, chunking à MAX_KMERS_PER_CHUNK ## Notes diff --git a/DevDocMD/theory/evolutionary_distances.md b/DevDocMD/theory/evolutionary_distances.md index 092b91db..722a56cf 100644 --- a/DevDocMD/theory/evolutionary_distances.md +++ b/DevDocMD/theory/evolutionary_distances.md @@ -1489,7 +1489,7 @@ instead of a per-partition `partial_*`, run the sequential source sweep: ```text for p in 0..n_partitions: # OUTER — sequential - open source partition p's layers (QueryLayer-style, obikpartition) + open source partition p's layers (QueryLayer-style, obikindex::partition) enumerate distinct canonical k-mers of p (one per MPHF slot) with their presence/count vectors # column-major, as query stage 2 par_iter over these source k-mers: # INNER — rayon, thread-local tally @@ -1513,7 +1513,7 @@ for p in 0..n_partitions: # OUTER — sequential ``` The inner lookup is precisely `QueryLayer::find_slot` + -`col_value(g, slot)` (`obikpartition/src/query_layer.rs`) — reuse or factor +`col_value(g, slot)` (`obikindex/src/partition/query_layer.rs`) — reuse or factor out that path rather than reimplementing MPHF access. Enumerating "all distinct k-mers of a partition with their vectors" is the `dump`/`query` stage-2 column-major scan already implemented in `dump_layer.rs` / diff --git a/DevDocMD/theory/indexing.refs.md b/DevDocMD/theory/indexing.refs.md index b7bea524..4eef9731 100644 --- a/DevDocMD/theory/indexing.refs.md +++ b/DevDocMD/theory/indexing.refs.md @@ -3,8 +3,8 @@ ## Code couvert -- `obikpartition/src/partition.rs` — routage par hash de minimiseur, choix des paramètres -- `obikpartition/src/lib.rs` — structure KmerPartition, nombre de partitions +- `obikindex/src/partition/partition.rs` — routage par hash de minimiseur, choix des paramètres +- `obikindex/src/partition/mod.rs` — structure KmerPartition, nombre de partitions ## Notes diff --git a/mkdocs.yml b/mkdocs.yml index 4fec9411..f1aa9f13 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,7 +47,7 @@ nav: - MPHF selection: implementation/mphf.md - Unitig evidence encoding: implementation/unitig_evidence.md - Evidence elimination (discussion): implementation/evidence_elimination.md - - obilayeredmap crate: implementation/obilayeredmap.md + - obikindex layer tier: implementation/layer_tier.md - PersistentCompactIntVec: implementation/persistent_compact_int_vec.md - PersistentBitVec: implementation/persistent_bit_vec.md - Merge command: implementation/merge.md diff --git a/src/Cargo.lock b/src/Cargo.lock index 3b836687..be87f8c3 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1515,24 +1515,6 @@ dependencies = [ "xxhash-rust", ] -[[package]] -name = "obikderep" -version = "0.1.0" -dependencies = [ - "niffler", - "obikindex", - "obikrope", - "obikseq", - "obilayeredmap", - "obiskbuilder", - "obiskio", - "obisys", - "rayon", - "sysinfo", - "tempfile", - "tracing", -] - [[package]] name = "obikentropy" version = "0.1.0" @@ -1545,6 +1527,7 @@ name = "obikindex" version = "0.1.0" dependencies = [ "anyhow", + "bitvec", "cacheline-ef", "crossbeam-channel", "epserde", @@ -1556,11 +1539,11 @@ dependencies = [ "obicompactvec", "obidebruinj", "obikentropy", - "obikpartition", + "obikrope", "obikseq", - "obilayeredmap", "obipipeline", "obiread", + "obiskbuilder", "obiskio", "obisys", "obitaxonomy", @@ -1568,6 +1551,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "sysinfo", "tempfile", "tracing", "tracing-subscriber", @@ -1583,13 +1567,10 @@ dependencies = [ "kodama", "obidebruinj", "obifastwrite", - "obikderep", "obikindex", - "obikpartitionner", "obikphylo", "obikrope", "obikseq", - "obilayeredmap", "obipipeline", "obiread", "obiskbuilder", @@ -1606,45 +1587,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "obikpartition" -version = "0.1.0" -dependencies = [ - "obicompactvec", - "obikseq", - "obilayeredmap", - "obiskio", - "tempfile", -] - -[[package]] -name = "obikpartitionner" -version = "0.1.0" -dependencies = [ - "cacheline-ef", - "epserde", - "memmap2", - "niffler", - "obicompactvec", - "obikderep", - "obikindex", - "obikrope", - "obikseq", - "obilayeredmap", - "obipipeline", - "obiread", - "obiskbuilder", - "obiskio", - "obisys", - "ptr_hash", - "rayon", - "serde", - "serde_json", - "sysinfo", - "tempfile", - "tracing", -] - [[package]] name = "obikphylo" version = "0.1.0" @@ -1652,11 +1594,8 @@ dependencies = [ "memmap2", "ndarray", "obicompactvec", - "obikderep", "obikindex", - "obikpartitionner", "obikseq", - "obilayeredmap", "obipipeline", "obiread", "obiskbuilder", @@ -1688,25 +1627,6 @@ dependencies = [ "xxhash-rust", ] -[[package]] -name = "obilayeredmap" -version = "0.1.0" -dependencies = [ - "bitvec", - "cacheline-ef", - "epserde", - "memmap2", - "ndarray", - "obicompactvec", - "obikseq", - "obiskio", - "ptr_hash", - "rayon", - "serde", - "serde_json", - "tempfile", -] - [[package]] name = "obipipeline" version = "0.1.0" diff --git a/src/Cargo.toml b/src/Cargo.toml index 6dd93601..1c306cb3 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -1,5 +1,5 @@ [workspace] resolver = "3" -members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obikpartitionner","obiskio","obidebruinj","obilayeredmap", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo", "obikpartition", "obikderep"] +members = ["obikseq", "obiread", "obiskbuilder", "obifastwrite", "obikmer","obikrope","obipipeline", "obiskio","obidebruinj", "obicompactvec", "obisys", "obikindex", "obitaxonomy", "obikentropy", "obikphylo"] [profile.release] debug = 1 diff --git a/src/obicompactvec/src/bitmatrix/packed.rs b/src/obicompactvec/src/bitmatrix/packed.rs index fcae6466..5451597d 100644 --- a/src/obicompactvec/src/bitmatrix/packed.rs +++ b/src/obicompactvec/src/bitmatrix/packed.rs @@ -129,7 +129,7 @@ pub fn pack_bit_matrix(dir: &Path) -> io::Result<()> { // A `matrix.pbmx` can already exist here even though columnar data is // still pending — e.g. copied verbatim from a merge's base source // before this layer was widened with more genome columns (see - // `obikpartitionner::merge_partition`). Only skip (re-)packing if the + // `obikindex::index::merge_layer::merge_partition`). Only skip (re-)packing if the // existing file already reflects the current column count; otherwise // the columnar files are newer and must be (re-)packed, overwriting the // stale one — never silently discarded as "leftover cleanup". diff --git a/src/obicompactvec/src/intmatrix.rs b/src/obicompactvec/src/intmatrix.rs index 2131e461..fe0f822c 100644 --- a/src/obicompactvec/src/intmatrix.rs +++ b/src/obicompactvec/src/intmatrix.rs @@ -232,7 +232,7 @@ pub fn pack_compact_int_matrix(dir: &Path) -> io::Result<()> { // A `matrix.pcmx` can already exist here even though columnar data is // still pending — e.g. copied verbatim from a merge's base source // before this layer was widened with more genome columns (see - // `obikpartitionner::merge_partition`). Only skip (re-)packing if the + // `obikindex::index::merge_layer::merge_partition`). Only skip (re-)packing if the // existing file already reflects the current column count; otherwise // the columnar files are newer and must be (re-)packed, overwriting the // stale one — never silently discarded as "leftover cleanup". diff --git a/src/obicompactvec/src/layer_meta.rs b/src/obicompactvec/src/layer_meta.rs index 28fff0c5..9c967a2f 100644 --- a/src/obicompactvec/src/layer_meta.rs +++ b/src/obicompactvec/src/layer_meta.rs @@ -2,7 +2,7 @@ use std::{fs, io, path::Path}; /// Lightweight metadata stored at the layer level (`layer_meta.json`). /// -/// Written by `obilayeredmap::MphfLayer::build` alongside `mphf.bin`. +/// Written by `obikindex::layer::MphfLayer::build` alongside `mphf.bin`. /// Read by `PersistentBitMatrix::open` to determine `n_rows` for the /// implicit (mono-genome presence/absence) case. pub struct LayerMeta { diff --git a/src/obicompactvec/src/storage_kind.rs b/src/obicompactvec/src/storage_kind.rs index 10374a85..d119582c 100644 --- a/src/obicompactvec/src/storage_kind.rs +++ b/src/obicompactvec/src/storage_kind.rs @@ -1,5 +1,5 @@ //! How a persistent matrix's data is physically laid out on disk — -//! orthogonal to *what* it stores (count vs. presence, `obilayeredmap`'s +//! orthogonal to *what* it stores (count vs. presence, `obikindex::layer`'s //! `LayerContent` concern, one crate up). `PersistentCompactIntMatrix` only //! ever reports `Columnar`/`Packed`; `PersistentBitMatrix` is the only type //! that can also report `Sparse`/`Implicit`. diff --git a/src/obikderep/Cargo.toml b/src/obikderep/Cargo.toml deleted file mode 100644 index 40a5e7a9..00000000 --- a/src/obikderep/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "obikderep" -version = "0.1.0" -edition = "2024" - -[dependencies] -niffler = "3.0.0" -obikseq = { path = "../obikseq" } -obikindex = { path = "../obikindex" } -obilayeredmap = { path = "../obilayeredmap" } -obiskio = { path = "../obiskio" } -obisys = { path = "../obisys" } -rayon = "1" -sysinfo = "0.39" -tracing = "0.1.44" - -[dev-dependencies] -tempfile = "3" -obikseq = { path = "../obikseq", features = ["test-utils"] } -obikrope = { path = "../obikrope" } -obiskbuilder = { path = "../obiskbuilder" } diff --git a/src/obikindex/Cargo.toml b/src/obikindex/Cargo.toml index 587c8746..77cb20cb 100644 --- a/src/obikindex/Cargo.toml +++ b/src/obikindex/Cargo.toml @@ -5,15 +5,15 @@ edition = "2024" [dependencies] obikseq = { path = "../obikseq" } -obikpartition = { path = "../obikpartition" } obitaxonomy = { path = "../obitaxonomy" } obiskio = { path = "../obiskio" } obisys = { path = "../obisys" } obicompactvec = { path = "../obicompactvec" } -obilayeredmap = { path = "../obilayeredmap" } obidebruinj = { path = "../obidebruinj" } obipipeline = { path = "../obipipeline" } obikentropy = { path = "../obikentropy" } +obiread = { path = "../obiread" } +obiskbuilder = { path = "../obiskbuilder" } cacheline-ef = "1.1" epserde = "0.8" ptr_hash = "1.1" @@ -26,10 +26,13 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" indicatif = "0.18" tracing = "0.1.44" +bitvec = "1" +sysinfo = "0.39" hwlocality = { version = "1.0.0-alpha.11", features = ["vendored"], optional = true } [dev-dependencies] -obiread = { path = "../obiread" } +obikseq = { path = "../obikseq", features = ["test-utils"] } +obikrope = { path = "../obikrope" } tempfile = "3" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } anyhow = "1" diff --git a/src/obikderep/src/dereplicate.rs b/src/obikindex/src/algorithms/dereplicator/dereplicate.rs similarity index 92% rename from src/obikderep/src/dereplicate.rs rename to src/obikindex/src/algorithms/dereplicator/dereplicate.rs index de02ec29..d3a54f84 100644 --- a/src/obikderep/src/dereplicate.rs +++ b/src/obikindex/src/algorithms/dereplicator/dereplicate.rs @@ -1,6 +1,7 @@ -//! Per-partition dereplication mechanics — private to this crate. -//! [`crate::Dereplicator`] is the public entry point; this module is the -//! two-phase split+merge algorithm it runs once per partition. +//! Per-partition dereplication mechanics — private to this module. +//! [`crate::algorithms::dereplicator::Dereplicator`] is the public entry +//! point; this module is the two-phase split+merge algorithm it runs once +//! per partition. use std::collections::HashMap; use std::fs; @@ -13,13 +14,13 @@ use niffler::send::compression::Format; use niffler::Level; use obikseq::superkmer::SuperKmer; use obikseq::Sequence; -use obilayeredmap::{dereplicated_superkmers_path, raw_superkmers_path}; +use crate::layer::{dereplicated_superkmers_path, raw_superkmers_path}; use obiskio::{SKFileMeta, SKFileReader, SKFileWriter, SKResult}; /// Scratch-file extension for this algorithm's own intermediate split /// buckets — never read by anything outside [`dereplicate_partition`], -/// unlike `raw`/`dereplicated` (see `obilayeredmap::{raw_superkmers_path, -/// dereplicated_superkmers_path}`, the actual cross-crate contract). +/// unlike `raw`/`dereplicated` (see `crate::layer::{raw_superkmers_path, +/// dereplicated_superkmers_path}`, the actual cross-module contract). const TEMP_EXT: &str = "skmer.zst"; /// Estimate the number of in-memory buckets needed to deduplicate the diff --git a/src/obikderep/src/lib.rs b/src/obikindex/src/algorithms/dereplicator/mod.rs similarity index 81% rename from src/obikderep/src/lib.rs rename to src/obikindex/src/algorithms/dereplicator/mod.rs index 080cfcea..d302673a 100644 --- a/src/obikderep/src/lib.rs +++ b/src/obikindex/src/algorithms/dereplicator/mod.rs @@ -1,26 +1,22 @@ -//! Superkmer dereplication — the second stage of the indexing pipeline, -//! after `obikpartitionner::PartitionRouter::run` (scatter) has written -//! each partition's raw superkmer file, before -//! `obikpartitionner::PartitionRouter::count_kmer` (counting) reads the -//! result. One algorithm, one crate — see -//! `DevDocMD/implementation/partition_layer_cache.md`'s "on avance pas à -//! pas" note: `obikpartitionner` used to also own dereplication and -//! counting; this crate is step one of splitting that bundle apart, -//! deliberately one algorithm at a time rather than all at once, so a -//! shared `Algorithm` pattern can be factored out later from real -//! examples instead of guessed at up front. +//! Superkmer dereplication — the sibling algorithm to +//! [`crate::algorithms::partitionner`]: runs after +//! `partitionner::PartitionRouter::run` (scatter) has written each +//! partition's raw superkmer file, before +//! `partitionner::PartitionRouter::count_kmer` (counting) reads the +//! result — see `DevDocMD/implementation/partition_layer_cache.md`. mod dereplicate; use std::sync::atomic::{AtomicU64, Ordering}; use niffler::Level; -use obikindex::KmerIndex; use obiskio::SKResult; use obisys::Progress; use rayon::prelude::*; use sysinfo::System; +use crate::index::KmerIndex; + use dereplicate::{dereplicate_partition, optimal_buckets}; /// Deduplicates every partition's raw superkmer file in place, replacing @@ -68,8 +64,8 @@ impl<'a> Dereplicator<'a> { /// `obisys::TracedBar`'s own methods take `&self`, not `&mut self`). /// `total: Some(n_partitions)` — known up front here, unlike /// `PartitionRouter::run`'s bases-processed count, so the caller can - /// render an actual progress bar rather than a spinner. This crate - /// never renders anything itself — see the module docs. + /// render an actual progress bar rather than a spinner. This algorithm + /// never renders anything itself. pub fn run(&self, on_progress: Option) -> SKResult<()> { let level = self.level; let sys = System::new_all(); @@ -88,7 +84,7 @@ impl<'a> Dereplicator<'a> { .map(|i| { let dir = self.index.layer_dir(i, 0); let result = if dir.exists() { - let raw_path = obilayeredmap::raw_superkmers_path(&dir); + let raw_path = crate::layer::raw_superkmers_path(&dir); let n_buckets = optimal_buckets(&raw_path, available_per_thread); dereplicate_partition(&dir, level, n_buckets) } else { diff --git a/src/obikindex/src/algorithms/mod.rs b/src/obikindex/src/algorithms/mod.rs new file mode 100644 index 00000000..ddc0e9c8 --- /dev/null +++ b/src/obikindex/src/algorithms/mod.rs @@ -0,0 +1,9 @@ +//! Indexing-pipeline algorithms: code that operates on a +//! [`crate::index::KmerIndex`] to build or transform its content, as +//! opposed to the `index`/`partition`/`layer` modules, which are the data +//! model itself. Each algorithm is its own submodule: [`partitionner`] +//! (routing raw super-kmers into partitions, then counting), [`dereplicator`] +//! (deduplicating a partition's raw super-kmers before counting). + +pub mod dereplicator; +pub mod partitionner; diff --git a/src/obikpartitionner/src/partition/count.rs b/src/obikindex/src/algorithms/partitionner/count.rs similarity index 98% rename from src/obikpartitionner/src/partition/count.rs rename to src/obikindex/src/algorithms/partitionner/count.rs index 4cd33f05..6d5a9990 100644 --- a/src/obikpartitionner/src/partition/count.rs +++ b/src/obikindex/src/algorithms/partitionner/count.rs @@ -11,7 +11,7 @@ use obiskio::{SKFileReader, SKResult}; use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; use tracing::debug; -use crate::kmer_sort::sort_unique_kmers; +use super::kmer_sort::sort_unique_kmers; pub(super) type Mphf = PtrHash>, Xx64, Vec>; diff --git a/src/obikpartitionner/src/kmer_sort.rs b/src/obikindex/src/algorithms/partitionner/kmer_sort.rs similarity index 100% rename from src/obikpartitionner/src/kmer_sort.rs rename to src/obikindex/src/algorithms/partitionner/kmer_sort.rs diff --git a/src/obikindex/src/algorithms/partitionner/mod.rs b/src/obikindex/src/algorithms/partitionner/mod.rs new file mode 100644 index 00000000..a6cd1cf5 --- /dev/null +++ b/src/obikindex/src/algorithms/partitionner/mod.rs @@ -0,0 +1,20 @@ +//! K-mer partitioning: routing super-kmers into per-partition, layer-0 +//! files, and counting unique canonical k-mers. Dereplication of the raw +//! super-kmers in between is [`crate::algorithms::dereplicator`], a +//! sibling algorithm, not a step of this one — see +//! `DevDocMD/implementation/partition_layer_cache.md`. +//! +//! Submodules: [`router`] (`PartitionRouter`, `KmerSpectrum`, the +//! routing/counting lifecycle API — partition/layer path naming itself +//! lives on `crate::index::KmerIndex`, not here), [`count`] (unique-kmer +//! enumeration, MPHF, abundance counting), `kmer_sort` (external sort +//! support for `count`). + +mod count; +mod kmer_sort; +mod router; + +#[cfg(test)] +mod tests; + +pub use router::{KmerSpectrum, PartitionRouter}; diff --git a/src/obikpartitionner/src/partition/router.rs b/src/obikindex/src/algorithms/partitionner/router.rs similarity index 94% rename from src/obikpartitionner/src/partition/router.rs rename to src/obikindex/src/algorithms/partitionner/router.rs index e8db0bff..35564d63 100644 --- a/src/obikpartitionner/src/partition/router.rs +++ b/src/obikindex/src/algorithms/partitionner/router.rs @@ -6,9 +6,9 @@ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use std::time::Instant; -use obikindex::KmerIndex; +use crate::index::KmerIndex; use obikseq::RoutableSuperKmer; -use obilayeredmap::Layer; +use crate::layer::Layer; use obiskio::SKResult; use obisys::{progress_bar, Progress}; use rayon::prelude::*; @@ -22,7 +22,7 @@ use obiskio::SKFileWriter; use obipipeline::{throttle, ThrottleGuard, Throttled}; use obiread::NucPage; -use crate::kmer_sort::chunk_size_from_ram; +use super::kmer_sort::chunk_size_from_ram; use super::count::count_partition; @@ -73,14 +73,14 @@ impl Drop for GuardedIter { // ── PartitionRouter ────────────────────────────────────────────────────────── /// Routes raw super-kmers into per-partition, layer-0 files, then -/// dereplicates and counts them — this crate's entire job now that -/// layer/query/merge/select/rebuild/dump/distance concerns have moved to -/// `obikindex` (they operate on built layers, which don't exist yet at this -/// stage — see `DevDocMD/implementation/partition_layer_cache.md`). +/// dereplicates (via the sibling [`crate::algorithms::dereplicator`] +/// algorithm) and counts them — the entry point of the indexing pipeline, +/// operating on layer-0 content that doesn't exist yet at this stage (see +/// `DevDocMD/implementation/partition_layer_cache.md`). /// /// Holds `&mut KmerIndex` — this is an algorithm operating on an index, not /// a data structure of its own; it owns no path-naming knowledge (every -/// path comes from `index.index_dir`/`obilayeredmap::layer_dir`/ +/// path comes from `index.index_dir`/`crate::layer::layer_dir`/ /// `Layer::create`), only the transient routing/dereplication/counting /// state a run needs. /// @@ -291,7 +291,7 @@ impl<'a> PartitionRouter<'a> { .into_par_iter() .map(|i| { let dir = self.layer0_dir(i); - let dedup_path = obilayeredmap::dereplicated_superkmers_path(&dir); + let dedup_path = crate::layer::dereplicated_superkmers_path(&dir); if !dedup_path.exists() { pb.inc(1); return Ok(()); @@ -343,9 +343,9 @@ impl<'a> PartitionRouter<'a> { /// Directory of partition `i`'s layer 0 — every raw/dereplicated /// superkmer file and provisional `mphf1.bin`/`counts1.bin` this router /// produces lives here, alongside where `build_index_layer` - /// (`obikindex`) will later turn it into the real layer 0. + /// (`crate::index`) will later turn it into the real layer 0. fn layer0_dir(&self, i: usize) -> PathBuf { - obilayeredmap::layer_dir(&self.index.index_dir(i), 0) + crate::layer::layer_dir(&self.index.index_dir(i), 0) } fn check_not_closed(&self) -> SKResult<()> { @@ -360,7 +360,7 @@ impl<'a> PartitionRouter<'a> { if self.writers[partition].is_none() { let dir = self.layer0_dir(partition); Layer::create(&dir).map_err(|e| io::Error::other(e.to_string()))?; - let file_path = obilayeredmap::raw_superkmers_path(&dir); + let file_path = crate::layer::raw_superkmers_path(&dir); let writer = SKFileWriter::create_with(file_path, Format::Zstd, self.level)?; self.writers[partition] = Some(writer); } diff --git a/src/obikpartitionner/src/partition/tests.rs b/src/obikindex/src/algorithms/partitionner/tests.rs similarity index 95% rename from src/obikpartitionner/src/partition/tests.rs rename to src/obikindex/src/algorithms/partitionner/tests.rs index f4dcd646..e8e41984 100644 --- a/src/obikpartitionner/src/partition/tests.rs +++ b/src/obikindex/src/algorithms/partitionner/tests.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; use std::fs; -use obikderep::Dereplicator; -use obikindex::{IndexConfig, KmerIndex}; +use crate::algorithms::dereplicator::Dereplicator; +use crate::{IndexConfig, KmerIndex}; +use crate::layer::IndexMode; use obikrope::Rope; use obikseq::SuperKmer; -use obilayeredmap::IndexMode; use obiskbuilder::build_superkmers; use super::count::count_partition; @@ -69,7 +69,7 @@ fn pipeline_counts(seqs: &[&[u8]]) -> (u64, u64) { Dereplicator::new(&index).run(None::).unwrap(); let part_dir = index.layer_dir(0, 0); - let dedup_path = obilayeredmap::dereplicated_superkmers_path(&part_dir); + let dedup_path = crate::layer::dereplicated_superkmers_path(&part_dir); if !dedup_path.exists() { return (0, 0); } diff --git a/src/obikindex/src/common.rs b/src/obikindex/src/index/common.rs similarity index 96% rename from src/obikindex/src/common.rs rename to src/obikindex/src/index/common.rs index 66d72f4b..1a8a3914 100644 --- a/src/obikindex/src/common.rs +++ b/src/obikindex/src/index/common.rs @@ -1,8 +1,8 @@ use std::path::Path; use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder}; -use obilayeredmap::meta::PartitionMeta; -use obilayeredmap::{layer_dir, IndexMode, OLMError}; +use crate::layer::meta::PartitionMeta; +use crate::layer::{layer_dir, IndexMode, OLMError}; use obiskio::{SKError, SKResult}; // ── olm_to_sk ──────────────────────────────────────────────────────────────── diff --git a/src/obikindex/src/distance.rs b/src/obikindex/src/index/distance.rs similarity index 97% rename from src/obikindex/src/distance.rs rename to src/obikindex/src/index/distance.rs index 2b651c1b..6425ab4f 100644 --- a/src/obikindex/src/distance.rs +++ b/src/obikindex/src/index/distance.rs @@ -1,10 +1,10 @@ use ndarray::Array2; use obicompactvec::traits::{BitPartials, CountPartials}; -use obilayeredmap::LayeredStore; +use crate::layer::LayeredStore; use rayon::prelude::*; -use crate::error::{OKIError, OKIResult}; -use crate::index::KmerIndex; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::kmer_index::KmerIndex; // ── Public API ──────────────────────────────────────────────────────────────── diff --git a/src/obikindex/src/dump.rs b/src/obikindex/src/index/dump.rs similarity index 97% rename from src/obikindex/src/dump.rs rename to src/obikindex/src/index/dump.rs index 5c9499dc..b9245f60 100644 --- a/src/obikindex/src/dump.rs +++ b/src/obikindex/src/index/dump.rs @@ -3,9 +3,9 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use rayon::prelude::*; -use crate::error::{OKIError, OKIResult}; -use crate::index::KmerIndex; -use crate::KmerFilter; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::kmer_index::KmerIndex; +use crate::index::KmerFilter; impl KmerIndex { /// Write a CSV table of all indexed kmers to `out`. diff --git a/src/obikindex/src/dump_layer.rs b/src/obikindex/src/index/dump_layer.rs similarity index 98% rename from src/obikindex/src/dump_layer.rs rename to src/obikindex/src/index/dump_layer.rs index d14b07f4..a4c77ff2 100644 --- a/src/obikindex/src/dump_layer.rs +++ b/src/obikindex/src/index/dump_layer.rs @@ -1,10 +1,10 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use obilayeredmap::{IndexMode, MphfLayer, OLMError}; +use crate::layer::{IndexMode, MphfLayer, OLMError}; use obiskio::{SKError, SKResult, UnitigFileReader}; -use crate::filter::{KmerFilter, passes_all}; -use crate::index::KmerIndex; +use crate::index::filter::{KmerFilter, passes_all}; +use crate::index::kmer_index::KmerIndex; fn olm_to_sk(e: OLMError) -> SKError { match e { diff --git a/src/obikindex/src/error.rs b/src/obikindex/src/index/error.rs similarity index 100% rename from src/obikindex/src/error.rs rename to src/obikindex/src/index/error.rs diff --git a/src/obikindex/src/filter.rs b/src/obikindex/src/index/filter.rs similarity index 100% rename from src/obikindex/src/filter.rs rename to src/obikindex/src/index/filter.rs diff --git a/src/obikindex/src/graph_pipeline.rs b/src/obikindex/src/index/graph_pipeline.rs similarity index 98% rename from src/obikindex/src/graph_pipeline.rs rename to src/obikindex/src/index/graph_pipeline.rs index 5b49a3c1..52ffed73 100644 --- a/src/obikindex/src/graph_pipeline.rs +++ b/src/obikindex/src/index/graph_pipeline.rs @@ -10,10 +10,10 @@ use obipipeline::{ use obidebruinj::GraphDeBruijn; use obikseq::CanonicalKmer; -use obilayeredmap::{IndexMode, TypedLayer}; +use crate::layer::{IndexMode, TypedLayer}; use obiskio::{SKError, SKResult}; -use crate::common::olm_to_sk; +use crate::index::common::olm_to_sk; // ── KmerGraphData ───────────────────────────────────────────────────────────── diff --git a/src/obikindex/src/index_layer.rs b/src/obikindex/src/index/index_layer.rs similarity index 91% rename from src/obikindex/src/index_layer.rs rename to src/obikindex/src/index/index_layer.rs index f72cbfe0..bf234c9e 100644 --- a/src/obikindex/src/index_layer.rs +++ b/src/obikindex/src/index/index_layer.rs @@ -5,14 +5,14 @@ use cacheline_ef::{CachelineEf, CachelineEfVec}; use epserde::prelude::*; use obicompactvec::{PersistentCompactIntMatrix, PersistentCompactIntVec}; use obidebruinj::GraphDeBruijn; -use obilayeredmap::meta::PartitionMeta; -use obilayeredmap::{IndexMode, layer::TypedLayer}; +use crate::layer::meta::PartitionMeta; +use crate::layer::{IndexMode, TypedLayer}; use obiskio::{SKError, SKFileMeta, SKFileReader}; use ptr_hash::{PtrHash, bucket_fn::CubicEps, hash::Xx64}; -use crate::common::olm_to_sk; -use crate::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; -use crate::index::KmerIndex; +use crate::index::common::olm_to_sk; +use crate::index::graph_pipeline::{materialize_layer, write_graph_as_unitigs}; +use crate::index::kmer_index::KmerIndex; type Mphf = PtrHash>, Xx64, Vec>; @@ -43,7 +43,7 @@ impl KmerIndex { block_bits: u8, ) -> Result { let layer0_dir = self.layer_dir(i, 0); - let dedup_path = obilayeredmap::dereplicated_superkmers_path(&layer0_dir); + let dedup_path = crate::layer::dereplicated_superkmers_path(&layer0_dir); if !dedup_path.exists() { return Ok(0); } @@ -123,7 +123,7 @@ impl KmerIndex { /// Deletes `dereplicated.skmer.zst` (+ sidecar), `mphf1.bin`, `counts1.bin`. pub fn remove_build_artifacts(&self, i: usize) { let layer0_dir = self.layer_dir(i, 0); - let dedup = obilayeredmap::dereplicated_superkmers_path(&layer0_dir); + let dedup = crate::layer::dereplicated_superkmers_path(&layer0_dir); remove_if_exists(&SKFileMeta::sidecar_path(&dedup)); remove_if_exists(&dedup); remove_if_exists(&layer0_dir.join("mphf1.bin")); diff --git a/src/obikindex/src/index.rs b/src/obikindex/src/index/kmer_index.rs similarity index 88% rename from src/obikindex/src/index.rs rename to src/obikindex/src/index/kmer_index.rs index 3c9321a6..d1259ac1 100644 --- a/src/obikindex/src/index.rs +++ b/src/obikindex/src/index/kmer_index.rs @@ -2,17 +2,17 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; -use obilayeredmap::meta::PartitionMeta; +use crate::layer::meta::PartitionMeta; use obisys::{Reporter, Stage, progress_bar}; use rayon::prelude::*; use tracing::info; use obikseq::{set_k, set_m}; -use crate::common::load_meta; -use crate::error::{OKIError, OKIResult}; -use crate::meta::{GenomeInfo, IndexConfig, IndexMeta}; -use crate::state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED}; +use crate::index::common::load_meta; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::meta::{GenomeInfo, IndexConfig, IndexMeta}; +use crate::index::state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED}; pub struct KmerIndex { pub(crate) root_path: PathBuf, @@ -30,6 +30,7 @@ impl KmerIndex { genome_info: Option, ) -> OKIResult { let root_path = path.as_ref().to_owned(); + fs::create_dir_all(&root_path).map_err(OKIError::Io)?; set_k(config.kmer_size); set_m(config.minimizer_size); let mut meta = IndexMeta::new(config); @@ -130,7 +131,7 @@ impl KmerIndex { pub fn with_counts(&self) -> bool { self.meta.config.with_counts } - pub fn evidence_mode(&self) -> &obilayeredmap::IndexMode { + pub fn evidence_mode(&self) -> &crate::layer::IndexMode { &self.meta.config.evidence } /// Coarse evidence mode (`Exact`/`Approx`/`Hybrid`, no `b`/`z` @@ -138,11 +139,11 @@ impl KmerIndex { /// full `IndexMode`. Same discriminant as `Layer::evidence_kind`, so a /// layer's own evidence kind can be compared directly against the /// index's. - pub fn evidence_kind(&self) -> obilayeredmap::EvidenceKind { + pub fn evidence_kind(&self) -> crate::layer::EvidenceKind { match self.meta.config.evidence { - obilayeredmap::IndexMode::Exact => obilayeredmap::EvidenceKind::Exact, - obilayeredmap::IndexMode::Approx { .. } => obilayeredmap::EvidenceKind::Approx, - obilayeredmap::IndexMode::Hybrid { .. } => obilayeredmap::EvidenceKind::Hybrid, + crate::layer::IndexMode::Exact => crate::layer::EvidenceKind::Exact, + crate::layer::IndexMode::Approx { .. } => crate::layer::EvidenceKind::Approx, + crate::layer::IndexMode::Hybrid { .. } => crate::layer::EvidenceKind::Hybrid, } } pub fn block_bits(&self) -> u8 { @@ -156,31 +157,32 @@ impl KmerIndex { } /// Path of partition `i`'s raw directory (`partitions/part_{i:05}`) — - /// delegates to `obikpartition`, the Partition tier's own naming - /// primitive (mirrors `layer_dir` delegating to `obilayeredmap`). - /// `obikpartitionner::PartitionRouter` reaches this same directory - /// only indirectly, through this method (it depends on `KmerIndex`, - /// not the other way around — see - /// `DevDocMD/implementation/partition_layer_cache.md`). + /// delegates to `crate::partition`, the Partition tier's own naming + /// primitive (mirrors `layer_dir` delegating to `crate::layer`). + /// `crate::algorithms::partitionner::PartitionRouter` reaches this same + /// directory only indirectly, through this method — it depends on + /// `KmerIndex`, not the other way around, even though both now live in + /// this crate — see `DevDocMD/implementation/partition_layer_cache.md`. pub fn partition_dir(&self, i: usize) -> PathBuf { - obikpartition::partition_dir(&self.root_path, i) + crate::partition::partition_dir(&self.root_path, i) } /// Path of partition `i`'s layered-index directory (`/index`). pub fn index_dir(&self, i: usize) -> PathBuf { - obikpartition::index_dir(&self.root_path, i) + crate::partition::index_dir(&self.root_path, i) } /// Path of layer `l` within partition `i`'s layered index. pub fn layer_dir(&self, i: usize, l: usize) -> PathBuf { - obilayeredmap::layer_dir(&self.index_dir(i), l) + crate::layer::layer_dir(&self.index_dir(i), l) } /// Partition `i`'s metadata (layer count, evidence mode). Returns /// `obiskio::SKResult`, not `OKIResult` — matches the error convention /// of the partition/layer-construction code below (moved here from - /// `obikpartitionner`, which predates `OKIError`); `?` still converts - /// it to `OKIResult` at any call site that needs one (`OKIError: From`). + /// the former `obikpartitionner` crate, which predates `OKIError`); + /// `?` still converts it to `OKIResult` at any call site that needs + /// one (`OKIError: From`). pub fn partition_meta(&self, i: usize) -> obiskio::SKResult { load_meta(&self.index_dir(i), "partition_meta") } @@ -195,7 +197,7 @@ impl KmerIndex { /// [`evidence_mode`](Self::evidence_mode): that one is the index-level /// config, this one is per-partition ground truth (the two agree in /// practice, but this is what `Layer::open` needs). - pub fn partition_mode(&self, i: usize) -> obiskio::SKResult { + pub fn partition_mode(&self, i: usize) -> obiskio::SKResult { Ok(self.partition_meta(i)?.mode) } @@ -230,10 +232,11 @@ impl KmerIndex { /// Write `spectrums/{label}.json` from an already-computed kmer /// spectrum (`f0`/`f1`/abundance histogram). Takes plain values rather - /// than `obikpartitionner::KmerSpectrum` — `KmerIndex` cannot depend on - /// `obikpartitionner` (that dependency runs the other way, see - /// `DevDocMD/implementation/partition_layer_cache.md`), and doesn't - /// need to: this is the only field of that type it actually uses. + /// than `crate::algorithms::partitionner::KmerSpectrum` — `KmerIndex` + /// is the data model, `PartitionRouter` the algorithm that depends on + /// it (see `DevDocMD/implementation/partition_layer_cache.md`), not + /// the other way around, and this is the only field of that type it + /// actually uses. pub fn write_spectrum(&self, f0: u64, f1: u64, counts: &BTreeMap) -> OKIResult<()> { let label = self .meta @@ -276,7 +279,7 @@ impl KmerIndex { let pb = progress_bar("index", n as u64, "partitions"); let order: Vec = (0..n).collect(); - let runner = crate::numa::PartitionRunner::new(); + let runner = crate::index::numa::PartitionRunner::new(); runner .run( &order, @@ -335,7 +338,7 @@ impl KmerIndex { let n = self.n_partitions(); let order: Vec = (0..n).collect(); let pb = progress_bar("pack", n as u64, "partitions"); - crate::numa::PartitionRunner::new().run( + crate::index::numa::PartitionRunner::new().run( &order, |i| -> OKIResult<()> { let index_dir = self.index_dir(i); diff --git a/src/obikindex/src/matrix_store.rs b/src/obikindex/src/index/matrix_store.rs similarity index 92% rename from src/obikindex/src/matrix_store.rs rename to src/obikindex/src/index/matrix_store.rs index 33ab9f83..0d92a5c1 100644 --- a/src/obikindex/src/matrix_store.rs +++ b/src/obikindex/src/index/matrix_store.rs @@ -1,9 +1,9 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; -use obilayeredmap::{LayeredStore, open_data}; +use crate::layer::{LayeredStore, open_data}; use obiskio::SKResult; -use crate::common::{load_meta, olm_to_sk}; -use crate::index::KmerIndex; +use crate::index::common::{load_meta, olm_to_sk}; +use crate::index::kmer_index::KmerIndex; impl KmerIndex { /// Open all count matrices for partition `part`, one per layer. diff --git a/src/obikindex/src/merge.rs b/src/obikindex/src/index/merge.rs similarity index 98% rename from src/obikindex/src/merge.rs rename to src/obikindex/src/index/merge.rs index b00e6f71..f9e328f1 100644 --- a/src/obikindex/src/merge.rs +++ b/src/obikindex/src/index/merge.rs @@ -6,14 +6,14 @@ use std::path::Path; use obisys::{Reporter, Stage, progress_bar, spinner}; use tracing::{debug, info}; -use obilayeredmap::IndexMode; +use crate::layer::IndexMode; -use crate::error::{OKIError, OKIResult}; -use crate::index::KmerIndex; -use crate::meta::{GenomeInfo, IndexMeta}; -use crate::state::{IndexState, SENTINEL_INDEXED}; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::kmer_index::KmerIndex; +use crate::index::meta::{GenomeInfo, IndexMeta}; +use crate::index::state::{IndexState, SENTINEL_INDEXED}; -pub use crate::merge_layer::MergeMode; +pub use crate::index::merge_layer::MergeMode; // ── per-partition diagnostic record ────────────────────────────────────────── @@ -217,7 +217,7 @@ impl KmerIndex { let srcs = &srcs; let evidence = &evidence; - let runner = crate::numa::PartitionRunner::new(); + let runner = crate::index::numa::PartitionRunner::new(); let mut part_stats: Vec = Vec::with_capacity(n_partitions); runner diff --git a/src/obikindex/src/merge_layer/mod.rs b/src/obikindex/src/index/merge_layer/mod.rs similarity index 99% rename from src/obikindex/src/merge_layer/mod.rs rename to src/obikindex/src/index/merge_layer/mod.rs index 4ef0a731..f8a35849 100644 --- a/src/obikindex/src/merge_layer/mod.rs +++ b/src/obikindex/src/index/merge_layer/mod.rs @@ -20,12 +20,12 @@ use tracing::debug; use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder}; use obikseq::CanonicalKmer; -use obilayeredmap::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir}; +use crate::layer::{IndexMode, TypedLayer, LayeredMap, MphfOnly, layer_dir}; use obiskio::{SKError, SKResult, UnitigFileReader}; -use crate::common::{ColBuilder, load_meta, olm_to_sk}; -use crate::graph_pipeline::{build_graph, materialize_layer}; -use crate::index::KmerIndex; +use crate::index::common::{ColBuilder, load_meta, olm_to_sk}; +use crate::index::graph_pipeline::{build_graph, materialize_layer}; +use crate::index::kmer_index::KmerIndex; mod src_layer; diff --git a/src/obikindex/src/merge_layer/src_layer.rs b/src/obikindex/src/index/merge_layer/src_layer.rs similarity index 98% rename from src/obikindex/src/merge_layer/src_layer.rs rename to src/obikindex/src/index/merge_layer/src_layer.rs index e7e1468f..4f104b6f 100644 --- a/src/obikindex/src/merge_layer/src_layer.rs +++ b/src/obikindex/src/index/merge_layer/src_layer.rs @@ -2,10 +2,10 @@ use std::path::Path; use obicompactvec::{MatrixGroupOps, PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use obilayeredmap::MphfOnly; +use crate::layer::MphfOnly; use obiskio::{SKError, SKResult}; -use crate::common::olm_to_sk; +use crate::index::common::olm_to_sk; use super::MergeMode; diff --git a/src/obikindex/src/meta.rs b/src/obikindex/src/index/meta.rs similarity index 98% rename from src/obikindex/src/meta.rs rename to src/obikindex/src/index/meta.rs index b0c5aa7e..e1987e50 100644 --- a/src/obikindex/src/meta.rs +++ b/src/obikindex/src/index/meta.rs @@ -3,7 +3,7 @@ use std::fs; use std::io; use std::path::Path; -use obilayeredmap::IndexMode; +use crate::layer::IndexMode; use serde::{Deserialize, Serialize}; pub const META_FILENAME: &str = "index.meta"; diff --git a/src/obikindex/src/index/mod.rs b/src/obikindex/src/index/mod.rs new file mode 100644 index 00000000..3fe91c5a --- /dev/null +++ b/src/obikindex/src/index/mod.rs @@ -0,0 +1,36 @@ +pub mod error; +pub mod meta; +pub mod predicate; +pub mod state; +mod common; +mod distance; +mod dump; +mod dump_layer; +pub mod filter; +mod graph_pipeline; +mod kmer_index; +mod index_layer; +mod matrix_store; +mod merge; +mod merge_layer; +mod numa; +mod query_layer; +mod rebuild; +mod rebuild_layer; +mod reindex; +mod select; +mod select_layer; +mod stats; + +pub use error::{OKIError, OKIResult}; +pub use distance::{DistanceMetric, DistanceOutput}; +pub use filter::{GroupQuorumFilter, KmerFilter, passes_all}; +pub use kmer_index::KmerIndex; +pub use merge_layer::MergeMode; +pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME}; +pub use predicate::{GroupFilterParams, MetaPred}; +pub use query_layer::{KmerDesc, QueryHit, QueryStats}; +pub use select_layer::{AggOp, OutputCol}; +pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED}; +pub use stats::IndexBitsPerKmer; +pub use numa::PartitionRunner; diff --git a/src/obikindex/src/numa/mod.rs b/src/obikindex/src/index/numa/mod.rs similarity index 100% rename from src/obikindex/src/numa/mod.rs rename to src/obikindex/src/index/numa/mod.rs diff --git a/src/obikindex/src/numa/runner.rs b/src/obikindex/src/index/numa/runner.rs similarity index 100% rename from src/obikindex/src/numa/runner.rs rename to src/obikindex/src/index/numa/runner.rs diff --git a/src/obikindex/src/numa/topology.rs b/src/obikindex/src/index/numa/topology.rs similarity index 100% rename from src/obikindex/src/numa/topology.rs rename to src/obikindex/src/index/numa/topology.rs diff --git a/src/obikindex/src/predicate.rs b/src/obikindex/src/index/predicate.rs similarity index 99% rename from src/obikindex/src/predicate.rs rename to src/obikindex/src/index/predicate.rs index dc808cd8..eef62ee7 100644 --- a/src/obikindex/src/predicate.rs +++ b/src/obikindex/src/index/predicate.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; -use crate::GroupQuorumFilter; +use crate::index::GroupQuorumFilter; use obitaxonomy::{TaxPath, TaxPattern}; -use crate::meta::{GenomeInfo, IndexMeta}; +use crate::index::meta::{GenomeInfo, IndexMeta}; // ── Operator ────────────────────────────────────────────────────────────────── diff --git a/src/obikindex/src/progress.rs b/src/obikindex/src/index/progress.rs similarity index 100% rename from src/obikindex/src/progress.rs rename to src/obikindex/src/index/progress.rs diff --git a/src/obikindex/src/query_layer.rs b/src/obikindex/src/index/query_layer.rs similarity index 99% rename from src/obikindex/src/query_layer.rs rename to src/obikindex/src/index/query_layer.rs index 41ff61b1..9fb8422e 100644 --- a/src/obikindex/src/query_layer.rs +++ b/src/obikindex/src/index/query_layer.rs @@ -3,10 +3,10 @@ use std::path::Path; use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use obilayeredmap::{IndexMode, MphfLayer, OLMError}; +use crate::layer::{IndexMode, MphfLayer, OLMError}; use obiskio::{SKError, SKResult}; -use crate::index::KmerIndex; +use crate::index::kmer_index::KmerIndex; fn olm_to_sk(e: OLMError) -> SKError { match e { diff --git a/src/obikindex/src/rebuild.rs b/src/obikindex/src/index/rebuild.rs similarity index 90% rename from src/obikindex/src/rebuild.rs rename to src/obikindex/src/index/rebuild.rs index 9c7ffb63..ce30f39d 100644 --- a/src/obikindex/src/rebuild.rs +++ b/src/obikindex/src/index/rebuild.rs @@ -1,13 +1,13 @@ use std::path::Path; -use crate::{KmerFilter, MergeMode}; +use crate::index::{KmerFilter, MergeMode}; use obisys::{Reporter, Stage, progress_bar}; use tracing::info; -use crate::error::{OKIError, OKIResult}; -use crate::index::KmerIndex; -use crate::meta::IndexMeta; -use crate::state::IndexState; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::kmer_index::KmerIndex; +use crate::index::meta::IndexMeta; +use crate::index::state::IndexState; impl KmerIndex { /// Rebuild `src` into a new compact single-layer index at `output`. @@ -62,7 +62,7 @@ impl KmerIndex { let block_bits = meta.config.block_bits; let order: Vec = (0..n_partitions).collect(); - let runner = crate::numa::PartitionRunner::new(); + let runner = crate::index::numa::PartitionRunner::new(); runner.run( &order, |i| dst_partition.rebuild_partition(src, i, filters, mode, n_genomes, block_bits), diff --git a/src/obikindex/src/rebuild_layer.rs b/src/obikindex/src/index/rebuild_layer.rs similarity index 96% rename from src/obikindex/src/rebuild_layer.rs rename to src/obikindex/src/index/rebuild_layer.rs index 32d74bcb..adaf6379 100644 --- a/src/obikindex/src/rebuild_layer.rs +++ b/src/obikindex/src/index/rebuild_layer.rs @@ -6,15 +6,15 @@ use obicompactvec::{ }; use obidebruinj::GraphDeBruijn; use obikseq::CanonicalKmer; -use obilayeredmap::meta::PartitionMeta; -use obilayeredmap::{IndexMode, MphfLayer, layer_dir}; +use crate::layer::meta::PartitionMeta; +use crate::layer::{IndexMode, MphfLayer, layer_dir}; use obiskio::{SKError, SKResult, UnitigFileReader}; -use crate::common::{load_meta, olm_to_sk}; -use crate::filter::KmerFilter; -use crate::graph_pipeline::materialize_layer; -use crate::merge_layer::{MergeMode, SrcLayerData}; -use crate::index::KmerIndex; +use crate::index::common::{load_meta, olm_to_sk}; +use crate::index::filter::KmerFilter; +use crate::index::graph_pipeline::materialize_layer; +use crate::index::merge_layer::{MergeMode, SrcLayerData}; +use crate::index::kmer_index::KmerIndex; // ── Builders — pair matrix builder + column builders for one mode ───────────── diff --git a/src/obikindex/src/reindex.rs b/src/obikindex/src/index/reindex.rs similarity index 92% rename from src/obikindex/src/reindex.rs rename to src/obikindex/src/index/reindex.rs index 2d3dff3b..b5b5503c 100644 --- a/src/obikindex/src/reindex.rs +++ b/src/obikindex/src/index/reindex.rs @@ -1,18 +1,18 @@ -use obilayeredmap::{IndexMode, layer::TypedLayer}; +use crate::layer::{IndexMode, TypedLayer}; use obisys::{Reporter, Stage, progress_bar}; use std::fs; use std::path::Path; use tracing::info; -use crate::error::{OKIError, OKIResult}; -use crate::index::KmerIndex; -use crate::state::IndexState; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::kmer_index::KmerIndex; +use crate::index::state::IndexState; const EVIDENCE_FILE: &str = "evidence.bin"; const FINGERPRINT_FILE: &str = "fingerprint.bin"; const UNITIG_IDX_FILE: &str = "unitigs.bin.idx"; -fn olm_to_oki(e: obilayeredmap::OLMError) -> OKIError { +fn olm_to_oki(e: crate::layer::OLMError) -> OKIError { OKIError::InvalidInput(e.to_string()) } @@ -44,7 +44,7 @@ impl KmerIndex { let pb = progress_bar("reindex", n as u64, "partitions"); let order: Vec = (0..n).collect(); - let runner = crate::numa::PartitionRunner::new(); + let runner = crate::index::numa::PartitionRunner::new(); runner.run( &order, |i| { diff --git a/src/obikindex/src/select.rs b/src/obikindex/src/index/select.rs similarity index 92% rename from src/obikindex/src/select.rs rename to src/obikindex/src/index/select.rs index 72f95fa3..9e39ad87 100644 --- a/src/obikindex/src/select.rs +++ b/src/obikindex/src/index/select.rs @@ -1,13 +1,13 @@ use std::path::Path; -use crate::OutputCol; +use crate::index::OutputCol; use obisys::{Reporter, Stage, progress_bar}; use tracing::info; -use crate::error::{OKIError, OKIResult}; -use crate::index::KmerIndex; -use crate::meta::{GenomeInfo, IndexMeta}; -use crate::state::IndexState; +use crate::index::error::{OKIError, OKIResult}; +use crate::index::kmer_index::KmerIndex; +use crate::index::meta::{GenomeInfo, IndexMeta}; +use crate::index::state::IndexState; impl KmerIndex { /// Create a new index at `output` by projecting/aggregating the genome columns @@ -56,7 +56,7 @@ impl KmerIndex { let pb = progress_bar("select", n_partitions as u64, "partitions"); let order: Vec = (0..n_partitions).collect(); - let runner = crate::numa::PartitionRunner::new(); + let runner = crate::index::numa::PartitionRunner::new(); runner .run( &order, @@ -111,7 +111,7 @@ impl KmerIndex { let pb = progress_bar("select", n_partitions as u64, "partitions"); let order: Vec = (0..n_partitions).collect(); - let runner = crate::numa::PartitionRunner::new(); + let runner = crate::index::numa::PartitionRunner::new(); runner .run( &order, diff --git a/src/obikindex/src/select_layer.rs b/src/obikindex/src/index/select_layer.rs similarity index 99% rename from src/obikindex/src/select_layer.rs rename to src/obikindex/src/index/select_layer.rs index cc54f937..3087538c 100644 --- a/src/obikindex/src/select_layer.rs +++ b/src/obikindex/src/index/select_layer.rs @@ -6,10 +6,10 @@ use obicompactvec::{ ColGroup, MatrixGroupOps, PersistentBitMatrix, PersistentBitMatrixBuilder, PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder, }; -use obilayeredmap::OLMError; +use crate::layer::OLMError; use obiskio::{SKError, SKResult}; -use crate::index::KmerIndex; +use crate::index::kmer_index::KmerIndex; // ── AggOp ───────────────────────────────────────────────────────────────────── diff --git a/src/obikindex/src/state.rs b/src/obikindex/src/index/state.rs similarity index 97% rename from src/obikindex/src/state.rs rename to src/obikindex/src/index/state.rs index 4db50ce1..0e2f8b4e 100644 --- a/src/obikindex/src/state.rs +++ b/src/obikindex/src/index/state.rs @@ -1,6 +1,6 @@ use std::path::Path; -use crate::meta::META_FILENAME; +use crate::index::meta::META_FILENAME; pub const SENTINEL_SCATTERED: &str = "scatter.done"; pub const SENTINEL_COUNTED: &str = "count.done"; diff --git a/src/obikindex/src/stats.rs b/src/obikindex/src/index/stats.rs similarity index 96% rename from src/obikindex/src/stats.rs rename to src/obikindex/src/index/stats.rs index 822db9e5..74abfece 100644 --- a/src/obikindex/src/stats.rs +++ b/src/obikindex/src/index/stats.rs @@ -5,8 +5,8 @@ use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix}; use obicompactvec::traits::ColumnWeights; use rayon::prelude::*; -use crate::error::OKIResult; -use crate::index::KmerIndex; +use crate::index::error::OKIResult; +use crate::index::kmer_index::KmerIndex; /// Bits per kmer broken down by index component. pub struct IndexBitsPerKmer { @@ -153,12 +153,12 @@ impl KmerIndex { if this_layer_dir.join("counts").exists() && !this_layer_dir.join("presence").exists() { - match obilayeredmap::open_data::(&index_dir, l) { + match crate::layer::open_data::(&index_dir, l) { Ok(m) => Box::new(m), Err(_) => continue, } } else { - match obilayeredmap::open_data::(&index_dir, l) { + match crate::layer::open_data::(&index_dir, l) { Ok(m) => Box::new(m), Err(_) => continue, } diff --git a/src/obikindex/src/tests/query_layer.rs b/src/obikindex/src/index/tests/query_layer.rs similarity index 95% rename from src/obikindex/src/tests/query_layer.rs rename to src/obikindex/src/index/tests/query_layer.rs index dc273518..702da3af 100644 --- a/src/obikindex/src/tests/query_layer.rs +++ b/src/obikindex/src/index/tests/query_layer.rs @@ -1,5 +1,5 @@ use super::*; -use crate::meta::IndexConfig; +use crate::index::meta::IndexConfig; // ── QueryStats::AddAssign ─────────────────────────────────────────────────── @@ -50,7 +50,7 @@ fn query_partition_with_missing_index_dir_returns_default_stats() { minimizer_size: 9, n_bits: 2, with_counts: false, - evidence: obilayeredmap::IndexMode::Exact, + evidence: crate::layer::IndexMode::Exact, block_bits: 0, }; let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index"); @@ -78,7 +78,7 @@ fn query_partition_with_empty_kmers_is_a_noop() { minimizer_size: 9, n_bits: 2, with_counts: false, - evidence: obilayeredmap::IndexMode::Exact, + evidence: crate::layer::IndexMode::Exact, block_bits: 0, }; let index = KmerIndex::create(tmp.path().join("idx"), config, None).expect("create index"); diff --git a/src/obilayeredmap/src/content_layer.rs b/src/obikindex/src/layer/content_layer.rs similarity index 96% rename from src/obilayeredmap/src/content_layer.rs rename to src/obikindex/src/layer/content_layer.rs index 80eb65a0..bea0dd5e 100644 --- a/src/obilayeredmap/src/content_layer.rs +++ b/src/obikindex/src/layer/content_layer.rs @@ -11,7 +11,7 @@ //! This is exactly the shape `obikphylo::siblings::cache::Mat` used to //! reimplement locally, minus its one sibling-specific method //! (`iter_minorants_batch`, which stays an extension trait over there — -//! `obilayeredmap` has no business knowing about sibling annexes). +//! `obikindex::layer` has no business knowing about sibling annexes). //! //! `Layer` is meant to eventually represent a layer's *whole* life — //! empty shell, under construction, ready to read — not just the @@ -27,10 +27,10 @@ use std::path::{Path, PathBuf}; use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix}; use obikseq::CanonicalKmer; -use crate::error::OLMResult; -use crate::layer::{LayerContent, TypedLayer, COUNTS_DIR, PRESENCE_DIR}; -use crate::meta::IndexMode; -use crate::mphf_layer::{EvidenceKind, EVIDENCE_FILE, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE}; +use crate::layer::error::OLMResult; +use crate::layer::typed_layer::{LayerContent, TypedLayer, COUNTS_DIR, PRESENCE_DIR}; +use crate::layer::meta::IndexMode; +use crate::layer::mphf_layer::{EvidenceKind, EVIDENCE_FILE, FINGERPRINT_FILE, MPHF_FILE, UNITIGS_FILE}; /// One layer, at any point in its life — see the module docs. Only /// [`Empty`](Layer::Empty) and the two ready-to-read states diff --git a/src/obilayeredmap/src/error.rs b/src/obikindex/src/layer/error.rs similarity index 100% rename from src/obilayeredmap/src/error.rs rename to src/obikindex/src/layer/error.rs diff --git a/src/obilayeredmap/src/evidence.rs b/src/obikindex/src/layer/evidence.rs similarity index 96% rename from src/obilayeredmap/src/evidence.rs rename to src/obikindex/src/layer/evidence.rs index 3bc8e86e..48736e42 100644 --- a/src/obilayeredmap/src/evidence.rs +++ b/src/obikindex/src/layer/evidence.rs @@ -6,7 +6,7 @@ use std::path::Path; use memmap2::Mmap; -use crate::error::{OLMError, OLMResult}; +use crate::layer::error::{OLMError, OLMResult}; pub struct Evidence { mmap: Mmap, diff --git a/src/obilayeredmap/src/fingerprint.rs b/src/obikindex/src/layer/fingerprint.rs similarity index 99% rename from src/obilayeredmap/src/fingerprint.rs rename to src/obikindex/src/layer/fingerprint.rs index ff0701ee..2297fc15 100644 --- a/src/obilayeredmap/src/fingerprint.rs +++ b/src/obikindex/src/layer/fingerprint.rs @@ -14,7 +14,7 @@ use std::path::Path; use bitvec::prelude::*; use memmap2::Mmap; -use crate::error::{OLMError, OLMResult}; +use crate::layer::error::{OLMError, OLMResult}; const MAGIC: &[u8; 4] = b"FPVF"; const HEADER: usize = 16; diff --git a/src/obilayeredmap/src/layered_store.rs b/src/obikindex/src/layer/layered_store.rs similarity index 100% rename from src/obilayeredmap/src/layered_store.rs rename to src/obikindex/src/layer/layered_store.rs diff --git a/src/obilayeredmap/src/map.rs b/src/obikindex/src/layer/map.rs similarity index 95% rename from src/obilayeredmap/src/map.rs rename to src/obikindex/src/layer/map.rs index 891be1be..5e77c1cc 100644 --- a/src/obilayeredmap/src/map.rs +++ b/src/obikindex/src/layer/map.rs @@ -6,10 +6,10 @@ use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind use obikseq::CanonicalKmer; use obiskio::{UnitigFileWriter, DEFAULT_BLOCK_BITS}; -use crate::error::{OLMError, OLMResult}; -use crate::layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData}; -use crate::meta::{IndexMode, PartitionMeta}; -use crate::mphf_layer::EvidenceKind; +use crate::layer::error::{OLMError, OLMResult}; +use crate::layer::typed_layer::{layer_dir, Hit, TypedLayer, LayerContent, LayerData}; +use crate::layer::meta::{IndexMode, PartitionMeta}; +use crate::layer::mphf_layer::EvidenceKind; /// Layered kmer index for a single partition. /// diff --git a/src/obilayeredmap/src/meta.rs b/src/obikindex/src/layer/meta.rs similarity index 98% rename from src/obilayeredmap/src/meta.rs rename to src/obikindex/src/layer/meta.rs index ed567d37..8c4de76b 100644 --- a/src/obilayeredmap/src/meta.rs +++ b/src/obikindex/src/layer/meta.rs @@ -3,7 +3,7 @@ use std::path::Path; use serde::{Deserialize, Serialize}; -use crate::error::OLMResult; +use crate::layer::error::OLMResult; const META_FILE: &str = "meta.json"; diff --git a/src/obilayeredmap/src/lib.rs b/src/obikindex/src/layer/mod.rs similarity index 92% rename from src/obilayeredmap/src/lib.rs rename to src/obikindex/src/layer/mod.rs index 5ef4e24d..9c700889 100644 --- a/src/obilayeredmap/src/lib.rs +++ b/src/obikindex/src/layer/mod.rs @@ -2,7 +2,7 @@ pub mod content_layer; pub mod error; pub mod evidence; pub mod fingerprint; -pub mod layer; +pub mod typed_layer; pub mod layered_store; pub mod map; pub mod meta; @@ -10,7 +10,7 @@ pub(crate) mod mphf_layer; pub use content_layer::Layer; pub use error::{OLMError, OLMResult}; -pub use layer::{ +pub use typed_layer::{ dereplicated_superkmers_path, layer_dir, open_data, raw_superkmers_path, HasLayerContent, HasStorageKind, Hit, LayerContent, LayerData, TypedLayer, }; diff --git a/src/obilayeredmap/src/mphf_layer.rs b/src/obikindex/src/layer/mphf_layer.rs similarity index 99% rename from src/obilayeredmap/src/mphf_layer.rs rename to src/obikindex/src/layer/mphf_layer.rs index 53588151..993cc6df 100644 --- a/src/obilayeredmap/src/mphf_layer.rs +++ b/src/obikindex/src/layer/mphf_layer.rs @@ -10,10 +10,10 @@ use obikseq::CanonicalKmer; use obiskio::{CanonicalKmerIter, UnitigFileReader, UnitigFileWriter, build_unitig_idx}; use ptr_hash::{PtrHash, PtrHashParams, bucket_fn::CubicEps, hash::Xx64}; -use crate::error::{OLMError, OLMResult}; -use crate::evidence::{Evidence, EvidenceWriter}; -use crate::fingerprint::{FingerprintVec, FingerprintVecWriter}; -use crate::meta::IndexMode; +use crate::layer::error::{OLMError, OLMResult}; +use crate::layer::evidence::{Evidence, EvidenceWriter}; +use crate::layer::fingerprint::{FingerprintVec, FingerprintVecWriter}; +use crate::layer::meta::IndexMode; pub(crate) const MPHF_FILE: &str = "mphf.bin"; pub(crate) const UNITIGS_FILE: &str = "unitigs.bin"; diff --git a/src/obilayeredmap/src/tests/layered_store.rs b/src/obikindex/src/layer/tests/layered_store.rs similarity index 100% rename from src/obilayeredmap/src/tests/layered_store.rs rename to src/obikindex/src/layer/tests/layered_store.rs diff --git a/src/obilayeredmap/src/tests/map.rs b/src/obikindex/src/layer/tests/map.rs similarity index 98% rename from src/obilayeredmap/src/tests/map.rs rename to src/obikindex/src/layer/tests/map.rs index cb083f95..38bc6edd 100644 --- a/src/obilayeredmap/src/tests/map.rs +++ b/src/obikindex/src/layer/tests/map.rs @@ -2,9 +2,9 @@ use super::*; use obicompactvec::{pack_bit_matrix, pack_sparse_bit_matrix, PersistentBitMatrix, PersistentCompactIntMatrix, StorageKind}; use obikseq::{set_k, Sequence as _, Unitig}; use obiskio::DEFAULT_BLOCK_BITS; -use crate::layer::LayerContent; -use crate::meta::IndexMode; -use crate::mphf_layer::EvidenceKind; +use crate::layer::typed_layer::LayerContent; +use crate::layer::meta::IndexMode; +use crate::layer::mphf_layer::EvidenceKind; use tempfile::tempdir; fn push_unitigs_and_layer( @@ -111,7 +111,7 @@ fn push_layer_from_map_convenience() { fn write_unitigs_at(dir: &Path, seqs: &[&[u8]]) { fs::create_dir_all(dir).unwrap(); - let mut w = UnitigFileWriter::create(&dir.join(crate::layer::UNITIGS_FILE)).unwrap(); + let mut w = UnitigFileWriter::create(&dir.join(crate::layer::typed_layer::UNITIGS_FILE)).unwrap(); for s in seqs { w.write(&Unitig::from_ascii(s)).unwrap(); } diff --git a/src/obilayeredmap/src/tests/layer.rs b/src/obikindex/src/layer/tests/typed_layer.rs similarity index 98% rename from src/obilayeredmap/src/tests/layer.rs rename to src/obikindex/src/layer/tests/typed_layer.rs index e409e7b3..722e1490 100644 --- a/src/obilayeredmap/src/tests/layer.rs +++ b/src/obikindex/src/layer/tests/typed_layer.rs @@ -111,7 +111,7 @@ fn count_layer_reports_count_content_and_columnar_storage() { assert_eq!(layer.content(), LayerContent::Count); assert_eq!(layer.storage_kind(), obicompactvec::StorageKind::Columnar); - assert_eq!(layer.evidence_kind(), crate::mphf_layer::EvidenceKind::Exact); + assert_eq!(layer.evidence_kind(), crate::layer::mphf_layer::EvidenceKind::Exact); } #[test] diff --git a/src/obilayeredmap/src/layer.rs b/src/obikindex/src/layer/typed_layer.rs similarity index 93% rename from src/obilayeredmap/src/layer.rs rename to src/obikindex/src/layer/typed_layer.rs index 9656931c..a005e7dd 100644 --- a/src/obilayeredmap/src/layer.rs +++ b/src/obikindex/src/layer/typed_layer.rs @@ -11,10 +11,10 @@ use obicompactvec::{ use obikseq::CanonicalKmer; use obiskio::{UnitigFileReader, UnitigFileWriter}; -use crate::error::{OLMError, OLMResult}; -use crate::meta::IndexMode; -use crate::mphf_layer::MphfLayer; -pub(crate) use crate::mphf_layer::UNITIGS_FILE; +use crate::layer::error::{OLMError, OLMResult}; +use crate::layer::meta::IndexMode; +use crate::layer::mphf_layer::MphfLayer; +pub(crate) use crate::layer::mphf_layer::UNITIGS_FILE; pub(crate) const COUNTS_DIR: &str = "counts"; pub(crate) const PRESENCE_DIR: &str = "presence"; @@ -31,7 +31,7 @@ pub trait LayerData: Sized { /// single source of truth for the on-disk `layer_N` naming convention /// (mirrors what `LayeredMap::open`/`push_layer` already use internally). /// -/// `obilayeredmap` operates within a single partition's index root; it has +/// `obikindex::layer` operates within a single partition's index root; it has /// no notion of "partition" at all. Turning a partition number into that /// root is `obikindex::KmerIndex::index_dir`'s job, one layer up — callers /// here only ever name a layer *number*, never build the path themselves. @@ -46,20 +46,21 @@ const SK_EXT: &str = "skmer.zst"; /// Path of a layer's raw, not-yet-dereplicated superkmer file — written by /// whichever algorithm routes superkmers into this layer (today: -/// `obikpartitionner::PartitionRouter`), read by whichever algorithm -/// dereplicates it (today: `obikderep::Dereplicator`). Naming this once -/// here, rather than in either algorithm crate, is what lets two -/// independent crates agree on the filename without depending on each -/// other — see `DevDocMD/implementation/partition_layer_cache.md`. +/// `crate::algorithms::partitionner::PartitionRouter`), read by whichever +/// algorithm dereplicates it (today: +/// `crate::algorithms::dereplicator::Dereplicator`). Naming this once +/// here, rather than in either algorithm submodule, is what lets the two +/// agree on the filename without depending on each other directly — see +/// `DevDocMD/implementation/partition_layer_cache.md`. pub fn raw_superkmers_path(layer_dir: &Path) -> PathBuf { layer_dir.join(format!("raw.{SK_EXT}")) } /// Path of a layer's dereplicated superkmer file — written by -/// `obikderep::Dereplicator`, read by whichever algorithm counts kmer -/// abundances from it (today: `obikpartitionner::PartitionRouter:: -/// count_kmer`) and, later, by `obikindex::build_index_layer` to build the -/// real layer. +/// `crate::algorithms::dereplicator::Dereplicator`, read by whichever +/// algorithm counts kmer abundances from it (today: +/// `crate::algorithms::partitionner::PartitionRouter::count_kmer`) and, +/// later, by `obikindex::build_index_layer` to build the real layer. pub fn dereplicated_superkmers_path(layer_dir: &Path) -> PathBuf { layer_dir.join(format!("dereplicated.{SK_EXT}")) } @@ -226,18 +227,18 @@ impl TypedLayer { } /// Iterate over all canonical kmers in the layer, in deterministic order. - pub fn iter_kmers(&self) -> crate::mphf_layer::KmerIter { + pub fn iter_kmers(&self) -> crate::layer::mphf_layer::KmerIter { self.mphf.iter_kmers() } /// Iterate over all canonical kmers, each paired with its zero-based /// sequence index in `unitigs.bin`. - pub fn enumerate_kmers(&self) -> std::iter::Enumerate { + pub fn enumerate_kmers(&self) -> std::iter::Enumerate { self.mphf.enumerate_kmers() } /// Iterate over the layer's canonical kmers in batches of `n`. - pub fn iter_kmers_batch(&self, n: usize) -> crate::mphf_layer::KmerBatchIter { + pub fn iter_kmers_batch(&self, n: usize) -> crate::layer::mphf_layer::KmerBatchIter { self.mphf.iter_kmers_batch(n) } @@ -268,7 +269,7 @@ impl TypedLayer { /// This already-open layer's evidence mode — reads the discriminant /// already in memory (`MphfLayer`'s own `LayerEvidence`), no disk /// access. Available regardless of `D`, unlike `content`/`storage_kind`. - pub fn evidence_kind(&self) -> crate::mphf_layer::EvidenceKind { + pub fn evidence_kind(&self) -> crate::layer::mphf_layer::EvidenceKind { self.mphf.evidence_kind() } } @@ -456,5 +457,5 @@ impl TypedLayer { } #[cfg(test)] -#[path = "tests/layer.rs"] +#[path = "tests/typed_layer.rs"] mod tests; diff --git a/src/obikindex/src/lib.rs b/src/obikindex/src/lib.rs index f8f33005..1610030e 100644 --- a/src/obikindex/src/lib.rs +++ b/src/obikindex/src/lib.rs @@ -1,36 +1,22 @@ -pub mod error; -pub mod meta; -pub mod predicate; -pub mod state; -mod common; -mod distance; -mod dump; -mod dump_layer; -pub mod filter; -mod graph_pipeline; -mod index; -mod index_layer; -mod matrix_store; -mod merge; -mod merge_layer; -mod numa; -mod query_layer; -mod rebuild; -mod rebuild_layer; -mod reindex; -mod select; -mod select_layer; -mod stats; +//! The `Index { Partition { Layer } }` model — see +//! `DevDocMD/implementation/partition_layer_cache.md`'s "Definitions" +//! section for the authoritative naming/scope discussion. Each tier is a +//! submodule: [`layer`] (was the `obilayeredmap` crate), [`partition`] +//! (was the `obikpartition` crate), [`index`] (this crate's original +//! content). [`algorithms`] (was the `obikpartitionner` and `obikderep` +//! crates) sits alongside the model: code that builds or transforms an +//! index's content rather than defining its shape. -pub use error::{OKIError, OKIResult}; -pub use distance::{DistanceMetric, DistanceOutput}; -pub use filter::{GroupQuorumFilter, KmerFilter, passes_all}; -pub use index::KmerIndex; -pub use merge_layer::MergeMode; -pub use meta::{validate_label, GenomeInfo, IndexConfig, IndexMeta, META_FILENAME}; -pub use predicate::{GroupFilterParams, MetaPred}; -pub use query_layer::{KmerDesc, QueryHit, QueryStats}; -pub use select_layer::{AggOp, OutputCol}; -pub use state::{IndexState, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED}; -pub use stats::IndexBitsPerKmer; -pub use numa::PartitionRunner; +pub mod algorithms; +pub mod index; +pub mod layer; +pub mod partition; + +pub use index::{ + validate_label, AggOp, DistanceMetric, DistanceOutput, GenomeInfo, GroupFilterParams, + GroupQuorumFilter, IndexBitsPerKmer, IndexConfig, IndexMeta, IndexState, KmerDesc, KmerFilter, + KmerIndex, MergeMode, MetaPred, OKIError, OKIResult, OutputCol, PartitionRunner, QueryHit, + QueryStats, META_FILENAME, SENTINEL_COUNTED, SENTINEL_INDEXED, SENTINEL_SCATTERED, + passes_all, +}; +pub use index::{filter, meta}; diff --git a/src/obikpartition/src/lib.rs b/src/obikindex/src/partition/mod.rs similarity index 77% rename from src/obikpartition/src/lib.rs rename to src/obikindex/src/partition/mod.rs index 4bbc27cc..01b713c2 100644 --- a/src/obikpartition/src/lib.rs +++ b/src/obikindex/src/partition/mod.rs @@ -1,29 +1,28 @@ //! The **Partition** tier of the `Index { Partition { Layer } }` model — //! see `DevDocMD/implementation/partition_layer_cache.md`'s "Definitions" -//! section for the authoritative naming/scope discussion this crate -//! implements. `obilayeredmap` already holds the **Layer** tier as its own -//! crate rather than living inside `obikindex`; this crate holds the tier -//! above it the same way. +//! section for the authoritative naming/scope discussion this module +//! implements. [`crate::layer`] holds the **Layer** tier the same way; +//! this module holds the tier above it. //! //! [`KmerPartition`] represents one partition's already-open layers — a //! read cache, opened once and held for the run, not rebuilt per lookup. //! It does no path computation of its own beyond the shared -//! `obilayeredmap::layer_dir` naming primitive: which partition, which -//! `index_dir`, how many layers, and `IndexMode` are `KmerIndex`'s -//! (`obikindex`) job to resolve and hand in as plain arguments — this -//! crate depends only on `obilayeredmap` and below, never on `obikindex`, -//! so it cannot reach back for them itself. +//! `crate::layer::layer_dir` naming primitive: which partition, which +//! `index_dir`, how many layers, and `IndexMode` are `crate::index:: +//! KmerIndex`'s job to resolve and hand in as plain arguments — this +//! module depends only on `crate::layer` and below, never on +//! `crate::index`, so it cannot reach back for them itself. //! //! Not yet wired into any caller: `obikphylo::siblings::cache:: -//! PartitionCache` (`Vec>`) and -//! `obikindex::query_layer::QueryLayer` (uncached, bypasses `Layer` +//! PartitionCache` (`Vec>`) and +//! `crate::index::query_layer::QueryLayer` (uncached, bypasses `Layer` //! entirely) both still reinvent a fragment of this. Migrating them is a //! separate, deferred step. use std::path::{Path, PathBuf}; use obikseq::CanonicalKmer; -use obilayeredmap::{layer_dir, IndexMode, Layer, OLMResult}; +use crate::layer::{layer_dir, IndexMode, Layer, OLMResult}; /// Partition subdirectory name, under an index's root — the single source /// of truth for the on-disk `partitions/part_NNNNN` naming convention. @@ -40,7 +39,7 @@ pub fn partition_dir(root: &Path, i: usize) -> PathBuf { } /// Path of partition `i`'s layered-index directory — `/index`, -/// the root every [`obilayeredmap::layer_dir`] call for this partition is +/// the root every [`crate::layer::layer_dir`] call for this partition is /// relative to. pub fn index_dir(root: &Path, i: usize) -> PathBuf { partition_dir(root, i).join("index") diff --git a/src/obikpartition/src/tests.rs b/src/obikindex/src/partition/tests.rs similarity index 94% rename from src/obikpartition/src/tests.rs rename to src/obikindex/src/partition/tests.rs index 93027916..42e29d9e 100644 --- a/src/obikpartition/src/tests.rs +++ b/src/obikindex/src/partition/tests.rs @@ -11,7 +11,7 @@ fn write_layer(root: &Path, l: usize, seqs: &[&[u8]], n_genomes: usize, mode: &I w.write(&Unitig::from_ascii(s)).unwrap(); } w.close().unwrap(); - obilayeredmap::TypedLayer::::build_presence( + crate::layer::TypedLayer::::build_presence( &dir, DEFAULT_BLOCK_BITS, mode, @@ -23,7 +23,7 @@ fn write_layer(root: &Path, l: usize, seqs: &[&[u8]], n_genomes: usize, mode: &I #[test] fn open_reads_every_layer_in_order() { - // k=4, process-wide in test builds — see obilayeredmap's own tests for + // k=4, process-wide in test builds — see obikindex::layer's own tests for // why a fixed, unshared k matters here. set_k(4); let dir = tempdir().unwrap(); diff --git a/src/obikmer/Cargo.toml b/src/obikmer/Cargo.toml index 6ed3c6b3..99b559e7 100644 --- a/src/obikmer/Cargo.toml +++ b/src/obikmer/Cargo.toml @@ -15,14 +15,11 @@ obifastwrite = { path = "../obifastwrite" } obidebruinj = { path = "../obidebruinj" } obipipeline = { path = "../obipipeline" } obikrope = { path = "../obikrope" } -obikpartitionner = { path = "../obikpartitionner" } -obikderep = { path = "../obikderep" } obisys = { path = "../obisys" } obiskio = { path = "../obiskio" } obikindex = { path = "../obikindex", default-features = false } obikphylo = { path = "../obikphylo" } obitaxonomy = { path = "../obitaxonomy" } -obilayeredmap = { path = "../obilayeredmap" } clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/src/obikmer/src/cmd/index/mod.rs b/src/obikmer/src/cmd/index/mod.rs index 21cbee65..e4a67bdf 100644 --- a/src/obikmer/src/cmd/index/mod.rs +++ b/src/obikmer/src/cmd/index/mod.rs @@ -2,10 +2,10 @@ use std::path::PathBuf; use std::time::Instant; use clap::Args; -use obikderep::Dereplicator; +use obikindex::algorithms::dereplicator::Dereplicator; +use obikindex::algorithms::partitionner::PartitionRouter; use obikindex::{validate_label, GenomeInfo, IndexConfig, IndexState, KmerIndex}; -use obikpartitionner::PartitionRouter; -use obilayeredmap::IndexMode; +use obikindex::layer::IndexMode; fn parse_key_value(s: &str) -> Result<(String, String), String> { let pos = s.find('=').ok_or_else(|| format!("invalid key=value: no '=' in '{s}'"))?; @@ -188,50 +188,49 @@ pub fn run(args: IndexArgs) { }; // ── Open or create the index ───────────────────────────────────────────── - let mut idx = if KmerIndex::exists(&output) && !args.force { - info!("resuming from existing index at {}", output.display()); - KmerIndex::open(&output).unwrap_or_else(|e| { - eprintln!("error opening index: {e}"); + if KmerIndex::exists(&output) { + if !args.force { + eprintln!( + "error: an index already exists at {} (use --force to overwrite it)", + output.display() + ); std::process::exit(1); - }) - } else { - if args.force && KmerIndex::exists(&output) { - info!("--force: removing existing index at {}", output.display()); - std::fs::remove_dir_all(&output).unwrap_or_else(|e| { - eprintln!("error removing existing index: {e}"); - std::process::exit(1); - }); } - let n_bits = partitions_to_bits(args.common.partitions); - let effective = 1usize << n_bits; - if effective != args.common.partitions { - info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective); - } - let block_bits = block_size_to_bits(args.block_size); - let config = IndexConfig { - kmer_size: effective_kmer_size, - minimizer_size: args.common.minimizer_size, - n_bits, - with_counts: args.with_counts, - evidence: evidence.clone(), - block_bits, - }; - let genome_info = args.label.as_ref().map(|label| { - validate_label(label).unwrap_or_else(|e| { - eprintln!("error: --label: {e}"); - std::process::exit(1); - }); - let mut info = GenomeInfo::new(label.clone()); - for (k, v) in &args.meta { - info.meta.insert(k.clone(), v.clone()); - } - info + info!("--force: removing existing index at {}", output.display()); + std::fs::remove_dir_all(&output).unwrap_or_else(|e| { + eprintln!("error removing existing index: {e}"); + std::process::exit(1); }); - KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| { - eprintln!("error creating index: {e}"); - std::process::exit(1); - }) + } + let n_bits = partitions_to_bits(args.common.partitions); + let effective = 1usize << n_bits; + if effective != args.common.partitions { + info!("partitions: {} → {} (next power of 2)", args.common.partitions, effective); + } + let block_bits = block_size_to_bits(args.block_size); + let config = IndexConfig { + kmer_size: effective_kmer_size, + minimizer_size: args.common.minimizer_size, + n_bits, + with_counts: args.with_counts, + evidence: evidence.clone(), + block_bits, }; + let genome_info = args.label.as_ref().map(|label| { + validate_label(label).unwrap_or_else(|e| { + eprintln!("error: --label: {e}"); + std::process::exit(1); + }); + let mut info = GenomeInfo::new(label.clone()); + for (k, v) in &args.meta { + info.meta.insert(k.clone(), v.clone()); + } + info + }); + let mut idx = KmerIndex::create(&output, config, genome_info).unwrap_or_else(|e| { + eprintln!("error creating index: {e}"); + std::process::exit(1); + }); // ── Stage 1: scatter ───────────────────────────────────────────────────── diff --git a/src/obikmer/src/cmd/query/mod.rs b/src/obikmer/src/cmd/query/mod.rs index 425342b8..ed1d9af1 100644 --- a/src/obikmer/src/cmd/query/mod.rs +++ b/src/obikmer/src/cmd/query/mod.rs @@ -13,7 +13,7 @@ use std::time::Instant; use clap::Args; use obikindex::KmerIndex; use obikrope::Rope; -use obilayeredmap::IndexMode; +use obikindex::layer::IndexMode; use obipipeline::{Throttled, ThrottleGuard, throttle}; use obiread::chunk::read_sequence_chunks_sized; use obisys::{Reporter, Stage, available_memory_bytes, spinner}; diff --git a/src/obikmer/src/cmd/reindex/mod.rs b/src/obikmer/src/cmd/reindex/mod.rs index 3fd2c650..25eab1ea 100644 --- a/src/obikmer/src/cmd/reindex/mod.rs +++ b/src/obikmer/src/cmd/reindex/mod.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use clap::Args; use obikindex::KmerIndex; -use obilayeredmap::IndexMode; +use obikindex::layer::IndexMode; use obisys::Reporter; use tracing::info; diff --git a/src/obikpartition/Cargo.toml b/src/obikpartition/Cargo.toml deleted file mode 100644 index 2ffbe8eb..00000000 --- a/src/obikpartition/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "obikpartition" -version = "0.1.0" -edition = "2024" - -[dependencies] -obikseq = { path = "../obikseq" } -obilayeredmap = { path = "../obilayeredmap" } - -[dev-dependencies] -tempfile = "3" -obikseq = { path = "../obikseq", features = ["test-utils"] } -obiskio = { path = "../obiskio" } -obicompactvec = { path = "../obicompactvec" } diff --git a/src/obikpartitionner/Cargo.toml b/src/obikpartitionner/Cargo.toml deleted file mode 100644 index 565efce5..00000000 --- a/src/obikpartitionner/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "obikpartitionner" -version = "0.1.0" -edition = "2024" - -[dev-dependencies] -tempfile = "3" -obikseq = { path = "../obikseq", features = ["test-utils"] } -obikrope = { path = "../obikrope" } -obikderep = { path = "../obikderep" } - -[dependencies] -niffler = "3.0.0" -obikseq = { path = "../obikseq" } -obikindex = { path = "../obikindex" } -obilayeredmap = { path = "../obilayeredmap" } -obipipeline = { path = "../obipipeline" } -obiread = { path = "../obiread" } -obiskbuilder = { path = "../obiskbuilder" } -obiskio = { path = "../obiskio" } -rayon = "1" -sysinfo = "0.39" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -tracing = "0.1.44" -cacheline-ef = "1.1" -epserde = "0.8" -memmap2 = "0.9.10" -obicompactvec = { path = "../obicompactvec" } -ptr_hash = "1.1" -obisys = { path = "../obisys" } diff --git a/src/obikpartitionner/src/lib.rs b/src/obikpartitionner/src/lib.rs deleted file mode 100644 index 0e4f7638..00000000 --- a/src/obikpartitionner/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod kmer_sort; -mod partition; - -pub use partition::{KmerSpectrum, PartitionRouter}; diff --git a/src/obikpartitionner/src/partition/mod.rs b/src/obikpartitionner/src/partition/mod.rs deleted file mode 100644 index 489c2074..00000000 --- a/src/obikpartitionner/src/partition/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! K-mer partitioning: routing super-kmers into per-partition, layer-0 -//! files, and counting unique canonical k-mers. Dereplication itself moved -//! to `obikderep` (2026-08-20) — see -//! `DevDocMD/implementation/partition_layer_cache.md`: one algorithm, one -//! crate, split off one at a time rather than all at once. -//! -//! Submodules: [`router`] (`PartitionRouter`, `KmerSpectrum`, the -//! routing/counting lifecycle API — partition/layer path naming itself -//! lives on `obikindex::KmerIndex`, not here), [`count`] (unique-kmer -//! enumeration, MPHF, abundance counting). - -mod count; -mod router; - -#[cfg(test)] -mod tests; - -pub use router::{KmerSpectrum, PartitionRouter}; diff --git a/src/obikphylo/Cargo.toml b/src/obikphylo/Cargo.toml index 2e986665..10f9f47d 100644 --- a/src/obikphylo/Cargo.toml +++ b/src/obikphylo/Cargo.toml @@ -6,11 +6,9 @@ edition = "2024" [dependencies] obikindex = { path = "../obikindex", default-features = false } obikseq = { path = "../obikseq" } -obikpartitionner = { path = "../obikpartitionner" } obiskio = { path = "../obiskio" } obisys = { path = "../obisys" } obicompactvec = { path = "../obicompactvec" } -obilayeredmap = { path = "../obilayeredmap" } obiskbuilder = { path = "../obiskbuilder" } obipipeline = { path = "../obipipeline" } memmap2 = "0.9" @@ -21,6 +19,5 @@ tracing = "0.1.44" [dev-dependencies] obiread = { path = "../obiread" } -obikderep = { path = "../obikderep" } tempfile = "3" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } diff --git a/src/obikphylo/src/lib.rs b/src/obikphylo/src/lib.rs index fc921e6e..53a34bab 100644 --- a/src/obikphylo/src/lib.rs +++ b/src/obikphylo/src/lib.rs @@ -1,5 +1,5 @@ //! Library-level phylogenetic functionality for `obikmer`, built as -//! extension traits over `obikindex::KmerIndex` and `obilayeredmap`'s +//! extension traits over `obikindex::KmerIndex` and `obikindex::layer`'s //! generic layer types — the `phylo` CLI command is a consumer of this //! crate, not the owner of this logic (see `DevDocMD/architecture/siblings.md`). //! diff --git a/src/obikphylo/src/siblings/build.rs b/src/obikphylo/src/siblings/build.rs index e400ac8a..7760d684 100644 --- a/src/obikphylo/src/siblings/build.rs +++ b/src/obikphylo/src/siblings/build.rs @@ -5,8 +5,8 @@ use std::sync::atomic::Ordering; use rayon::prelude::*; use obikseq::CanonicalKmer; -use obilayeredmap::MphfLayer; -use obilayeredmap::meta::IndexMode; +use obikindex::layer::MphfLayer; +use obikindex::layer::meta::IndexMode; use obipipeline::ThrottleGuard; use obisys::progress_bar; diff --git a/src/obikphylo/src/siblings/cache.rs b/src/obikphylo/src/siblings/cache.rs index 1381fbd0..03450e47 100644 --- a/src/obikphylo/src/siblings/cache.rs +++ b/src/obikphylo/src/siblings/cache.rs @@ -1,7 +1,7 @@ use rayon::prelude::*; use obikseq::CanonicalKmer; -use obilayeredmap::Layer; +use obikindex::layer::Layer; use obisys::progress_bar; use obikindex::{KmerIndex, OKIResult}; @@ -24,7 +24,7 @@ use obikindex::{KmerIndex, OKIResult}; /// partition for the entire run, regardless of how many source layers or /// lookups follow. /// -/// `mats[partition][layer]` holds `obilayeredmap::Layer` — the +/// `mats[partition][layer]` holds `obikindex::layer::Layer` — the /// format-erased `Count`/`Presence` handle, not a sibling-specific type: /// this module used to bundle its own `Mat` enum here, duplicating exactly /// what `Layer` now does one crate down (see diff --git a/src/obikphylo/src/siblings/family_scan.rs b/src/obikphylo/src/siblings/family_scan.rs index ee55e643..678449d6 100644 --- a/src/obikphylo/src/siblings/family_scan.rs +++ b/src/obikphylo/src/siblings/family_scan.rs @@ -54,13 +54,13 @@ use std::sync::atomic::{AtomicU8, Ordering}; use rayon::prelude::*; use obikseq::CanonicalKmer; -use obilayeredmap::meta::PartitionMeta; +use obikindex::layer::meta::PartitionMeta; use obipipeline::{ThrottleGuard, throttle}; use obikindex::{OKIError, OKIResult}; use obikindex::KmerIndex; -use obilayeredmap::Layer; +use obikindex::layer::Layer; use super::cache::PartitionCache; use super::helpers::central_base; diff --git a/src/obikphylo/src/siblings/iter.rs b/src/obikphylo/src/siblings/iter.rs index 7e69ca61..4c9b9148 100644 --- a/src/obikphylo/src/siblings/iter.rs +++ b/src/obikphylo/src/siblings/iter.rs @@ -1,6 +1,6 @@ //! Phylo/sibling-domain iteration over a layer — an extension trait, not a //! new field on `MphfLayer`/`TypedLayer`: "family"/"minorant" are phylo -//! concepts, `obilayeredmap` stays kmer/slot-mapping only (see +//! concepts, `obikindex::layer` stays kmer/slot-mapping only (see //! `DevDocMD/architecture/siblings.md`). //! //! The sibling annex is persisted in the same order as `iter_kmers()` @@ -12,7 +12,7 @@ //! batch — never collected whole into memory (see the project's "no full //! collect" rule). //! -//! Four iterator types, deliberately mirroring `obilayeredmap`'s own +//! Four iterator types, deliberately mirroring `obikindex::layer`'s own //! `KmerIter`/`KmerBatchIter` pair (single item vs. `Vec` batch) — plus the //! minorant-filtered variant of each, since "all siblings" and "one row per //! family" are both common cases: @@ -29,7 +29,7 @@ use std::sync::Arc; use obikseq::CanonicalKmer; -use obilayeredmap::{KmerIter, TypedLayer, LayerData}; +use obikindex::layer::{KmerIter, TypedLayer, LayerData}; use super::{FamilyMask, SiblingAnnex}; @@ -172,42 +172,42 @@ impl SiblingLayerExt for TypedLayer { } } -/// Same extension, over `obilayeredmap::Layer` (the format-erased +/// Same extension, over `obikindex::layer::Layer` (the format-erased /// `Count`/`Presence` handle `PartitionCache` actually holds) — dispatch /// only, both arms return the same concrete iterator types (they don't -/// depend on which `D` is inside), so no boxing is needed. `obilayeredmap` +/// depend on which `D` is inside), so no boxing is needed. `obikindex::layer` /// itself can't implement this: "family"/"minorant" are phylo concepts, it /// stays kmer/slot-mapping only (see the module docs above). -impl SiblingLayerExt for obilayeredmap::Layer { +impl SiblingLayerExt for obikindex::layer::Layer { fn iter_siblings(&self, annex: Arc) -> SiblingIter { match self { - obilayeredmap::Layer::Count(l) => l.iter_siblings(annex), - obilayeredmap::Layer::Presence(l) => l.iter_siblings(annex), - obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"), + obikindex::layer::Layer::Count(l) => l.iter_siblings(annex), + obikindex::layer::Layer::Presence(l) => l.iter_siblings(annex), + obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings() called on an Empty layer"), } } fn iter_siblings_batch(&self, annex: Arc, batch_size: usize) -> SiblingBatchIter { match self { - obilayeredmap::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size), - obilayeredmap::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size), - obilayeredmap::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"), + obikindex::layer::Layer::Count(l) => l.iter_siblings_batch(annex, batch_size), + obikindex::layer::Layer::Presence(l) => l.iter_siblings_batch(annex, batch_size), + obikindex::layer::Layer::Empty { .. } => panic!("iter_siblings_batch() called on an Empty layer"), } } fn iter_minorants(&self, annex: Arc) -> MinorantIter { match self { - obilayeredmap::Layer::Count(l) => l.iter_minorants(annex), - obilayeredmap::Layer::Presence(l) => l.iter_minorants(annex), - obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"), + obikindex::layer::Layer::Count(l) => l.iter_minorants(annex), + obikindex::layer::Layer::Presence(l) => l.iter_minorants(annex), + obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants() called on an Empty layer"), } } fn iter_minorants_batch(&self, annex: Arc, batch_size: usize) -> MinorantBatchIter { match self { - obilayeredmap::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size), - obilayeredmap::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size), - obilayeredmap::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"), + obikindex::layer::Layer::Count(l) => l.iter_minorants_batch(annex, batch_size), + obikindex::layer::Layer::Presence(l) => l.iter_minorants_batch(annex, batch_size), + obikindex::layer::Layer::Empty { .. } => panic!("iter_minorants_batch() called on an Empty layer"), } } } diff --git a/src/obikphylo/src/siblings/mod.rs b/src/obikphylo/src/siblings/mod.rs index 40458964..6556bedd 100644 --- a/src/obikphylo/src/siblings/mod.rs +++ b/src/obikphylo/src/siblings/mod.rs @@ -71,7 +71,7 @@ pub(crate) use siblingannex::{FamilyMask, SiblingAnnex, SiblingAnnexBuilder}; pub use stats::{SiblingAnnexStats, SiblingStatsExt}; pub use subsample::EntropyBias; -use obilayeredmap::OLMError; +use obikindex::layer::OLMError; use obikindex::OKIError; diff --git a/src/obikphylo/src/siblings/tests.rs b/src/obikphylo/src/siblings/tests.rs index db332f64..c0a2504d 100644 --- a/src/obikphylo/src/siblings/tests.rs +++ b/src/obikphylo/src/siblings/tests.rs @@ -2,13 +2,13 @@ use std::io::Write; use std::path::Path; use obikseq::{CanonicalKmer, Kmer, Sequence}; -use obilayeredmap::MphfLayer; +use obikindex::layer::MphfLayer; use obisys::Reporter; use tempfile::tempdir; -use obikderep::Dereplicator; +use obikindex::algorithms::dereplicator::Dereplicator; +use obikindex::algorithms::partitionner::PartitionRouter; use obikindex::{GenomeInfo, IndexConfig, KmerIndex, MergeMode}; -use obikpartitionner::PartitionRouter; use super::alignment::SnpAlignmentExt; use super::build::SiblingAnnexBuildExt; @@ -63,7 +63,7 @@ fn build_single_genome_index(dir: &Path, label: &str, seq: &[u8]) -> KmerIndex { minimizer_size: M, n_bits: 0, // 1 partition — keeps the test deterministic and simple with_counts: false, - evidence: obilayeredmap::IndexMode::Exact, + evidence: obikindex::layer::IndexMode::Exact, block_bits: 0, }; let mut idx = KmerIndex::create(&index_path, config, Some(GenomeInfo::new(label))) @@ -169,7 +169,7 @@ fn sibling_annex_works_after_pack_sparse() { // is_multi.prsb`) and the generic `TypedLayer` methods it relies on // actually round-trip through the real build pipeline, not just the // unit-level `TypedLayer` tests in - // `obilayeredmap`. + // `obikindex::layer`. let dir = tempdir().unwrap(); let g1 = build_single_genome_index(dir.path(), "g1", b"AACCGCTTAAG"); let g2 = build_single_genome_index(dir.path(), "g2", b"AACCGGTTAAG"); diff --git a/src/obilayeredmap/Cargo.toml b/src/obilayeredmap/Cargo.toml deleted file mode 100644 index ef573ef9..00000000 --- a/src/obilayeredmap/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "obilayeredmap" -version = "0.1.0" -edition = "2024" - -[dependencies] -obikseq = { path = "../obikseq" } -obiskio = { path = "../obiskio" } -obicompactvec = { path = "../obicompactvec" } -ptr_hash = "1.1" -cacheline-ef = "1.1" -epserde = "0.8" -rayon = "1" -ndarray = "0.17" -bitvec = "1" -memmap2 = "0.9" -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -[dev-dependencies] -tempfile = "3" -obikseq = { path = "../obikseq", features = ["test-utils"] }